def run_forced_phot(cat, tim, ceres=True, derivs=False, agn=False, do_forced=True, do_apphot=True, get_model=False, ps=None, timing=False, fixed_also=False, ceres_threads=1): ''' fixed_also: if derivs=True, also run without derivatives and report that flux too? ''' if timing: tlast = Time() if ps is not None: import pylab as plt opti = None forced_kwargs = {} if ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 try: opti = CeresOptimizer(BW=B, BH=B, threads=ceres_threads) except: if ceres_threads > 1: raise RuntimeError( 'ceres_threads requested but not supported by tractor.ceres version' ) opti = CeresOptimizer(BW=B, BH=B) #forced_kwargs.update(verbose=True) # nsize = 0 for src in cat: # Limit sizes of huge models # from tractor.galaxy import ProfileGalaxy # if isinstance(src, ProfileGalaxy): # px,py = tim.wcs.positionToPixel(src.getPosition()) # h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval) # MAXHALF = 128 # if h > MAXHALF: # #print('halfsize', h,'for',src,'-> setting to',MAXHALF) # nsize += 1 # src.halfsize = MAXHALF src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) #print('Limited the size of', nsize, 'large galaxy models') if derivs: realsrcs = [] derivsrcs = [] Iderivs = [] for i, src in enumerate(cat): from tractor import PointSource realsrcs.append(src) if not isinstance(src, PointSource): continue Iderivs.append(i) brightness_dra = src.getBrightness().copy() brightness_ddec = src.getBrightness().copy() brightness_dra.setParams(np.zeros(brightness_dra.numberOfParams())) brightness_ddec.setParams( np.zeros(brightness_ddec.numberOfParams())) brightness_dra.freezeAllBut(tim.band) brightness_ddec.freezeAllBut(tim.band) dsrc = SourceDerivatives(src, [brightness_dra, brightness_ddec], tim, ps) derivsrcs.append(dsrc) Iderivs = np.array(Iderivs) if fixed_also: pass else: # For convenience, put all the real sources at the front of # the list, so we can pull the IVs off the front of the list. cat = realsrcs + derivsrcs if agn: from tractor.galaxy import ExpGalaxy, DevGalaxy, FixedCompositeGalaxy from tractor import PointSource from legacypipe.survey import SimpleGalaxy, RexGalaxy realsrcs = [] agnsrcs = [] iagn = [] for i, src in enumerate(cat): realsrcs.append(src) ## ?? if isinstance(src, (SimpleGalaxy, RexGalaxy)): #print('Skipping SIMP or REX:', src) continue if isinstance(src, (ExpGalaxy, DevGalaxy, FixedCompositeGalaxy)): iagn.append(i) bright = src.getBrightness().copy() bright.setParams(np.zeros(bright.numberOfParams())) bright.freezeAllBut(tim.band) agn = PointSource(src.pos, bright) agn.freezeAllBut('brightness') #print('Adding "agn"', agn, 'to', src) #print('agn params:', agn.getParamNames()) agnsrcs.append(src) iagn = np.array(iagn) cat = realsrcs + agnsrcs print('Added AGN to', len(iagn), 'galaxies') tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') disable_galaxy_cache() F = fits_table() if do_forced: if timing and (derivs or agn): t = Time() print('Setting up:', t - tlast) tlast = t if derivs: if fixed_also: print('Forced photom with fixed positions:') R = tr.optimize_forced_photometry(variance=True, fitstats=False, shared_params=False, priors=False, **forced_kwargs) F.flux_fixed = np.array([ src.getBrightness().getFlux(tim.band) for src in cat ]).astype(np.float32) N = len(cat) F.flux_fixed_ivar = R.IV[:N].astype(np.float32) if timing: t = Time() print('Forced photom with fixed positions finished:', t - tlast) tlast = t cat = realsrcs + derivsrcs tr.setCatalog(Catalog(*cat)) print('Forced photom with position derivatives:') if ps is None and not get_model: forced_kwargs.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, priors=False, **forced_kwargs) if ps is not None or get_model: (data, mod, ie, chi, roi) = R.ims1[0] if ps is not None: ima = dict(vmin=-2. * tim.sig1, vmax=5. * tim.sig1, interpolation='nearest', origin='lower', cmap='gray') imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5, cmap='RdBu') plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() if derivs: trx = Tractor([tim], realsrcs) trx.freezeParam('images') modx = trx.getModelImage(0) chix = (data - modx) * tim.getInvError() plt.clf() plt.imshow(modx, **ima) plt.title('Model without derivatives: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chix, **imchi) plt.title('Chi without derivatives: %s' % tim.name) ps.savefig() if derivs or agn: cat = realsrcs N = len(cat) F.flux = np.array([ src.getBrightness().getFlux(tim.band) for src in cat ]).astype(np.float32) F.flux_ivar = R.IV[:N].astype(np.float32) F.fracflux = R.fitstats.profracflux[:N].astype(np.float32) F.rchisq = R.fitstats.prochi2[:N].astype(np.float32) try: F.fracmasked = R.fitstats.promasked[:N].astype(np.float32) except: print( 'No "fracmasked" available (only in recent Tractor versions)') if derivs: F.flux_dra = np.zeros(len(F), np.float32) F.flux_ddec = np.zeros(len(F), np.float32) F.flux_dra[Iderivs] = np.array( [src.getParams()[0] for src in derivsrcs]).astype(np.float32) F.flux_ddec[Iderivs] = np.array( [src.getParams()[1] for src in derivsrcs]).astype(np.float32) F.flux_dra_ivar = np.zeros(len(F), np.float32) F.flux_ddec_ivar = np.zeros(len(F), np.float32) F.flux_dra_ivar[Iderivs] = R.IV[N::2].astype(np.float32) F.flux_ddec_ivar[Iderivs] = R.IV[N + 1::2].astype(np.float32) if agn: F.flux_agn = np.zeros(len(F), np.float32) F.flux_agn_ivar = np.zeros(len(F), np.float32) F.flux_agn[iagn] = np.array( [src.getParams()[0] for src in agnsrcs]) F.flux_agn_ivar[iagn] = R.IV[N:].astype(np.float32) if timing: t = Time() print('Forced photom:', t - tlast) tlast = t if do_apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack( [tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() #print('Apertures:', apertures, 'pixels') #print('apxy shape', apxy.shape) # --> (2,N) # The aperture photometry routine doesn't like pixel positions outside the image H, W = img.shape Iap = np.flatnonzero((apxy[0, :] >= 0) * (apxy[1, :] >= 0) * (apxy[0, :] <= W - 1) * (apxy[1, :] <= H - 1)) print('Aperture photometry for', len(Iap), 'of', len(apxy[0, :]), 'sources within image bounds') for rad in apertures: aper = photutils.CircularAperture(apxy[:, Iap], rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = np.zeros((len(F), len(apertures)), np.float32) F.apflux[Iap, :] = ap.astype(np.float32) apimgerr = np.vstack(apimgerr).T apiv = np.zeros(apimgerr.shape, np.float32) apiv[apimgerr != 0] = 1. / apimgerr[apimgerr != 0]**2 F.apflux_ivar = np.zeros((len(F), len(apertures)), np.float32) F.apflux_ivar[Iap, :] = apiv if timing: print('Aperture photom:', Time() - tlast) if get_model: return F, mod return F
def unwise_forcedphot(cat, tiles, bands=[1, 2, 3, 4], roiradecbox=None, unwise_dir='.', use_ceres=True, ceres_block=8, save_fits=False, ps=None, psf_broadening=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. ''' # Severely limit sizes of models for src in cat: if isinstance(src, PointSource): src.fixedRadius = 10 else: src.halfsize = 10 wantims = ((ps is not None) or save_fits) wanyband = 'w' fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp'] Nsrcs = len(cat) phot = fits_table() phot.tile = np.array([' '] * Nsrcs) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) for band in bands: print('Photometering WISE band', band) wband = 'w%i' % band # The tiles have some overlap, so for each source, keep the # fit in the tile whose center is closest to the source. tiledists = np.empty(Nsrcs) tiledists[:] = 1e100 flux_invvars = np.zeros(Nsrcs, np.float32) fitstats = dict([(k, np.zeros(Nsrcs, np.float32)) for k in fskeys]) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) for tile in tiles: print('Reading tile', tile.coadd_id) tim = get_unwise_tractor_image(unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: print('Actually, no overlap with tile', tile.coadd_id) continue if psf_broadening is not None: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): # print('Broadening PSF: from', psf) p0 = psf.getParams() #print('Params:', p0) pnames = psf.getParamNames() #print('Param names:', pnames) p1 = [p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames)] #print('Broadened:', p1) psf.setParams(p1) print('Broadened PSF:', psf) else: print( 'WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) print('Read image with shape', tim.shape) # Select sources in play. wcs = tim.wcs.wcs H, W = tim.shape ok, x, y = wcs.radec2pixelxy(ra, dec) x = (x - 1.).astype(np.float32) y = (y - 1.).astype(np.float32) margin = 10. I = np.flatnonzero((x >= -margin) * (x < W + margin) * (y >= -margin) * (y < H + margin)) print(len(I), 'within the image + margin') inbox = ((x[I] >= -0.5) * (x[I] < (W - 0.5)) * (y[I] >= -0.5) * (y[I] < (H - 0.5))) print(sum(inbox), 'strictly within the image') # Compute L_inf distance to (full) tile center. tilewcs = unwise_tile_wcs(tile.ra, tile.dec) cx, cy = tilewcs.crpix ok, tx, ty = tilewcs.radec2pixelxy(ra[I], dec[I]) td = np.maximum(np.abs(tx - cx), np.abs(ty - cy)) closest = (td < tiledists[I]) tiledists[I[closest]] = td[closest] keep = inbox * closest # Source indices (in the full "cat") to keep (the fit values for) srci = I[keep] if not len(srci): print('No sources to be kept; skipping.') continue phot.tile[srci] = tile.coadd_id nexp[srci] = tim.nuims[np.clip(np.round(y[srci]).astype(int), 0, H - 1), np.clip(np.round(x[srci]).astype(int), 0, W - 1)] # Source indices in the margins margi = I[np.logical_not(keep)] # sources in the box -- at the start of the subcat list. subcat = [cat[i] for i in srci] # include *copies* of sources in the margins # (that way we automatically don't save the results) subcat.extend([cat[i].copy() for i in margi]) assert(len(subcat) == len(I)) # FIXME -- set source radii, ...? minsb = 0. fitsky = False # Look in image and set radius based on peak height?? tractor = Tractor([tim], subcat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) kwa = dict(fitstat_extras=[('pronexp', [tim.nims])]) t0 = Time() R = tractor.optimize_forced_photometry( minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True, variance=True, shared_params=False, wantims=wantims, **kwa) print('unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] print('Ceres termination status:', term) # Running out of memory can cause failure to converge # and term status = 2. # Fail completely in this case. if term != 0: raise RuntimeError( 'Ceres terminated with status %i' % term) if wantims: ims0 = R.ims0 ims1 = R.ims1 IV, fs = R.IV, R.fitstats if save_fits: import fitsio (dat, mod, ie, chi, roi) = ims1[0] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if ps: tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, ie, chi, roi) = ims1[0] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() # Save results for this tile. # the "keep" sources are at the beginning of the "subcat" list flux_invvars[srci] = IV[:len(srci)].astype(np.float32) if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[srci] = (tim.mjdmin + tim.mjdmax) / 2. if fs is None: continue for k in fskeys: x = getattr(fs, k) # fitstats are returned only for un-frozen sources fitstats[k][srci] = np.array(x).astype(np.float32)[:len(srci)] # Note, this is *outside* the loop over tiles. # The fluxes are saved in the source objects, and will be set based on # the 'tiledists' logic above. nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their default # (1-sigma or whatever) initial fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set(wband + '_nanomaggies', nm.astype(np.float32)) phot.set(wband + '_nanomaggies_ivar', nm_ivar) dnm = np.zeros(len(nm_ivar), np.float32) okiv = (nm_ivar > 0) dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32) okflux = (nm > 0) mag = np.zeros(len(nm), np.float32) mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux]) ).astype(np.float32) dmag = np.zeros(len(nm), np.float32) ok = (okiv * okflux) dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok]) ).astype(np.float32) mag[np.logical_not(okflux)] = np.nan dmag[np.logical_not(ok)] = np.nan phot.set(wband + '_mag', mag) phot.set(wband + '_mag_err', dmag) for k in fskeys: phot.set(wband + '_' + k, fitstats[k]) phot.set(wband + '_nexp', nexp) if not np.all(mjd == 0): phot.set(wband + '_mjd', mjd) return phot
print 'Forced photom...' tr = Tractor([tim], cat) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) kwa = {} if opt.ceres: B = 8 kwa.update(use_ceres=True, BW=B, BH=B) if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa) if opt.plots: (data,mod,ie,chi,roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig()
def main(decals=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0,x1,y0,y1) = opt.zoom zoomslice = (slice(y0,y1), slice(x0,x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for decals.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if decals is None: decals = Decals() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from decals-ccds.fits table T = decals.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut(np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert(len(T) == 1) im = decals.get_image_object(T[0]) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True) print('Got tim:', tim) if opt.catfn in ['DR1', 'DR2']: if opt.catalog_path is None: opt.catalog_path = opt.catfn.lower() margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, decals=decals) for b in bricks: # there is some overlap with this brick... read the catalog. fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3], 'tractor-%s.fits' % b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec) W,H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) * (yy >= -margin) * (yy <= (H+margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') TT.append(T) T = merge_tables(TT) T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE) # print('Got cat:', cat) print('Forced photom...') opti = None if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)) ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x-1).astype(np.float32) F.y = (y-1).astype(np.float32) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap ap = 1./(np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap if opt.forced: kwa = {} if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa) if opt.plots: (data,mod,ie,chi,roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([src.getBrightness().getFlux(tim.band) for src in cat]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2 .astype(np.float32) program_name = sys.argv[0] version_hdr = get_version_header(program_name, decals.decals_dir) # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) fname = os.path.join(d2, d1, fname) print('Trimmed filename to', fname) #version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file')) version_hdr.add_record(dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext')) version_hdr.add_record(dict(name='CAMERA', value='DECam', comment='Dark Energy Camera')) version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num')) version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name')) version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record(dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image')) keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV', 'INSTRUME'] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = {'mjd':'sec', 'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'} columns = F.get_columns() for i,col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) print('Finished forced phot:', Time()-t0) return 0
color='r') plt.axis(ax) ps.savefig() tractor.freezeParam('images') for src in srcs: src.freezeAllBut('brightness') from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tractor.optimizer = opti print('Forced phot...') kwa = {} R = tractor.optimize_forced_photometry(shared_params=False, variance=True, **kwa) print('R:', R, dir(R)) mod = tractor.getModelImage(0) plt.clf() plt.imshow(mod, **ima) plt.title('Forced photometry model') ps.savefig() plt.clf() plt.imshow(subtim.getImage(), **ima) plt.title('CFHT data') ps.savefig()
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None, use_ceres=True, ceres_block=8, save_fits=False, get_models=False, ps=None, psf_broadening=None, pixelized_psf=False, get_masks=None, move_crpix=False, modelsky_dir=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. *get_masks*: the WCS to resample mask bits into. ''' from tractor import NanoMaggies, PointSource, Tractor, ExpGalaxy, DevGalaxy, FixedCompositeGalaxy if not pixelized_psf and psf_broadening is None: # PSF broadening in post-reactivation data, by band. # Newer version from Aaron's email to decam-chatter, 2018-06-14. broadening = { 1: 1.0405, 2: 1.0346, 3: None, 4: None } psf_broadening = broadening[band] if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt wantims = (plots or save_fits or get_models) wanyband = 'w' if get_models: models = {} wband = 'w%i' % band fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp'] Nsrcs = len(cat) phot = fits_table() # Filled in based on unique tile overlap phot.wise_coadd_id = np.array([' '] * Nsrcs) phot.set(wband + '_psfdepth', np.zeros(len(phot), np.float32)) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) central_flux = np.zeros(Nsrcs, np.float32) fitstats = {} tims = [] if get_masks: mh,mw = get_masks.shape maskmap = np.zeros((mh,mw), np.uint32) for tile in tiles: print('Reading WISE tile', tile.coadd_id, 'band', band) tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: print('Actually, no overlap with tile', tile.coadd_id) continue if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data' % tag) ps.savefig() plt.clf() plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), range=(-5,10), bins=100) plt.xlabel('Per-pixel intensity (Sigma)') plt.title(tag) ps.savefig() if move_crpix and band in [1, 2]: realwcs = tim.wcs.wcs x,y = realwcs.crpix tile_crpix = tile.get('crpix_w%i' % band) dx = tile_crpix[0] - 1024.5 dy = tile_crpix[1] - 1024.5 realwcs.set_crpix(x+dx, y+dy) #print('CRPIX', x,y, 'shift by', dx,dy, 'to', realwcs.crpix) if modelsky_dir and band in [1, 2]: fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band)) if not os.path.exists(fn): raise RuntimeError('WARNING: does not exist:', fn) x0,x1,y0,y1 = tim.roi bg = fitsio.FITS(fn)[2][y0:y1, x0:x1] #print('Read background map:', bg.shape, bg.dtype, 'vs image', tim.shape) if plots: plt.clf() plt.subplot(1,2,1) plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) plt.subplot(1,2,2) plt.imshow(bg, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) tag = '%s W%i' % (tile.coadd_id, band) plt.suptitle(tag) ps.savefig() plt.clf() ha = dict(range=(-5,10), bins=100, histtype='step') plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), color='b', label='Original', **ha) plt.hist(((tim.getImage()-bg) * tim.inverr)[tim.inverr > 0].ravel(), color='g', label='Minus Background', **ha) plt.axvline(0, color='k', alpha=0.5) plt.xlabel('Per-pixel intensity (Sigma)') plt.legend() plt.title(tag + ': background') ps.savefig() # Actually subtract the background! tim.data -= bg # Floor the per-pixel variances if band in [1,2]: # in Vega nanomaggies per pixel floor_sigma = {1: 0.5, 2: 2.0} with np.errstate(divide='ignore'): new_ie = 1. / np.hypot(1./tim.inverr, floor_sigma[band]) new_ie[tim.inverr == 0] = 0. if plots: plt.clf() plt.plot((1. / tim.inverr[tim.inverr>0]).ravel(), (1./new_ie[tim.inverr>0]).ravel(), 'b.') plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band)) plt.xlabel('original') plt.ylabel('floored') ps.savefig() tim.inverr = new_ie # Read mask file? if get_masks: from astrometry.util.resample import resample_with_wcs, OverlapError # unwise_dir can be a colon-separated list of paths tilemask = None for d in tile.unwise_dir.split(':'): fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id, 'unwise-%s-msk.fits.gz' % tile.coadd_id) if os.path.exists(fn): print('Reading unWISE mask file', fn) x0,x1,y0,y1 = tim.roi tilemask = fitsio.FITS(fn)[0][y0:y1,x0:x1] break if tilemask is None: print('unWISE mask file for tile', tile.coadd_id, 'does not exist') else: try: tanwcs = tim.wcs.wcs assert(tanwcs.shape == tilemask.shape) Yo,Xo,Yi,Xi,_ = resample_with_wcs(get_masks, tanwcs, intType=np.int16) # Only deal with mask pixels that are set. I, = np.nonzero(tilemask[Yi,Xi] > 0) # Trim to unique area for this tile rr,dd = get_masks.pixelxy2radec(Yo[I]+1, Xo[I]+1) good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I = I[good] maskmap[Yo[I],Xo[I]] = tilemask[Yi[I], Xi[I]] except OverlapError: # Shouldn't happen by this point print('No overlap between WISE tile', tile.coadd_id, 'and brick') # The tiles have some overlap, so zero out pixels outside the # tile's unique area. th,tw = tim.shape xx,yy = np.meshgrid(np.arange(tw), np.arange(th)) rr,dd = tim.wcs.wcs.pixelxy2radec(xx+1, yy+1) unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) #print(np.sum(unique), 'of', (th*tw), 'pixels in this tile are unique') tim.inverr[unique == False] = 0. del xx,yy,rr,dd,unique if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage() * (tim.inverr > 0), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data (unique)' % tag) ps.savefig() if pixelized_psf: import unwise_psf if (band == 1) or (band == 2): # we only have updated PSFs for W1 and W2 psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id, modelname='neo4_unwisecat') else: psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id) if band == 4: # oversample (the unwise_psf models are at native W4 5.5"/pix, # while the unWISE coadds are made at 2.75"/pix. ph,pw = psfimg.shape subpsf = np.zeros((ph*2-1, pw*2-1), np.float32) from astrometry.util.util import lanczos3_interpolate xx,yy = np.meshgrid(np.arange(0., pw-0.51, 0.5, dtype=np.float32), np.arange(0., ph-0.51, 0.5, dtype=np.float32)) xx = xx.ravel() yy = yy.ravel() ix = xx.astype(np.int32) iy = yy.astype(np.int32) dx = (xx - ix).astype(np.float32) dy = (yy - iy).astype(np.float32) psfimg = psfimg.astype(np.float32) rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg]) if plots: plt.clf() plt.imshow(psfimg, interpolation='nearest', origin='lower') plt.title('Original PSF model') ps.savefig() plt.clf() plt.imshow(subpsf, interpolation='nearest', origin='lower') plt.title('Subsampled PSF model') ps.savefig() psfimg = subpsf del xx, yy, ix, iy, dx, dy from tractor.psf import PixelizedPSF psfimg /= psfimg.sum() fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0} psfimg *= fluxrescales[band] tim.psf = PixelizedPSF(psfimg) if psf_broadening is not None and not pixelized_psf: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): # print('Broadening PSF: from', psf) p0 = psf.getParams() pnames = psf.getParamNames() p1 = [p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames)] psf.setParams(p1) print('Broadened PSF:', psf) else: print('WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) wcs = tim.wcs.wcs ok,x,y = wcs.radec2pixelxy(ra, dec) x = np.round(x - 1.).astype(int) y = np.round(y - 1.).astype(int) good = (x >= 0) * (x < tw) * (y >= 0) * (y < th) # Which sources are in this brick's unique area? usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I, = np.nonzero(good * usrc) nexp[I] = tim.nuims[y[I], x[I]] if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. phot.wise_coadd_id[I] = tile.coadd_id central_flux[I] = tim.getImage()[y[I], x[I]] del x,y,good,usrc # PSF norm for depth psf = tim.getPsf() h,w = tim.shape patch = psf.getPointSourcePatch(h//2, w//2).patch psfnorm = np.sqrt(np.sum(patch**2)) # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. psfdepth = 1. / (tim.sig1 / psfnorm)**2 phot.get(wband + '_psfdepth')[I] = psfdepth tim.tile = tile tims.append(tim) if plots: plt.clf() mn,mx = 0.1, 20000 plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100, range=(np.log10(mn), np.log10(mx))) logt = np.arange(0, 5) plt.xticks(logt, ['%i' % i for i in 10.**logt]) plt.title('Central fluxes (W%i)' % band) plt.axvline(np.log10(20000), color='k') plt.axvline(np.log10(1000), color='k') ps.savefig() # Eddie's non-secret recipe: #- central pixel <= 1000: 19x19 pix box size #- central pixel in 1000 - 20000: 59x59 box size #- central pixel > 20000 or saturated: 149x149 box size #- object near "bright star": 299x299 box size nbig = nmedium = nsmall = 0 for src,cflux in zip(cat, central_flux): if cflux > 20000: R = 100 nbig += 1 elif cflux > 1000: R = 30 nmedium += 1 else: R = 15 nsmall += 1 if isinstance(src, PointSource): src.fixedRadius = R else: ### FIXME -- sizes for galaxies..... can we set PSF size separately? galrad = 0 # RexGalaxy is a subclass of ExpGalaxy if isinstance(src, (ExpGalaxy, DevGalaxy)): galrad = src.shape.re elif isinstance(src, FixedCompositeGalaxy): galrad = max(src.shapeExp.re, src.shapeDev.re) pixscale = 2.75 src.halfsize = int(np.hypot(R, galrad * 5 / pixscale)) #print('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small') minsb = 0. fitsky = False tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) kwa = dict(fitstat_extras=[('pronexp', [tim.nims for tim in tims])]) t0 = Time() R = tractor.optimize_forced_photometry( minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True, variance=True, shared_params=False, wantims=wantims, **kwa) print('unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge # and term status = 2. # Fail completely in this case. if term != 0: print('Ceres termination status:', term) raise RuntimeError( 'Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 flux_invvars = R.IV if R.fitstats is not None: for k in fskeys: x = getattr(R.fitstats, k) fitstats[k] = np.array(x).astype(np.float32) if save_fits: for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, chi, roi) = ims1[i] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if plots: # Create models for just the brightest sources bright_cat = [src for src in cat if src.getBrightness().getBand(wanyband) > 1000] print('Bright soures:', len(bright_cat)) btr = Tractor(tims, bright_cat) for tim in tims: mod = btr.getModelImage(tim) tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) sig1 = tim.sig1 plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: bright-star models' % tag) ps.savefig() if get_models: for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, chi, roi) = ims1[i] models[(tile.coadd_id, band)] = (mod, dat, ie, tim.roi, tim.wcs.wcs) if plots: for i,tim in enumerate(tims): tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, ie, chi, roi) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their default # (1-sigma or whatever) initial fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set(wband + '_nanomaggies', nm.astype(np.float32)) phot.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32)) dnm = np.zeros(len(nm_ivar), np.float32) okiv = (nm_ivar > 0) dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32) okflux = (nm > 0) mag = np.zeros(len(nm), np.float32) mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux]) ).astype(np.float32) dmag = np.zeros(len(nm), np.float32) ok = (okiv * okflux) dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok]) ).astype(np.float32) mag[np.logical_not(okflux)] = np.nan dmag[np.logical_not(ok)] = np.nan phot.set(wband + '_mag', mag) phot.set(wband + '_mag_err', dmag) for k in fskeys: phot.set(wband + '_' + k, fitstats[k]) phot.set(wband + '_nexp', nexp) if not np.all(mjd == 0): phot.set(wband + '_mjd', mjd) rtn = wphotduck() rtn.phot = phot rtn.models = None rtn.maskmap = None if get_models: rtn.models = models if get_masks: rtn.maskmap = maskmap return rtn
plt.axis(ax) ps.savefig() tractor.freezeParam('images') for src in srcs: src.freezeAllBut('brightness') from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tractor.optimizer = opti print('Forced phot...') kwa = {} R = tractor.optimize_forced_photometry(shared_params=False, variance=True, **kwa) print('R:', R, dir(R)) mod = tractor.getModelImage(0) plt.clf() plt.imshow(mod, **ima) plt.title('Forced photometry model') ps.savefig() plt.clf() plt.imshow(subtim.getImage(), **ima) plt.title('CFHT data') ps.savefig()
print('Tractor image', tim.name) plt.clf() plt.imshow(timg, interpolation='nearest', origin='lower') ps.savefig() print('Tractor model', tim.name) plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower') ps.savefig() tractor.freezeParam('images') print('Params:') tractor.printThawedParams() tractor.optimize_forced_photometry(priors=False, shared_params=False) mod = tractor.getModelImage(0) print('Tractor model (forced phot)', tim.name) plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower') ps.savefig() comod[Yo, Xo] += wt * mod[Yi - y0, Xi - x0] coimg /= np.maximum(cowt, 1e-18) comod /= np.maximum(cowt, 1e-18) coimgs.append(coimg) comods.append(comod)
def galex_coadds(onegal, galaxy=None, radius_mosaic=30, radius_mask=None, pixscale=1.5, ref_pixscale=0.262, output_dir=None, galex_dir=None, log=None, centrals=True, verbose=False): '''Generate custom GALEX cutouts. radius_mosaic and radius_mask in arcsec pixscale: GALEX pixel scale in arcsec/pixel. ''' import fitsio import matplotlib.pyplot as plt from astrometry.libkd.spherematch import match_radec from astrometry.util.resample import resample_with_wcs, OverlapError from tractor import (Tractor, NanoMaggies, Image, LinearPhotoCal, NCircularGaussianPSF, ConstantFitsWcs, ConstantSky) from legacypipe.survey import imsave_jpeg from legacypipe.catalog import read_fits_catalog if galaxy is None: galaxy = 'galaxy' if galex_dir is None: galex_dir = os.environ.get('GALEX_DIR') if output_dir is None: output_dir = '.' if radius_mask is None: radius_mask = radius_mosaic radius_search = 5.0 # [arcsec] else: radius_search = radius_mask W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels] targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0, -pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W), float(H)) # Read the custom Tractor catalog tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy)) if not os.path.isfile(tractorfile): print('Missing Tractor catalog {}'.format(tractorfile)) return 0 cat = fits_table(tractorfile) print('Read {} sources from {}'.format(len(cat), tractorfile), flush=True, file=log) keep = np.ones(len(cat)).astype(bool) if centrals: # Find the large central galaxy and mask out (ignore) all the models # which are within its elliptical mask. # This algorithm will have to change for mosaics not centered on large # galaxies, e.g., in galaxy groups. m1, m2, d12 = match_radec(cat.ra, cat.dec, onegal['RA'], onegal['DEC'], radius_search / 3600.0, nearest=False) if len(m1) == 0: print('No central galaxies found at the central coordinates!', flush=True, file=log) else: pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions for mm in m1: morphtype = cat.type[mm].strip() if morphtype == 'EXP' or morphtype == 'COMP': e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[ mm], cat.shapeexp_r[mm] # [arcsec] elif morphtype == 'DEV' or morphtype == 'COMP': e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[ mm], cat.shapedev_r[mm] # [arcsec] else: r50 = None if r50: majoraxis = r50 * 5 / pixscale # [pixels] ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2) these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis, ba * majoraxis, np.radians(phi), cat.bx * pixfactor, cat.by * pixfactor) if np.sum(these) > 0: #keep[these] = False pass print('Hack!') keep[mm] = False #srcs = read_fits_catalog(cat) #_srcs = np.array(srcs)[~keep].tolist() #mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0) #import matplotlib.pyplot as plt ##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png') #plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png') #pdb.set_trace() srcs = read_fits_catalog(cat) for src in srcs: src.freezeAllBut('brightness') #srcs_nocentral = np.array(srcs)[keep].tolist() # Find all overlapping GALEX tiles and then read the tims. galex_tiles = _read_galex_tiles(targetwcs, galex_dir, log=log, verbose=verbose) gbands = ['n', 'f'] nicegbands = ['NUV', 'FUV'] zps = dict(n=20.08, f=18.82) coimgs, comods, coresids, coimgs_central, comods_nocentral = [], [], [], [], [] for niceband, band in zip(nicegbands, gbands): J = np.flatnonzero(galex_tiles.get('has_' + band)) print(len(J), 'GALEX tiles have coverage in band', band) coimg = np.zeros((H, W), np.float32) comod = np.zeros((H, W), np.float32) cowt = np.zeros((H, W), np.float32) comod_nocentral = np.zeros((H, W), np.float32) for src in srcs: src.setBrightness(NanoMaggies(**{band: 1})) for j in J: brick = galex_tiles[j] fn = os.path.join( galex_dir, brick.tilename.strip(), '%s-%sd-intbgsub.fits.gz' % (brick.brickname, band)) #print(fn) gwcs = Tan(*[ float(f) for f in [ brick.crval1, brick.crval2, brick.crpix1, brick.crpix2, brick.cdelt1, 0., 0., brick.cdelt2, 3840., 3840. ] ]) img = fitsio.read(fn) #print('Read', img.shape) try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, gwcs, [], 3) except OverlapError: continue K = np.flatnonzero(img[Yi, Xi] != 0.) if len(K) == 0: continue Yo, Xo, Yi, Xi = Yo[K], Xo[K], Yi[K], Xi[K] wt = brick.get(band + 'exptime') coimg[Yo, Xo] += wt * img[Yi, Xi] cowt[Yo, Xo] += wt x0, x1, y0, y1 = min(Xi), max(Xi), min(Yi), max(Yi) subwcs = gwcs.get_subimage(x0, y0, x1 - x0 + 1, y1 - y0 + 1) twcs = ConstantFitsWcs(subwcs) timg = img[y0:y1 + 1, x0:x1 + 1] tie = np.ones_like(timg) ## HACK! #hdr = fitsio.read_header(fn) #zp = hdr[''] zp = zps[band] photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp), band=band) tsky = ConstantSky(0.0) # HACK -- circular Gaussian PSF of fixed size... # in arcsec #fwhms = dict(NUV=6.0, FUV=6.0) # -> sigma in pixels #sig = fwhms[band] / 2.35 / twcs.pixel_scale() sig = 6.0 / np.sqrt(8 * np.log(2)) / twcs.pixel_scale() tpsf = NCircularGaussianPSF([sig], [1.]) tim = Image(data=timg, inverr=tie, psf=tpsf, wcs=twcs, sky=tsky, photocal=photocal, name='GALEX ' + band + brick.brickname) ## Build the model image with and without the central galaxy model. tractor = Tractor([tim], srcs) mod = tractor.getModelImage(0) tractor.freezeParam('images') tractor.optimize_forced_photometry(priors=False, shared_params=False) mod = tractor.getModelImage(0) srcs_nocentral = np.array(srcs)[keep].tolist() #srcs_nocentral = np.array(srcs)[nocentral].tolist() tractor_nocentral = Tractor([tim], srcs_nocentral) mod_nocentral = tractor_nocentral.getModelImage(0) comod[Yo, Xo] += wt * mod[Yi - y0, Xi - x0] comod_nocentral[Yo, Xo] += wt * mod_nocentral[Yi - y0, Xi - x0] coimg /= np.maximum(cowt, 1e-18) comod /= np.maximum(cowt, 1e-18) comod_nocentral /= np.maximum(cowt, 1e-18) coresid = coimg - comod # Subtract the model image which excludes the central (comod_nocentral) # from the data (coimg) to isolate the light of the central # (coimg_central). coimg_central = coimg - comod_nocentral coimgs.append(coimg) comods.append(comod) coresids.append(coresid) comods_nocentral.append(comod_nocentral) coimgs_central.append(coimg_central) # Write out the final images with and without the central, making sure # to apply the zeropoint to go from counts/s to AB nanomaggies. # https://asd.gsfc.nasa.gov/archive/galex/FAQ/counts_background.html for thisimg, imtype in zip((coimg, comod, comod_nocentral), ('image', 'model', 'model-nocentral')): fitsfile = os.path.join( output_dir, '{}-{}-{}.fits'.format(galaxy, imtype, niceband)) if verbose: print('Writing {}'.format(fitsfile)) fitsio.write(fitsfile, thisimg * 10**(-0.4 * (zp - 22.5)), clobber=True) # Build a color mosaic (but note that the images here are in units of # background-subtracted counts/s). #_galex_rgb = _galex_rgb_moustakas #_galex_rgb = _galex_rgb_dstn _galex_rgb = _galex_rgb_official for imgs, imtype in zip( (coimgs, comods, coresids, comods_nocentral, coimgs_central), ('image', 'model', 'resid', 'model-nocentral', 'image-central')): rgb = _galex_rgb(imgs) jpgfile = os.path.join(output_dir, '{}-{}-FUVNUV.jpg'.format(galaxy, imtype)) if verbose: print('Writing {}'.format(jpgfile)) imsave_jpeg(jpgfile, rgb, origin='lower') return 1
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None, use_ceres=True, ceres_block=8, save_fits=False, get_models=False, ps=None, psf_broadening=None, pixelized_psf=False, get_masks=None, move_crpix=False, modelsky_dir=None, tag=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. *get_masks*: the WCS to resample mask bits into. ''' from tractor import PointSource, Tractor, ExpGalaxy, DevGalaxy from tractor.sersic import SersicGalaxy if tag is None: tag = '' else: tag = tag + ': ' if not pixelized_psf and psf_broadening is None: # PSF broadening in post-reactivation data, by band. # Newer version from Aaron's email to decam-chatter, 2018-06-14. broadening = {1: 1.0405, 2: 1.0346, 3: None, 4: None} psf_broadening = broadening[band] if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt wantims = (plots or save_fits or get_models) wanyband = 'w' if get_models: models = [] wband = 'w%i' % band Nsrcs = len(cat) phot = fits_table() # Filled in based on unique tile overlap phot.wise_coadd_id = np.array([' '] * Nsrcs, dtype='U8') phot.wise_x = np.zeros(Nsrcs, np.float32) phot.wise_y = np.zeros(Nsrcs, np.float32) phot.set('psfdepth_%s' % wband, np.zeros(Nsrcs, np.float32)) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) central_flux = np.zeros(Nsrcs, np.float32) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) fskeys = ['prochi2', 'profracflux'] fitstats = {} if get_masks: mh, mw = get_masks.shape maskmap = np.zeros((mh, mw), np.uint32) tims = [] for tile in tiles: info(tag + 'Reading WISE tile', tile.coadd_id, 'band', band) tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: debug('Actually, no overlap with WISE coadd tile', tile.coadd_id) continue if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data' % tag) ps.savefig() plt.clf() plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), range=(-5, 10), bins=100) plt.xlabel('Per-pixel intensity (Sigma)') plt.title(tag) ps.savefig() if move_crpix and band in [1, 2]: realwcs = tim.wcs.wcs x, y = realwcs.crpix tile_crpix = tile.get('crpix_w%i' % band) dx = tile_crpix[0] - 1024.5 dy = tile_crpix[1] - 1024.5 realwcs.set_crpix(x + dx, y + dy) debug('unWISE', tile.coadd_id, 'band', band, 'CRPIX', x, y, 'shift by', dx, dy, 'to', realwcs.crpix) if modelsky_dir and band in [1, 2]: fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band)) if not os.path.exists(fn): raise RuntimeError('WARNING: does not exist:', fn) x0, x1, y0, y1 = tim.roi bg = fitsio.FITS(fn)[2][y0:y1, x0:x1] assert (bg.shape == tim.shape) if plots: plt.clf() plt.subplot(1, 2, 1) plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) plt.subplot(1, 2, 2) plt.imshow(bg, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) tag = '%s W%i' % (tile.coadd_id, band) plt.suptitle(tag) ps.savefig() plt.clf() ha = dict(range=(-5, 10), bins=100, histtype='step') plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), color='b', label='Original', **ha) plt.hist(((tim.getImage() - bg) * tim.inverr)[tim.inverr > 0].ravel(), color='g', label='Minus Background', **ha) plt.axvline(0, color='k', alpha=0.5) plt.xlabel('Per-pixel intensity (Sigma)') plt.legend() plt.title(tag + ': background') ps.savefig() # Actually subtract the background! tim.data -= bg # Floor the per-pixel variances, # and add Poisson contribution from sources if band in [1, 2]: # in Vega nanomaggies per pixel floor_sigma = {1: 0.5, 2: 2.0} poissons = {1: 0.15, 2: 0.3} with np.errstate(divide='ignore'): new_ie = 1. / np.sqrt( (1. / tim.inverr)**2 + floor_sigma[band] + poissons[band]**2 * np.maximum(0., tim.data)) new_ie[tim.inverr == 0] = 0. if plots: plt.clf() plt.plot((1. / tim.inverr[tim.inverr > 0]).ravel(), (1. / new_ie[tim.inverr > 0]).ravel(), 'b.') plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band)) plt.xlabel('original') plt.ylabel('floored') ps.savefig() assert (np.all(np.isfinite(new_ie))) assert (np.all(new_ie >= 0.)) tim.inverr = new_ie # Expand a 3-pixel radius around weight=0 (saturated) pixels # from Eddie via crowdsource # https://github.com/schlafly/crowdsource/blob/7069da3e7d9d3124be1cbbe1d21ffeb63fc36dcc/python/wise_proc.py#L74 ## FIXME -- W3/W4 ?? satlimit = 85000 msat = ((tim.data > satlimit) | ((tim.nims == 0) & (tim.nuims > 1))) from scipy.ndimage.morphology import binary_dilation xx, yy = np.mgrid[-3:3 + 1, -3:3 + 1] dilate = xx**2 + yy**2 <= 3**2 msat = binary_dilation(msat, dilate) nbefore = np.sum(tim.inverr == 0) tim.inverr[msat] = 0 nafter = np.sum(tim.inverr == 0) debug('Masking an additional', (nafter - nbefore), 'near-saturated pixels in unWISE', tile.coadd_id, 'band', band) # Read mask file? if get_masks: from astrometry.util.resample import resample_with_wcs, OverlapError # unwise_dir can be a colon-separated list of paths tilemask = None for d in tile.unwise_dir.split(':'): fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id, 'unwise-%s-msk.fits.gz' % tile.coadd_id) if os.path.exists(fn): debug('Reading unWISE mask file', fn) x0, x1, y0, y1 = tim.roi tilemask = fitsio.FITS(fn)[0][y0:y1, x0:x1] break if tilemask is None: info('unWISE mask file for tile', tile.coadd_id, 'does not exist') else: try: tanwcs = tim.wcs.wcs assert (tanwcs.shape == tilemask.shape) Yo, Xo, Yi, Xi, _ = resample_with_wcs(get_masks, tanwcs, intType=np.int16) # Only deal with mask pixels that are set. I, = np.nonzero(tilemask[Yi, Xi] > 0) # Trim to unique area for this tile rr, dd = get_masks.pixelxy2radec(Xo[I] + 1, Yo[I] + 1) good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I = I[good] maskmap[Yo[I], Xo[I]] = tilemask[Yi[I], Xi[I]] except OverlapError: # Shouldn't happen by this point print('Warning: no overlap between WISE tile', tile.coadd_id, 'and brick') if plots: plt.clf() plt.imshow(tilemask, interpolation='nearest', origin='lower') plt.title('Tile %s: mask' % tile.coadd_id) ps.savefig() plt.clf() plt.imshow(maskmap, interpolation='nearest', origin='lower') plt.title('Tile %s: accumulated maskmap' % tile.coadd_id) ps.savefig() # The tiles have some overlap, so zero out pixels outside the # tile's unique area. th, tw = tim.shape xx, yy = np.meshgrid(np.arange(tw), np.arange(th)) rr, dd = tim.wcs.wcs.pixelxy2radec(xx + 1, yy + 1) unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) debug('Tile', tile.coadd_id, '- total of', np.sum(unique), 'unique pixels out of', len(unique.flat), 'total pixels') if get_models: # Save the inverr before blanking out non-unique pixels, for making coadds with no gaps! # (actually, slightly more subtly, expand unique area by 1 pixel) from scipy.ndimage.morphology import binary_dilation du = binary_dilation(unique) tim.coadd_inverr = tim.inverr * du tim.inverr[unique == False] = 0. del xx, yy, rr, dd, unique if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage() * (tim.inverr > 0), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data (unique)' % tag) ps.savefig() if pixelized_psf: from unwise_psf import unwise_psf if (band == 1) or (band == 2): # we only have updated PSFs for W1 and W2 psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id, modelname='neo6_unwisecat') else: psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id) if band == 4: # oversample (the unwise_psf models are at native W4 5.5"/pix, # while the unWISE coadds are made at 2.75"/pix. ph, pw = psfimg.shape subpsf = np.zeros((ph * 2 - 1, pw * 2 - 1), np.float32) from astrometry.util.util import lanczos3_interpolate xx, yy = np.meshgrid( np.arange(0., pw - 0.51, 0.5, dtype=np.float32), np.arange(0., ph - 0.51, 0.5, dtype=np.float32)) xx = xx.ravel() yy = yy.ravel() ix = xx.astype(np.int32) iy = yy.astype(np.int32) dx = (xx - ix).astype(np.float32) dy = (yy - iy).astype(np.float32) psfimg = psfimg.astype(np.float32) rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg]) if plots: plt.clf() plt.imshow(psfimg, interpolation='nearest', origin='lower') plt.title('Original PSF model') ps.savefig() plt.clf() plt.imshow(subpsf, interpolation='nearest', origin='lower') plt.title('Subsampled PSF model') ps.savefig() psfimg = subpsf del xx, yy, ix, iy, dx, dy from tractor.psf import PixelizedPSF psfimg /= psfimg.sum() fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0} psfimg *= fluxrescales[band] tim.psf = PixelizedPSF(psfimg) if psf_broadening is not None and not pixelized_psf: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): debug('Broadening PSF: from', psf) p0 = psf.getParams() pnames = psf.getParamNames() p1 = [ p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames) ] psf.setParams(p1) debug('Broadened PSF:', psf) else: print( 'WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) wcs = tim.wcs.wcs _, fx, fy = wcs.radec2pixelxy(ra, dec) x = np.round(fx - 1.).astype(int) y = np.round(fy - 1.).astype(int) good = (x >= 0) * (x < tw) * (y >= 0) * (y < th) # Which sources are in this brick's unique area? usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I, = np.nonzero(good * usrc) nexp[I] = tim.nuims[y[I], x[I]] if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. phot.wise_coadd_id[I] = tile.coadd_id phot.wise_x[I] = fx[I] - 1. phot.wise_y[I] = fy[I] - 1. central_flux[I] = tim.getImage()[y[I], x[I]] del x, y, good, usrc # PSF norm for depth psf = tim.getPsf() h, w = tim.shape patch = psf.getPointSourcePatch(h // 2, w // 2).patch psfnorm = np.sqrt(np.sum(patch**2)) # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. # In the small empty patches of the sky (eg W4 in 0922p702), we get sig1 = NaN if np.isfinite(tim.sig1): phot.get('psfdepth_%s' % wband)[I] = 1. / (tim.sig1 / psfnorm)**2 tim.tile = tile tims.append(tim) if plots: plt.clf() mn, mx = 0.1, 20000 plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100, range=(np.log10(mn), np.log10(mx))) logt = np.arange(0, 5) plt.xticks(logt, ['%i' % i for i in 10.**logt]) plt.title('Central fluxes (W%i)' % band) plt.axvline(np.log10(20000), color='k') plt.axvline(np.log10(1000), color='k') ps.savefig() # Eddie's non-secret recipe: #- central pixel <= 1000: 19x19 pix box size #- central pixel in 1000 - 20000: 59x59 box size #- central pixel > 20000 or saturated: 149x149 box size #- object near "bright star": 299x299 box size nbig = nmedium = nsmall = 0 for src, cflux in zip(cat, central_flux): if cflux > 20000: R = 100 nbig += 1 elif cflux > 1000: R = 30 nmedium += 1 else: R = 15 nsmall += 1 if isinstance(src, PointSource): src.fixedRadius = R else: ### FIXME -- sizes for galaxies..... can we set PSF size separately? galrad = 0 # RexGalaxy is a subclass of ExpGalaxy if isinstance(src, (ExpGalaxy, DevGalaxy, SersicGalaxy)): galrad = src.shape.re pixscale = 2.75 src.halfsize = int(np.hypot(R, galrad * 5 / pixscale)) debug('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small') tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) t0 = Time() R = tractor.optimize_forced_photometry(fitstats=True, variance=True, shared_params=False, wantims=wantims) info(tag + 'unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge and term # status = 2. Fail completely in this case. if term != 0: info(tag + 'Ceres termination status:', term) raise RuntimeError('Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 # can happen if empty source list (we still want to generate coadds) if ims1 is None: ims1 = R.ims0 flux_invvars = R.IV if R.fitstats is not None: for k in fskeys: x = getattr(R.fitstats, k) fitstats[k] = np.array(x).astype(np.float32) if save_fits: for i, tim in enumerate(tims): tile = tim.tile (dat, mod, _, chi, _) = ims1[i] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if plots: # Create models for just the brightest sources bright_cat = [ src for src in cat if src.getBrightness().getBand(wanyband) > 1000 ] debug('Bright soures:', len(bright_cat)) btr = Tractor(tims, bright_cat) for tim in tims: mod = btr.getModelImage(tim) tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) sig1 = tim.sig1 plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: bright-star models' % tag) ps.savefig() if get_models: for i, tim in enumerate(tims): tile = tim.tile (dat, mod, _, _, _) = ims1[i] models.append( (tile.coadd_id, band, tim.wcs.wcs, dat, mod, tim.coadd_inverr)) if plots: for i, tim in enumerate(tims): tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, _, chi, _) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their initial # fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set('flux_%s' % wband, nm.astype(np.float32)) phot.set('flux_ivar_%s' % wband, nm_ivar.astype(np.float32)) for k in fskeys: phot.set(k + '_' + wband, fitstats.get(k, np.zeros(len(phot), np.float32))) phot.set('nobs_%s' % wband, nexp) phot.set('mjd_%s' % wband, mjd) rtn = wphotduck() rtn.phot = phot rtn.models = None rtn.maskmap = None if get_models: rtn.models = models if get_masks: rtn.maskmap = maskmap return rtn
def galex_forcedphot(galex_dir, cat, tiles, band, roiradecbox, pixelized_psf=False, ps=None): ''' Given a list of tractor sources *cat* and a list of GALEX tiles *tiles* (a fits_table with RA,Dec,tilename) runs forced photometry, returning a FITS table the same length as *cat*. ''' from tractor import Tractor from astrometry.util.ttime import Time if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt use_ceres = True wantims = True get_models = True gband = 'galex' phot = fits_table() tims = [] for tile in tiles: info('Reading GALEX tile', tile.visitname.strip(), 'band', band) tim = galex_tractor_image(tile, band, galex_dir, roiradecbox, gband) if tim is None: debug('Actually, no overlap with tile', tile.tilename) continue # if plots: # sig1 = tim.sig1 # plt.clf() # plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', # cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) # plt.colorbar() # tag = '%s W%i' % (tile.tilename, band) # plt.title('%s: tim data' % tag) # ps.savefig() if pixelized_psf: psfimg = galex_psf(band, galex_dir) tim.psf = PixelizedPSF(psfimg) # if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): # mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. # # PSF norm for depth # psf = tim.getPsf() # h,w = tim.shape # patch = psf.getPointSourcePatch(h//2, w//2).patch # psfnorm = np.sqrt(np.sum(patch**2)) # # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. # psfdepth = 1. / (tim.sig1 / psfnorm)**2 # phot.get(wband + '_psfdepth')[I] = psfdepth tim.tile = tile tims.append(tim) tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer ceres_block = 8 tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(gband) t0 = Time() R = tractor.optimize_forced_photometry( fitstats=True, variance=True, shared_params=False, wantims=wantims) info('GALEX forced photometry took', Time() - t0) #info('Result:', R) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge and term # status = 2. Fail completely in this case. if term != 0: info('Ceres termination status:', term) raise RuntimeError('Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 flux_invvars = R.IV # if plots: # # Create models for just the brightest sources # bright_cat = [src for src in cat # if src.getBrightness().getBand(wanyband) > 1000] # debug('Bright soures:', len(bright_cat)) # btr = Tractor(tims, bright_cat) # for tim in tims: # mod = btr.getModelImage(tim) # tile = tim.tile # tag = '%s W%i' % (tile.tilename, band) # sig1 = tim.sig1 # plt.clf() # plt.imshow(mod, interpolation='nearest', origin='lower', # cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) # plt.colorbar() # plt.title('%s: bright-star models' % tag) # ps.savefig() if get_models: models = [] for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, _, _) = ims1[i] models.append((tile.visitname, band, tim.wcs.wcs, dat, mod, ie)) if plots: for i,tim in enumerate(tims): tile = tim.tile tag = '%s %s' % (tile.tilename, band) (dat, mod, _, chi, _) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(gband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their default # (1-sigma or whatever) initial fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. niceband = band + 'uv' phot.set('flux_' + niceband, nm.astype(np.float32)) phot.set('flux_ivar_' + niceband, nm_ivar.astype(np.float32)) #phot.set(band + '_mjd', mjd) rtn = gphotduck() rtn.phot = phot rtn.models = None if get_models: rtn.models = models return rtn
def main(decals=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0, x1, y0, y1) = opt.zoom zoomslice = (slice(y0, y1), slice(x0, x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for decals.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if decals is None: decals = Decals() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from decals-ccds.fits table T = decals.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut( np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert (len(T) == 1) im = decals.get_image_object(T[0]) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True) print('Got tim:', tim) if opt.catfn in ['DR1', 'DR2']: if opt.catalog_path is None: opt.catalog_path = opt.catfn.lower() margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, decals=decals) for b in bricks: # there is some overlap with this brick... read the catalog. fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3], 'tractor-%s.fits' % b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec) W, H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) * (yy >= -margin) * (yy <= (H + margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') TT.append(T) T = merge_tables(TT) T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE) # print('Got cat:', cat) print('Forced photom...') opti = None if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)) ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x - 1).astype(np.float32) F.y = (y - 1).astype(np.float32) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack( [tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap ap = 1. / (np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap if opt.forced: kwa = {} if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa) if opt.plots: (data, mod, ie, chi, roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([ src.getBrightness().getFlux(tim.band) for src in cat ]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2.astype(np.float32) program_name = sys.argv[0] version_hdr = get_version_header(program_name, decals.decals_dir) # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) fname = os.path.join(d2, d1, fname) print('Trimmed filename to', fname) #version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file')) version_hdr.add_record( dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file')) version_hdr.add_record( dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext')) version_hdr.add_record( dict(name='CAMERA', value='DECam', comment='Dark Energy Camera')) version_hdr.add_record( dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num')) version_hdr.add_record( dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name')) version_hdr.add_record( dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record( dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image')) keys = [ 'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME' ] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = { 'mjd': 'sec', 'exptime': 'sec', 'flux': 'nanomaggy', 'flux_ivar': '1/nanomaggy^2' } columns = F.get_columns() for i, col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) print('Finished forced phot:', Time() - t0) return 0