def one_tile(tile, opt, savepickle, ps, tiles, tiledir, tempoutdir, T=None, hdr=None): bands = opt.bands outfn = opt.output % (tile.coadd_id) savewise_outfn = opt.save_wise_output % (tile.coadd_id) sband = 'r' bandnum = 'ugriz'.index(sband) tt0 = Time() print() print('Coadd tile', tile.coadd_id) thisdir = get_unwise_tile_dir(tiledir, tile.coadd_id) fn = os.path.join(thisdir, 'unwise-%s-w%i-img-m.fits' % (tile.coadd_id, bands[0])) if os.path.exists(fn): print('Reading', fn) wcs = Tan(fn) else: print('File', fn, 'does not exist; faking WCS') from unwise_coadd import get_coadd_tile_wcs wcs = get_coadd_tile_wcs(tile.ra, tile.dec) r0, r1, d0, d1 = wcs.radec_bounds() print('RA,Dec bounds:', r0, r1, d0, d1) H, W = wcs.get_height(), wcs.get_width() if T is None: T = merge_tables([ fits_table( fn, columns=[ x.upper() for x in [ 'chi2_psf_r', 'chi2_model_r', 'mag_psf_r', #'mag_disk_r', 'mag_spheroid_r', 'spheroid_reff_world', 'spheroid_aspect_world', 'spheroid_theta_world', #'disk_scale_world', 'disk_aspect_world', #'disk_theta_world', 'alphamodel_j2000', 'deltamodel_j2000' ] ], column_map=dict( CHI2_PSF_R='chi2_psf', CHI2_MODEL_R='chi2_model', MAG_PSF_R='mag_psf', MAG_SPHEROID_R='mag_spheroid_r', )) for fn in ['DES_SNX3cat_000001.fits', 'DES_SNX3cat_000002.fits'] ]) T.mag_disk = np.zeros(len(T), np.float32) + 99. print('Read total of', len(T), 'DES sources') ok, T.x, T.y = wcs.radec2pixelxy(T.alphamodel_j2000, T.deltamodel_j2000) margin = int(60. * wcs.pixel_scale()) print('Margin:', margin, 'pixels') T.cut((T.x > -margin) * (T.x < (W + margin)) * (T.y > -margin) * (T.y < (H + margin))) print('Cut to', len(T), 'in bounds') if opt.photoObjsOnly: return print(len(T), 'objects') if len(T) == 0: return defaultflux = 1. # hack T.x = (T.x - 1.).astype(np.float32) T.y = (T.y - 1.).astype(np.float32) margin = 20. I = np.flatnonzero((T.x >= -margin) * (T.x < W + margin) * (T.y >= -margin) * (T.y < H + margin)) T.cut(I) print('Cut to margins: N objects:', len(T)) if len(T) == 0: return wanyband = wband = 'w' classmap = {} print('Creating tractor sources...') cat = get_se_modelfit_cat(T, bands=[wanyband]) print('Created', len(T), 'sources') assert (len(cat) == len(T)) pixscale = wcs.pixel_scale() # crude intrinsic source radii, in pixels sourcerad = np.zeros(len(cat)) for i in range(len(cat)): src = cat[i] if isinstance(src, PointSource): continue elif isinstance(src, HoggGalaxy): sourcerad[i] = (src.nre * src.shape.re / pixscale) elif isinstance(src, FixedCompositeGalaxy): sourcerad[i] = max(src.shapeExp.re * ExpGalaxy.nre, src.shapeDev.re * DevGalaxy.nre) / pixscale print('sourcerad range:', min(sourcerad), max(sourcerad)) # Find WISE-only catalog sources wfn = os.path.join(tempoutdir, 'wise-sources-%s.fits' % (tile.coadd_id)) WISE = read_wise_sources(wfn, r0, r1, d0, d1, allwise=True) for band in bands: mag = WISE.get('w%impro' % band) nm = NanoMaggies.magToNanomaggies(mag) WISE.set('w%inm' % band, nm) print('Band', band, 'max WISE catalog flux:', max(nm)) print(' (min mag:', mag.min(), ')') unmatched = np.ones(len(WISE), bool) I, J, d = match_radec(WISE.ra, WISE.dec, T.ra, T.dec, 4. / 3600.) unmatched[I] = False UW = WISE[unmatched] print('Got', len(UW), 'unmatched WISE sources') if opt.savewise: fitwiseflux = {} for band in bands: fitwiseflux[band] = np.zeros(len(UW)) # Record WISE fluxes for catalog matches. # (this provides decent initialization for 'minsb' approx.) wiseflux = {} for band in bands: wiseflux[band] = np.zeros(len(T)) if len(I) == 0: continue # X[I] += Y[J] with duplicate I doesn't work. #wiseflux[band][J] += WISE.get('w%inm' % band)[I] lhs = wiseflux[band] rhs = WISE.get('w%inm' % band)[I] print('Band', band, 'max matched WISE flux:', max(rhs)) for j, f in zip(J, rhs): lhs[j] += f ok, UW.x, UW.y = wcs.radec2pixelxy(UW.ra, UW.dec) UW.x -= 1. UW.y -= 1. T.coadd_id = np.array([tile.coadd_id] * len(T)) inbounds = np.flatnonzero( (T.x >= -0.5) * (T.x < W - 0.5) * (T.y >= -0.5) * (T.y < H - 0.5)) print('Before looping over bands:', Time() - tt0) for band in bands: tb0 = Time() print() print('Coadd tile', tile.coadd_id) print('Band', band) wband = 'w%i' % band imfn = os.path.join(thisdir, 'unwise-%s-w%i-img-m.fits' % (tile.coadd_id, band)) ivfn = os.path.join( thisdir, 'unwise-%s-w%i-invvar-m.fits.gz' % (tile.coadd_id, band)) ppfn = os.path.join( thisdir, 'unwise-%s-w%i-std-m.fits.gz' % (tile.coadd_id, band)) nifn = os.path.join( thisdir, 'unwise-%s-w%i-n-m.fits.gz' % (tile.coadd_id, band)) print('Reading', imfn) wcs = Tan(imfn) r0, r1, d0, d1 = wcs.radec_bounds() print('RA,Dec bounds:', r0, r1, d0, d1) ra, dec = wcs.radec_center() print('Center:', ra, dec) img = fitsio.read(imfn) print('Reading', ivfn) invvar = fitsio.read(ivfn) print('Reading', ppfn) pp = fitsio.read(ppfn) print('Reading', nifn) nims = fitsio.read(nifn) print('Median # ims:', np.median(nims)) good = (nims > 0) invvar[np.logical_not(good)] = 0. sig1 = 1. / np.sqrt(np.median(invvar[good])) minsig = getattr(opt, 'minsig%i' % band) minsb = sig1 * minsig print('Sigma1:', sig1, 'minsig', minsig, 'minsb', minsb) # Load the average PSF model (generated by wise_psf.py) print('Reading PSF from', opt.psffn) P = fits_table(opt.psffn, hdu=band) psf = GaussianMixturePSF(P.amp, P.mean, P.var) # Render the PSF profile for figuring out source radii for # approximation purposes. R = 100 psf.radius = R pat = psf.getPointSourcePatch(0., 0.) assert (pat.x0 == pat.y0) assert (pat.x0 == -R) psfprofile = pat.patch[R, R:] #print 'PSF profile:', psfprofile # Reset default flux based on min radius defaultflux = minsb / psfprofile[opt.minradius] print('Setting default flux', defaultflux) # Set WISE source radii based on flux UW.rad = np.zeros(len(UW), int) wnm = UW.get('w%inm' % band) for r, pro in enumerate(psfprofile): flux = minsb / pro UW.rad[wnm > flux] = r UW.rad = np.maximum(UW.rad + 1, 3) # Set SDSS fluxes based on WISE catalog matches. wf = wiseflux[band] I = np.flatnonzero(wf > defaultflux) wfi = wf[I] print('Initializing', len(I), 'fluxes based on catalog matches') for i, flux in zip(I, wf[I]): assert (np.isfinite(flux)) cat[i].getBrightness().setBand(wanyband, flux) # Set SDSS radii based on WISE flux rad = np.zeros(len(I), int) for r, pro in enumerate(psfprofile): flux = minsb / pro rad[wfi > flux] = r srad2 = np.zeros(len(cat), int) srad2[I] = rad del rad # Set radii for i in range(len(cat)): src = cat[i] # set fluxes b = src.getBrightness() if b.getBand(wanyband) <= defaultflux: b.setBand(wanyband, defaultflux) R = max([opt.minradius, sourcerad[i], srad2[i]]) # ?? This is used to select which sources are in-range sourcerad[i] = R if isinstance(src, PointSource): src.fixedRadius = R src.minradius = opt.minradius elif (isinstance(src, HoggGalaxy) or isinstance(src, FixedCompositeGalaxy)): src.halfsize = R # We used to dice the image into blocks/cells... fullIV = np.zeros(len(cat)) fskeys = [ 'prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp' ] fitstats = dict([(k, np.zeros(len(cat))) for k in fskeys]) twcs = ConstantFitsWcs(wcs) sky = 0. tsky = ConstantSky(sky) if ps: tag = '%s W%i' % (tile.coadd_id, band) plt.clf() n, b, p = plt.hist(img.ravel(), bins=100, range=(-10 * sig1, 20 * sig1), log=True, histtype='step', color='b') mx = max(n) plt.ylim(0.1, mx) plt.xlim(-10 * sig1, 20 * sig1) plt.axvline(sky, color='r') plt.title('%s: Pixel histogram' % tag) ps.savefig() if savepickle: mods = [] cats = [] # SDSS and WISE source margins beyond the image margins ( + source radii ) smargin = 1 wmargin = 1 tim = Image(data=img, invvar=invvar, psf=psf, wcs=twcs, sky=tsky, photocal=LinearPhotoCal(1., band=wanyband), name='Coadd %s W%i' % (tile.coadd_id, band)) # Relevant SDSS sources: m = smargin + sourcerad I = np.flatnonzero(((T.x + m) >= -0.5) * ((T.x - m) < (W - 0.5)) * ((T.y + m) >= -0.5) * ((T.y - m) < (H - 0.5))) inbox = ((T.x[I] >= -0.5) * (T.x[I] < (W - 0.5)) * (T.y[I] >= -0.5) * (T.y[I] < (H - 0.5))) # Inside this cell srci = I[inbox] # In the margin margi = I[np.logical_not(inbox)] # sources in the ROI box subcat = [cat[i] for i in srci] # include *copies* of sources in the margins # (that way we automatically don't save the results) subcat.extend([cat[i].copy() for i in margi]) assert (len(subcat) == len(I)) # add WISE-only sources in the expanded region m = wmargin + UW.rad J = np.flatnonzero(((UW.x + m) >= -0.5) * ((UW.x - m) < (W - 0.5)) * ((UW.y + m) >= -0.5) * ((UW.y - m) < (H - 0.5))) if opt.savewise: jinbox = ((UW.x[J] >= -0.5) * (UW.x[J] < (W - 0.5)) * (UW.y[J] >= -0.5) * (UW.y[J] < (H - 0.5))) uwcat = [] wnm = UW.get('w%inm' % band) nomag = 0 for ji, j in enumerate(J): if not np.isfinite(wnm[j]): nomag += 1 continue ptsrc = PointSource(RaDecPos(UW.ra[j], UW.dec[j]), NanoMaggies(**{wanyband: wnm[j]})) ptsrc.radius = UW.rad[j] subcat.append(ptsrc) if opt.savewise: if jinbox[ji]: uwcat.append((j, ptsrc)) print('WISE-only:', nomag, 'of', len(J), 'had invalid mags') print('Sources:', len(srci), 'in the box,', len(I) - len(srci), 'in the margins, and', len(J), 'WISE-only') print('Creating a Tractor with image', tim.shape, 'and', len(subcat), 'sources') tractor = Tractor([tim], subcat) tractor.disable_cache() print('Running forced photometry...') t0 = Time() tractor.freezeParamsRecursive('*') if opt.sky: tractor.thawPathsTo('sky') print('Initial sky values:') for tim in tractor.getImages(): print(tim.getSky()) tractor.thawPathsTo(wanyband) wantims = (savepickle or (ps is not None) or opt.save_fits) R = tractor.optimize_forced_photometry(minsb=minsb, mindlnp=1., sky=opt.sky, minFlux=None, fitstats=True, fitstat_extras=[('pronexp', [nims])], variance=True, shared_params=False, use_ceres=opt.ceres, BW=opt.ceresblock, BH=opt.ceresblock, wantims=wantims, negfluxval=0.1 * sig1) print('That took', Time() - t0) if wantims: ims0 = R.ims0 ims1 = R.ims1 IV, fs = R.IV, R.fitstats if opt.sky: print('Fit sky values:') for tim in tractor.getImages(): print(tim.getSky()) if opt.savewise: for (j, src) in uwcat: fitwiseflux[band][j] = src.getBrightness().getBand(wanyband) if opt.save_fits: (dat, mod, ie, chi, roi) = ims1[0] tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True) fitsio.write('%s-mod.fits' % tag, mod, clobber=True) fitsio.write('%s-chi.fits' % tag, chi, clobber=True) if ps: tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, ie, chi, roi) = ims1[0] plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() # plt.clf() # plt.imshow(1./ie, interpolation='nearest', origin='lower', # cmap='gray', vmin=0, vmax=10*sig1) # plt.colorbar() # plt.title('%s: sigma' % tag) # ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() # plt.clf() # plt.imshow(np.round(chi), interpolation='nearest', origin='lower', # cmap='jet', vmin=-5, vmax=+5) # plt.colorbar() # plt.title('Chi') # ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-20, vmax=+20) plt.colorbar() plt.title('%s: chi 2' % tag) ps.savefig() plt.clf() n, b, p = plt.hist(chi.ravel(), bins=100, range=(-10, 10), log=True, histtype='step', color='b') mx = max(n) plt.ylim(0.1, mx) plt.axvline(0, color='r') plt.title('%s: chi' % tag) ps.savefig() # fn = ps.basefn + '-chi.fits' # fitsio.write(fn, chi, clobber=True) # print 'Wrote', fn if savepickle: if ims1 is None: mod = None else: im, mod, ie, chi, roi = ims1[0] mods.append(mod) cats.append((srci, margi, UW.x[J], UW.y[J], T.x[srci], T.y[srci], T.x[margi], T.y[margi], [src.copy() for src in cat], [src.copy() for src in subcat])) if len(srci): # Save fit stats fullIV[srci] = IV[:len(srci)] for k in fskeys: x = getattr(fs, k) fitstats[k][srci] = np.array(x) nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = fullIV T.set(wband + '_nanomaggies', nm.astype(np.float32)) T.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32)) dnm = np.zeros(len(nm_ivar), np.float32) okiv = (nm_ivar > 0) dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32) okflux = (nm > 0) mag = np.zeros(len(nm), np.float32) mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])).astype( np.float32) dmag = np.zeros(len(nm), np.float32) ok = (okiv * okflux) dmag[ok] = (np.abs( (-2.5 / np.log(10.)) * dnm[ok] / nm[ok])).astype(np.float32) mag[np.logical_not(okflux)] = np.nan dmag[np.logical_not(ok)] = np.nan T.set(wband + '_mag', mag) T.set(wband + '_mag_err', dmag) for k in fskeys: T.set(wband + '_' + k, fitstats[k].astype(np.float32)) if ps: I, J, d = match_radec(WISE.ra, WISE.dec, T.ra, T.dec, 4. / 3600.) plt.clf() lo, cathi = 10, 18 if band == 3: lo, cathi = 8, 13 elif band == 4: #lo,cathi = 4.5, 10.5 lo, cathi = 4.5, 12 loghist(WISE.get('w%impro' % band)[I], T.get(wband + '_mag')[J], range=((lo, cathi), (lo, cathi)), bins=200) plt.xlabel('WISE W%i mag' % band) plt.ylabel('Tractor W%i mag' % band) plt.title('WISE catalog vs Tractor forced photometry') plt.axis([cathi, lo, cathi, lo]) ps.savefig() print('Tile', tile.coadd_id, 'band', wband, 'took', Time() - tb0) T.cut(inbounds) T.delete_column('psfflux') T.delete_column('cmodelflux') T.delete_column('devflux') T.delete_column('expflux') T.treated_as_pointsource = T.treated_as_pointsource.astype(np.uint8) T.pointsource = T.pointsource.astype(np.uint8) T.writeto(outfn, header=hdr) print('Wrote', outfn) if savepickle: fn = opt.output % (tile.coadd_id) fn = fn.replace('.fits', '.pickle') pickle_to_file((mods, cats, T, sourcerad), fn) print('Pickled', fn) print('Tile', tile.coadd_id, 'took', Time() - tt0)
I = match_radec(W.ra, W.dec, T.ra, T.dec, r, indexlist=True) from astrometry.sdss.common import munu_to_radec_deg from astrometry.util.miscutils import polygons_intersect from unwise_coadd import get_coadd_tile_wcs # Bit o' margin (pixels) margin = 20. lo,hi = 0.5 - margin, 2048.5 + margin poly1 = np.array([[lo,lo], [lo,hi], [hi,hi], [hi,lo]]) poly2 = np.zeros((4,2)) keep = [] for i,ilist in enumerate(I): print 'Tile', i, W.coadd_id[i], if ilist is None: continue wcs = get_coadd_tile_wcs(W.ra[i], W.dec[i]) ilist = np.array(ilist) ok,x,y = wcs.radec2pixelxy(T.ra[ilist], T.dec[ilist]) # center of SDSS field within tile? if np.any(ok * (x >= lo) * (x <= hi) * (y >= lo) * (y <= hi)): print 'Center within tile' keep.append(i) continue gotone = False for ii in ilist: m0,m1 = T.mu_start[ii], T.mu_end[ii] n0,n1 = T.nu_start[ii], T.nu_end[ii] node,incl = T.node[ii], T.incl[ii] mu,nu = np.array([m0,m0,m1,m1]), np.array([n0,n1,n1,n0])
def one_tile(tile, opt, savepickle, ps, tiles, tiledir, tempoutdir, T=None, hdr=None): bands = opt.bands outfn = opt.output % (tile.coadd_id) savewise_outfn = opt.save_wise_output % (tile.coadd_id) sband = 'r' bandnum = 'ugriz'.index(sband) tt0 = Time() print print 'Coadd tile', tile.coadd_id thisdir = get_unwise_tile_dir(tiledir, tile.coadd_id) fn = os.path.join(thisdir, 'unwise-%s-w%i-img-m.fits' % (tile.coadd_id, bands[0])) if os.path.exists(fn): print 'Reading', fn wcs = Tan(fn) else: print 'File', fn, 'does not exist; faking WCS' from unwise_coadd import get_coadd_tile_wcs wcs = get_coadd_tile_wcs(tile.ra, tile.dec) r0,r1,d0,d1 = wcs.radec_bounds() print 'RA,Dec bounds:', r0,r1,d0,d1 H,W = wcs.get_height(), wcs.get_width() if T is None: T = merge_tables([fits_table(fn, columns=[x.upper() for x in [ 'chi2_psf_r', 'chi2_model_r', 'mag_psf_r', #'mag_disk_r', 'mag_spheroid_r', 'spheroid_reff_world', 'spheroid_aspect_world', 'spheroid_theta_world', #'disk_scale_world', 'disk_aspect_world', #'disk_theta_world', 'alphamodel_j2000', 'deltamodel_j2000']], column_map=dict(CHI2_PSF_R='chi2_psf', CHI2_MODEL_R='chi2_model', MAG_PSF_R='mag_psf', MAG_SPHEROID_R='mag_spheroid_r', )) for fn in ['DES_SNX3cat_000001.fits', 'DES_SNX3cat_000002.fits']] ) T.mag_disk = np.zeros(len(T), np.float32) + 99. print 'Read total of', len(T), 'DES sources' ok,T.x,T.y = wcs.radec2pixelxy(T.alphamodel_j2000, T.deltamodel_j2000) margin = int(60. * wcs.pixel_scale()) print 'Margin:', margin, 'pixels' T.cut((T.x > -margin) * (T.x < (W+margin)) * (T.y > -margin) * (T.y < (H+margin))) print 'Cut to', len(T), 'in bounds' if opt.photoObjsOnly: return print len(T), 'objects' if len(T) == 0: return defaultflux = 1. # hack T.x = (T.x - 1.).astype(np.float32) T.y = (T.y - 1.).astype(np.float32) margin = 20. I = np.flatnonzero((T.x >= -margin) * (T.x < W+margin) * (T.y >= -margin) * (T.y < H+margin)) T.cut(I) print 'Cut to margins: N objects:', len(T) if len(T) == 0: return wanyband = wband = 'w' classmap = {} print 'Creating tractor sources...' cat = get_se_modelfit_cat(T, bands=[wanyband]) print 'Created', len(T), 'sources' assert(len(cat) == len(T)) pixscale = wcs.pixel_scale() # crude intrinsic source radii, in pixels sourcerad = np.zeros(len(cat)) for i in range(len(cat)): src = cat[i] if isinstance(src, PointSource): continue elif isinstance(src, HoggGalaxy): sourcerad[i] = (src.nre * src.shape.re / pixscale) elif isinstance(src, FixedCompositeGalaxy): sourcerad[i] = max(src.shapeExp.re * ExpGalaxy.nre, src.shapeDev.re * DevGalaxy.nre) / pixscale print 'sourcerad range:', min(sourcerad), max(sourcerad) # Find WISE-only catalog sources wfn = os.path.join(tempoutdir, 'wise-sources-%s.fits' % (tile.coadd_id)) WISE = read_wise_sources(wfn, r0,r1,d0,d1, allwise=True) for band in bands: mag = WISE.get('w%impro' % band) nm = NanoMaggies.magToNanomaggies(mag) WISE.set('w%inm' % band, nm) print 'Band', band, 'max WISE catalog flux:', max(nm) print ' (min mag:', mag.min(), ')' unmatched = np.ones(len(WISE), bool) I,J,d = match_radec(WISE.ra, WISE.dec, T.ra, T.dec, 4./3600.) unmatched[I] = False UW = WISE[unmatched] print 'Got', len(UW), 'unmatched WISE sources' if opt.savewise: fitwiseflux = {} for band in bands: fitwiseflux[band] = np.zeros(len(UW)) # Record WISE fluxes for catalog matches. # (this provides decent initialization for 'minsb' approx.) wiseflux = {} for band in bands: wiseflux[band] = np.zeros(len(T)) if len(I) == 0: continue # X[I] += Y[J] with duplicate I doesn't work. #wiseflux[band][J] += WISE.get('w%inm' % band)[I] lhs = wiseflux[band] rhs = WISE.get('w%inm' % band)[I] print 'Band', band, 'max matched WISE flux:', max(rhs) for j,f in zip(J, rhs): lhs[j] += f ok,UW.x,UW.y = wcs.radec2pixelxy(UW.ra, UW.dec) UW.x -= 1. UW.y -= 1. T.coadd_id = np.array([tile.coadd_id] * len(T)) inbounds = np.flatnonzero((T.x >= -0.5) * (T.x < W-0.5) * (T.y >= -0.5) * (T.y < H-0.5)) print 'Before looping over bands:', Time()-tt0 for band in bands: tb0 = Time() print print 'Coadd tile', tile.coadd_id print 'Band', band wband = 'w%i' % band imfn = os.path.join(thisdir, 'unwise-%s-w%i-img-m.fits' % (tile.coadd_id, band)) ivfn = os.path.join(thisdir, 'unwise-%s-w%i-invvar-m.fits.gz' % (tile.coadd_id, band)) ppfn = os.path.join(thisdir, 'unwise-%s-w%i-std-m.fits.gz' % (tile.coadd_id, band)) nifn = os.path.join(thisdir, 'unwise-%s-w%i-n-m.fits.gz' % (tile.coadd_id, band)) print 'Reading', imfn wcs = Tan(imfn) r0,r1,d0,d1 = wcs.radec_bounds() print 'RA,Dec bounds:', r0,r1,d0,d1 ra,dec = wcs.radec_center() print 'Center:', ra,dec img = fitsio.read(imfn) print 'Reading', ivfn invvar = fitsio.read(ivfn) print 'Reading', ppfn pp = fitsio.read(ppfn) print 'Reading', nifn nims = fitsio.read(nifn) print 'Median # ims:', np.median(nims) good = (nims > 0) invvar[np.logical_not(good)] = 0. sig1 = 1./np.sqrt(np.median(invvar[good])) minsig = getattr(opt, 'minsig%i' % band) minsb = sig1 * minsig print 'Sigma1:', sig1, 'minsig', minsig, 'minsb', minsb # Load the average PSF model (generated by wise_psf.py) print 'Reading PSF from', opt.psffn P = fits_table(opt.psffn, hdu=band) psf = GaussianMixturePSF(P.amp, P.mean, P.var) # Render the PSF profile for figuring out source radii for # approximation purposes. R = 100 psf.radius = R pat = psf.getPointSourcePatch(0., 0.) assert(pat.x0 == pat.y0) assert(pat.x0 == -R) psfprofile = pat.patch[R, R:] #print 'PSF profile:', psfprofile # Reset default flux based on min radius defaultflux = minsb / psfprofile[opt.minradius] print 'Setting default flux', defaultflux # Set WISE source radii based on flux UW.rad = np.zeros(len(UW), int) wnm = UW.get('w%inm' % band) for r,pro in enumerate(psfprofile): flux = minsb / pro UW.rad[wnm > flux] = r UW.rad = np.maximum(UW.rad + 1, 3) # Set SDSS fluxes based on WISE catalog matches. wf = wiseflux[band] I = np.flatnonzero(wf > defaultflux) wfi = wf[I] print 'Initializing', len(I), 'fluxes based on catalog matches' for i,flux in zip(I, wf[I]): assert(np.isfinite(flux)) cat[i].getBrightness().setBand(wanyband, flux) # Set SDSS radii based on WISE flux rad = np.zeros(len(I), int) for r,pro in enumerate(psfprofile): flux = minsb / pro rad[wfi > flux] = r srad2 = np.zeros(len(cat), int) srad2[I] = rad del rad # Set radii for i in range(len(cat)): src = cat[i] # set fluxes b = src.getBrightness() if b.getBand(wanyband) <= defaultflux: b.setBand(wanyband, defaultflux) R = max([opt.minradius, sourcerad[i], srad2[i]]) # ?? This is used to select which sources are in-range sourcerad[i] = R if isinstance(src, PointSource): src.fixedRadius = R src.minradius = opt.minradius elif (isinstance(src, HoggGalaxy) or isinstance(src, FixedCompositeGalaxy)): src.halfsize = R # We used to dice the image into blocks/cells... fullIV = np.zeros(len(cat)) fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp'] fitstats = dict([(k, np.zeros(len(cat))) for k in fskeys]) twcs = ConstantFitsWcs(wcs) sky = 0. tsky = ConstantSky(sky) if ps: tag = '%s W%i' % (tile.coadd_id, band) plt.clf() n,b,p = plt.hist(img.ravel(), bins=100, range=(-10*sig1, 20*sig1), log=True, histtype='step', color='b') mx = max(n) plt.ylim(0.1, mx) plt.xlim(-10*sig1, 20*sig1) plt.axvline(sky, color='r') plt.title('%s: Pixel histogram' % tag) ps.savefig() if savepickle: mods = [] cats = [] # SDSS and WISE source margins beyond the image margins ( + source radii ) smargin = 1 wmargin = 1 tim = Image(data=img, invvar=invvar, psf=psf, wcs=twcs, sky=tsky, photocal=LinearPhotoCal(1., band=wanyband), name='Coadd %s W%i' % (tile.coadd_id, band)) # Relevant SDSS sources: m = smargin + sourcerad I = np.flatnonzero(((T.x+m) >= -0.5) * ((T.x-m) < (W-0.5)) * ((T.y+m) >= -0.5) * ((T.y-m) < (H-0.5))) inbox = ((T.x[I] >= -0.5) * (T.x[I] < (W-0.5)) * (T.y[I] >= -0.5) * (T.y[I] < (H-0.5))) # Inside this cell srci = I[inbox] # In the margin margi = I[np.logical_not(inbox)] # sources in the ROI box subcat = [cat[i] for i in srci] # include *copies* of sources in the margins # (that way we automatically don't save the results) subcat.extend([cat[i].copy() for i in margi]) assert(len(subcat) == len(I)) # add WISE-only sources in the expanded region m = wmargin + UW.rad J = np.flatnonzero(((UW.x+m) >= -0.5) * ((UW.x-m) < (W-0.5)) * ((UW.y+m) >= -0.5) * ((UW.y-m) < (H-0.5))) if opt.savewise: jinbox = ((UW.x[J] >= -0.5) * (UW.x[J] < (W-0.5)) * (UW.y[J] >= -0.5) * (UW.y[J] < (H-0.5))) uwcat = [] wnm = UW.get('w%inm' % band) nomag = 0 for ji,j in enumerate(J): if not np.isfinite(wnm[j]): nomag += 1 continue ptsrc = PointSource(RaDecPos(UW.ra[j], UW.dec[j]), NanoMaggies(**{wanyband: wnm[j]})) ptsrc.radius = UW.rad[j] subcat.append(ptsrc) if opt.savewise: if jinbox[ji]: uwcat.append((j, ptsrc)) print 'WISE-only:', nomag, 'of', len(J), 'had invalid mags' print 'Sources:', len(srci), 'in the box,', len(I)-len(srci), 'in the margins, and', len(J), 'WISE-only' print 'Creating a Tractor with image', tim.shape, 'and', len(subcat), 'sources' tractor = Tractor([tim], subcat) tractor.disable_cache() print 'Running forced photometry...' t0 = Time() tractor.freezeParamsRecursive('*') if opt.sky: tractor.thawPathsTo('sky') print 'Initial sky values:' for tim in tractor.getImages(): print tim.getSky() tractor.thawPathsTo(wanyband) wantims = (savepickle or (ps is not None) or opt.save_fits) R = tractor.optimize_forced_photometry( minsb=minsb, mindlnp=1., sky=opt.sky, minFlux=None, fitstats=True, fitstat_extras=[('pronexp', [nims])], variance=True, shared_params=False, use_ceres=opt.ceres, BW=opt.ceresblock, BH=opt.ceresblock, wantims=wantims, negfluxval=0.1*sig1) print 'That took', Time()-t0 if wantims: ims0 = R.ims0 ims1 = R.ims1 IV,fs = R.IV, R.fitstats if opt.sky: print 'Fit sky values:' for tim in tractor.getImages(): print tim.getSky() if opt.savewise: for (j,src) in uwcat: fitwiseflux[band][j] = src.getBrightness().getBand(wanyband) if opt.save_fits: (dat,mod,ie,chi,roi) = ims1[0] tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True) fitsio.write('%s-mod.fits' % tag, mod, clobber=True) fitsio.write('%s-chi.fits' % tag, chi, clobber=True) if ps: tag = '%s W%i' % (tile.coadd_id, band) (dat,mod,ie,chi,roi) = ims1[0] plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3*sig1, vmax=10*sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() # plt.clf() # plt.imshow(1./ie, interpolation='nearest', origin='lower', # cmap='gray', vmin=0, vmax=10*sig1) # plt.colorbar() # plt.title('%s: sigma' % tag) # ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3*sig1, vmax=10*sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() # plt.clf() # plt.imshow(np.round(chi), interpolation='nearest', origin='lower', # cmap='jet', vmin=-5, vmax=+5) # plt.colorbar() # plt.title('Chi') # ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-20, vmax=+20) plt.colorbar() plt.title('%s: chi 2' % tag) ps.savefig() plt.clf() n,b,p = plt.hist(chi.ravel(), bins=100, range=(-10, 10), log=True, histtype='step', color='b') mx = max(n) plt.ylim(0.1, mx) plt.axvline(0, color='r') plt.title('%s: chi' % tag) ps.savefig() # fn = ps.basefn + '-chi.fits' # fitsio.write(fn, chi, clobber=True) # print 'Wrote', fn if savepickle: if ims1 is None: mod = None else: im,mod,ie,chi,roi = ims1[0] mods.append(mod) cats.append(( srci, margi, UW.x[J], UW.y[J], T.x[srci], T.y[srci], T.x[margi], T.y[margi], [src.copy() for src in cat], [src.copy() for src in subcat])) if len(srci): # Save fit stats fullIV[srci] = IV[:len(srci)] for k in fskeys: x = getattr(fs, k) fitstats[k][srci] = np.array(x) nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = fullIV T.set(wband + '_nanomaggies', nm.astype(np.float32)) T.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32)) dnm = np.zeros(len(nm_ivar), np.float32) okiv = (nm_ivar > 0) dnm[okiv] = (1./np.sqrt(nm_ivar[okiv])).astype(np.float32) okflux = (nm > 0) mag = np.zeros(len(nm), np.float32) mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])).astype(np.float32) dmag = np.zeros(len(nm), np.float32) ok = (okiv * okflux) dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok])).astype(np.float32) mag[np.logical_not(okflux)] = np.nan dmag[np.logical_not(ok)] = np.nan T.set(wband + '_mag', mag) T.set(wband + '_mag_err', dmag) for k in fskeys: T.set(wband + '_' + k, fitstats[k].astype(np.float32)) if ps: I,J,d = match_radec(WISE.ra, WISE.dec, T.ra, T.dec, 4./3600.) plt.clf() lo,cathi = 10,18 if band == 3: lo,cathi = 8, 13 elif band == 4: #lo,cathi = 4.5, 10.5 lo,cathi = 4.5, 12 loghist(WISE.get('w%impro'%band)[I], T.get(wband+'_mag')[J], range=((lo,cathi),(lo,cathi)), bins=200) plt.xlabel('WISE W%i mag' % band) plt.ylabel('Tractor W%i mag' % band) plt.title('WISE catalog vs Tractor forced photometry') plt.axis([cathi,lo,cathi,lo]) ps.savefig() print 'Tile', tile.coadd_id, 'band', wband, 'took', Time()-tb0 T.cut(inbounds) T.delete_column('psfflux') T.delete_column('cmodelflux') T.delete_column('devflux') T.delete_column('expflux') T.treated_as_pointsource = T.treated_as_pointsource.astype(np.uint8) T.pointsource = T.pointsource.astype(np.uint8) T.writeto(outfn, header=hdr) print 'Wrote', outfn if savepickle: fn = opt.output % (tile.coadd_id) fn = fn.replace('.fits','.pickle') pickle_to_file((mods, cats, T, sourcerad), fn) print 'Pickled', fn print 'Tile', tile.coadd_id, 'took', Time()-tt0