def one_tile(tile,
opt,
savepickle,
ps,
tiles,
tiledir,
tempoutdir,
T=None,
hdr=None):
bands = opt.bands
outfn = opt.output % (tile.coadd_id)
savewise_outfn = opt.save_wise_output % (tile.coadd_id)
sband = 'r'
bandnum = 'ugriz'.index(sband)
tt0 = Time()
print()
print('Coadd tile', tile.coadd_id)
thisdir = get_unwise_tile_dir(tiledir, tile.coadd_id)
fn = os.path.join(thisdir,
'unwise-%s-w%i-img-m.fits' % (tile.coadd_id, bands[0]))
if os.path.exists(fn):
print('Reading', fn)
wcs = Tan(fn)
else:
print('File', fn, 'does not exist; faking WCS')
from unwise_coadd import get_coadd_tile_wcs
wcs = get_coadd_tile_wcs(tile.ra, tile.dec)
r0, r1, d0, d1 = wcs.radec_bounds()
print('RA,Dec bounds:', r0, r1, d0, d1)
H, W = wcs.get_height(), wcs.get_width()
if T is None:
T = merge_tables([
fits_table(
fn,
columns=[
x.upper() for x in [
'chi2_psf_r',
'chi2_model_r',
'mag_psf_r',
#'mag_disk_r',
'mag_spheroid_r',
'spheroid_reff_world',
'spheroid_aspect_world',
'spheroid_theta_world',
#'disk_scale_world', 'disk_aspect_world',
#'disk_theta_world',
'alphamodel_j2000',
'deltamodel_j2000'
]
],
column_map=dict(
CHI2_PSF_R='chi2_psf',
CHI2_MODEL_R='chi2_model',
MAG_PSF_R='mag_psf',
MAG_SPHEROID_R='mag_spheroid_r',
))
for fn in ['DES_SNX3cat_000001.fits', 'DES_SNX3cat_000002.fits']
])
T.mag_disk = np.zeros(len(T), np.float32) + 99.
print('Read total of', len(T), 'DES sources')
ok, T.x, T.y = wcs.radec2pixelxy(T.alphamodel_j2000,
T.deltamodel_j2000)
margin = int(60. * wcs.pixel_scale())
print('Margin:', margin, 'pixels')
T.cut((T.x > -margin) * (T.x < (W + margin)) * (T.y > -margin) *
(T.y < (H + margin)))
print('Cut to', len(T), 'in bounds')
if opt.photoObjsOnly:
return
print(len(T), 'objects')
if len(T) == 0:
return
defaultflux = 1.
# hack
T.x = (T.x - 1.).astype(np.float32)
T.y = (T.y - 1.).astype(np.float32)
margin = 20.
I = np.flatnonzero((T.x >= -margin) * (T.x < W + margin) *
(T.y >= -margin) * (T.y < H + margin))
T.cut(I)
print('Cut to margins: N objects:', len(T))
if len(T) == 0:
return
wanyband = wband = 'w'
classmap = {}
print('Creating tractor sources...')
cat = get_se_modelfit_cat(T, bands=[wanyband])
print('Created', len(T), 'sources')
assert (len(cat) == len(T))
pixscale = wcs.pixel_scale()
# crude intrinsic source radii, in pixels
sourcerad = np.zeros(len(cat))
for i in range(len(cat)):
src = cat[i]
if isinstance(src, PointSource):
continue
elif isinstance(src, HoggGalaxy):
sourcerad[i] = (src.nre * src.shape.re / pixscale)
elif isinstance(src, FixedCompositeGalaxy):
sourcerad[i] = max(src.shapeExp.re * ExpGalaxy.nre,
src.shapeDev.re * DevGalaxy.nre) / pixscale
print('sourcerad range:', min(sourcerad), max(sourcerad))
# Find WISE-only catalog sources
wfn = os.path.join(tempoutdir, 'wise-sources-%s.fits' % (tile.coadd_id))
WISE = read_wise_sources(wfn, r0, r1, d0, d1, allwise=True)
for band in bands:
mag = WISE.get('w%impro' % band)
nm = NanoMaggies.magToNanomaggies(mag)
WISE.set('w%inm' % band, nm)
print('Band', band, 'max WISE catalog flux:', max(nm))
print(' (min mag:', mag.min(), ')')
unmatched = np.ones(len(WISE), bool)
I, J, d = match_radec(WISE.ra, WISE.dec, T.ra, T.dec, 4. / 3600.)
unmatched[I] = False
UW = WISE[unmatched]
print('Got', len(UW), 'unmatched WISE sources')
if opt.savewise:
fitwiseflux = {}
for band in bands:
fitwiseflux[band] = np.zeros(len(UW))
# Record WISE fluxes for catalog matches.
# (this provides decent initialization for 'minsb' approx.)
wiseflux = {}
for band in bands:
wiseflux[band] = np.zeros(len(T))
if len(I) == 0:
continue
# X[I] += Y[J] with duplicate I doesn't work.
#wiseflux[band][J] += WISE.get('w%inm' % band)[I]
lhs = wiseflux[band]
rhs = WISE.get('w%inm' % band)[I]
print('Band', band, 'max matched WISE flux:', max(rhs))
for j, f in zip(J, rhs):
lhs[j] += f
ok, UW.x, UW.y = wcs.radec2pixelxy(UW.ra, UW.dec)
UW.x -= 1.
UW.y -= 1.
T.coadd_id = np.array([tile.coadd_id] * len(T))
inbounds = np.flatnonzero(
(T.x >= -0.5) * (T.x < W - 0.5) * (T.y >= -0.5) * (T.y < H - 0.5))
print('Before looping over bands:', Time() - tt0)
for band in bands:
tb0 = Time()
print()
print('Coadd tile', tile.coadd_id)
print('Band', band)
wband = 'w%i' % band
imfn = os.path.join(thisdir,
'unwise-%s-w%i-img-m.fits' % (tile.coadd_id, band))
ivfn = os.path.join(
thisdir, 'unwise-%s-w%i-invvar-m.fits.gz' % (tile.coadd_id, band))
ppfn = os.path.join(
thisdir, 'unwise-%s-w%i-std-m.fits.gz' % (tile.coadd_id, band))
nifn = os.path.join(
thisdir, 'unwise-%s-w%i-n-m.fits.gz' % (tile.coadd_id, band))
print('Reading', imfn)
wcs = Tan(imfn)
r0, r1, d0, d1 = wcs.radec_bounds()
print('RA,Dec bounds:', r0, r1, d0, d1)
ra, dec = wcs.radec_center()
print('Center:', ra, dec)
img = fitsio.read(imfn)
print('Reading', ivfn)
invvar = fitsio.read(ivfn)
print('Reading', ppfn)
pp = fitsio.read(ppfn)
print('Reading', nifn)
nims = fitsio.read(nifn)
print('Median # ims:', np.median(nims))
good = (nims > 0)
invvar[np.logical_not(good)] = 0.
sig1 = 1. / np.sqrt(np.median(invvar[good]))
minsig = getattr(opt, 'minsig%i' % band)
minsb = sig1 * minsig
print('Sigma1:', sig1, 'minsig', minsig, 'minsb', minsb)
# Load the average PSF model (generated by wise_psf.py)
print('Reading PSF from', opt.psffn)
P = fits_table(opt.psffn, hdu=band)
psf = GaussianMixturePSF(P.amp, P.mean, P.var)
# Render the PSF profile for figuring out source radii for
# approximation purposes.
R = 100
psf.radius = R
pat = psf.getPointSourcePatch(0., 0.)
assert (pat.x0 == pat.y0)
assert (pat.x0 == -R)
psfprofile = pat.patch[R, R:]
#print 'PSF profile:', psfprofile
# Reset default flux based on min radius
defaultflux = minsb / psfprofile[opt.minradius]
print('Setting default flux', defaultflux)
# Set WISE source radii based on flux
UW.rad = np.zeros(len(UW), int)
wnm = UW.get('w%inm' % band)
for r, pro in enumerate(psfprofile):
flux = minsb / pro
UW.rad[wnm > flux] = r
UW.rad = np.maximum(UW.rad + 1, 3)
# Set SDSS fluxes based on WISE catalog matches.
wf = wiseflux[band]
I = np.flatnonzero(wf > defaultflux)
wfi = wf[I]
print('Initializing', len(I), 'fluxes based on catalog matches')
for i, flux in zip(I, wf[I]):
assert (np.isfinite(flux))
cat[i].getBrightness().setBand(wanyband, flux)
# Set SDSS radii based on WISE flux
rad = np.zeros(len(I), int)
for r, pro in enumerate(psfprofile):
flux = minsb / pro
rad[wfi > flux] = r
srad2 = np.zeros(len(cat), int)
srad2[I] = rad
del rad
# Set radii
for i in range(len(cat)):
src = cat[i]
# set fluxes
b = src.getBrightness()
if b.getBand(wanyband) <= defaultflux:
b.setBand(wanyband, defaultflux)
R = max([opt.minradius, sourcerad[i], srad2[i]])
# ?? This is used to select which sources are in-range
sourcerad[i] = R
if isinstance(src, PointSource):
src.fixedRadius = R
src.minradius = opt.minradius
elif (isinstance(src, HoggGalaxy)
or isinstance(src, FixedCompositeGalaxy)):
src.halfsize = R
# We used to dice the image into blocks/cells...
fullIV = np.zeros(len(cat))
fskeys = [
'prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp'
]
fitstats = dict([(k, np.zeros(len(cat))) for k in fskeys])
twcs = ConstantFitsWcs(wcs)
sky = 0.
tsky = ConstantSky(sky)
if ps:
tag = '%s W%i' % (tile.coadd_id, band)
plt.clf()
n, b, p = plt.hist(img.ravel(),
bins=100,
range=(-10 * sig1, 20 * sig1),
log=True,
histtype='step',
color='b')
mx = max(n)
plt.ylim(0.1, mx)
plt.xlim(-10 * sig1, 20 * sig1)
plt.axvline(sky, color='r')
plt.title('%s: Pixel histogram' % tag)
ps.savefig()
if savepickle:
mods = []
cats = []
# SDSS and WISE source margins beyond the image margins ( + source radii )
smargin = 1
wmargin = 1
tim = Image(data=img,
invvar=invvar,
psf=psf,
wcs=twcs,
sky=tsky,
photocal=LinearPhotoCal(1., band=wanyband),
name='Coadd %s W%i' % (tile.coadd_id, band))
# Relevant SDSS sources:
m = smargin + sourcerad
I = np.flatnonzero(((T.x + m) >= -0.5) * ((T.x - m) < (W - 0.5)) *
((T.y + m) >= -0.5) * ((T.y - m) < (H - 0.5)))
inbox = ((T.x[I] >= -0.5) * (T.x[I] < (W - 0.5)) * (T.y[I] >= -0.5) *
(T.y[I] < (H - 0.5)))
# Inside this cell
srci = I[inbox]
# In the margin
margi = I[np.logical_not(inbox)]
# sources in the ROI box
subcat = [cat[i] for i in srci]
# include *copies* of sources in the margins
# (that way we automatically don't save the results)
subcat.extend([cat[i].copy() for i in margi])
assert (len(subcat) == len(I))
# add WISE-only sources in the expanded region
m = wmargin + UW.rad
J = np.flatnonzero(((UW.x + m) >= -0.5) * ((UW.x - m) < (W - 0.5)) *
((UW.y + m) >= -0.5) * ((UW.y - m) < (H - 0.5)))
if opt.savewise:
jinbox = ((UW.x[J] >= -0.5) * (UW.x[J] < (W - 0.5)) *
(UW.y[J] >= -0.5) * (UW.y[J] < (H - 0.5)))
uwcat = []
wnm = UW.get('w%inm' % band)
nomag = 0
for ji, j in enumerate(J):
if not np.isfinite(wnm[j]):
nomag += 1
continue
ptsrc = PointSource(RaDecPos(UW.ra[j], UW.dec[j]),
NanoMaggies(**{wanyband: wnm[j]}))
ptsrc.radius = UW.rad[j]
subcat.append(ptsrc)
if opt.savewise:
if jinbox[ji]:
uwcat.append((j, ptsrc))
print('WISE-only:', nomag, 'of', len(J), 'had invalid mags')
print('Sources:', len(srci), 'in the box,',
len(I) - len(srci), 'in the margins, and', len(J), 'WISE-only')
print('Creating a Tractor with image', tim.shape, 'and', len(subcat),
'sources')
tractor = Tractor([tim], subcat)
tractor.disable_cache()
print('Running forced photometry...')
t0 = Time()
tractor.freezeParamsRecursive('*')
if opt.sky:
tractor.thawPathsTo('sky')
print('Initial sky values:')
for tim in tractor.getImages():
print(tim.getSky())
tractor.thawPathsTo(wanyband)
wantims = (savepickle or (ps is not None) or opt.save_fits)
R = tractor.optimize_forced_photometry(minsb=minsb,
mindlnp=1.,
sky=opt.sky,
minFlux=None,
fitstats=True,
fitstat_extras=[('pronexp',
[nims])],
variance=True,
shared_params=False,
use_ceres=opt.ceres,
BW=opt.ceresblock,
BH=opt.ceresblock,
wantims=wantims,
negfluxval=0.1 * sig1)
print('That took', Time() - t0)
if wantims:
ims0 = R.ims0
ims1 = R.ims1
IV, fs = R.IV, R.fitstats
if opt.sky:
print('Fit sky values:')
for tim in tractor.getImages():
print(tim.getSky())
if opt.savewise:
for (j, src) in uwcat:
fitwiseflux[band][j] = src.getBrightness().getBand(wanyband)
if opt.save_fits:
(dat, mod, ie, chi, roi) = ims1[0]
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' % tag, dat, clobber=True)
fitsio.write('%s-mod.fits' % tag, mod, clobber=True)
fitsio.write('%s-chi.fits' % tag, chi, clobber=True)
if ps:
tag = '%s W%i' % (tile.coadd_id, band)
(dat, mod, ie, chi, roi) = ims1[0]
plt.clf()
plt.imshow(dat,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=10 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
# plt.clf()
# plt.imshow(1./ie, interpolation='nearest', origin='lower',
# cmap='gray', vmin=0, vmax=10*sig1)
# plt.colorbar()
# plt.title('%s: sigma' % tag)
# ps.savefig()
plt.clf()
plt.imshow(mod,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=10 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-5,
vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
# plt.clf()
# plt.imshow(np.round(chi), interpolation='nearest', origin='lower',
# cmap='jet', vmin=-5, vmax=+5)
# plt.colorbar()
# plt.title('Chi')
# ps.savefig()
plt.clf()
plt.imshow(chi,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-20,
vmax=+20)
plt.colorbar()
plt.title('%s: chi 2' % tag)
ps.savefig()
plt.clf()
n, b, p = plt.hist(chi.ravel(),
bins=100,
range=(-10, 10),
log=True,
histtype='step',
color='b')
mx = max(n)
plt.ylim(0.1, mx)
plt.axvline(0, color='r')
plt.title('%s: chi' % tag)
ps.savefig()
# fn = ps.basefn + '-chi.fits'
# fitsio.write(fn, chi, clobber=True)
# print 'Wrote', fn
if savepickle:
if ims1 is None:
mod = None
else:
im, mod, ie, chi, roi = ims1[0]
mods.append(mod)
cats.append((srci, margi, UW.x[J], UW.y[J], T.x[srci], T.y[srci],
T.x[margi], T.y[margi], [src.copy() for src in cat],
[src.copy() for src in subcat]))
if len(srci):
# Save fit stats
fullIV[srci] = IV[:len(srci)]
for k in fskeys:
x = getattr(fs, k)
fitstats[k][srci] = np.array(x)
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = fullIV
T.set(wband + '_nanomaggies', nm.astype(np.float32))
T.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32))
dnm = np.zeros(len(nm_ivar), np.float32)
okiv = (nm_ivar > 0)
dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32)
okflux = (nm > 0)
mag = np.zeros(len(nm), np.float32)
mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])).astype(
np.float32)
dmag = np.zeros(len(nm), np.float32)
ok = (okiv * okflux)
dmag[ok] = (np.abs(
(-2.5 / np.log(10.)) * dnm[ok] / nm[ok])).astype(np.float32)
mag[np.logical_not(okflux)] = np.nan
dmag[np.logical_not(ok)] = np.nan
T.set(wband + '_mag', mag)
T.set(wband + '_mag_err', dmag)
for k in fskeys:
T.set(wband + '_' + k, fitstats[k].astype(np.float32))
if ps:
I, J, d = match_radec(WISE.ra, WISE.dec, T.ra, T.dec, 4. / 3600.)
plt.clf()
lo, cathi = 10, 18
if band == 3:
lo, cathi = 8, 13
elif band == 4:
#lo,cathi = 4.5, 10.5
lo, cathi = 4.5, 12
loghist(WISE.get('w%impro' % band)[I],
T.get(wband + '_mag')[J],
range=((lo, cathi), (lo, cathi)),
bins=200)
plt.xlabel('WISE W%i mag' % band)
plt.ylabel('Tractor W%i mag' % band)
plt.title('WISE catalog vs Tractor forced photometry')
plt.axis([cathi, lo, cathi, lo])
ps.savefig()
print('Tile', tile.coadd_id, 'band', wband, 'took', Time() - tb0)
T.cut(inbounds)
T.delete_column('psfflux')
T.delete_column('cmodelflux')
T.delete_column('devflux')
T.delete_column('expflux')
T.treated_as_pointsource = T.treated_as_pointsource.astype(np.uint8)
T.pointsource = T.pointsource.astype(np.uint8)
T.writeto(outfn, header=hdr)
print('Wrote', outfn)
if savepickle:
fn = opt.output % (tile.coadd_id)
fn = fn.replace('.fits', '.pickle')
pickle_to_file((mods, cats, T, sourcerad), fn)
print('Pickled', fn)
print('Tile', tile.coadd_id, 'took', Time() - tt0)
I = match_radec(W.ra, W.dec, T.ra, T.dec, r, indexlist=True)
from astrometry.sdss.common import munu_to_radec_deg
from astrometry.util.miscutils import polygons_intersect
from unwise_coadd import get_coadd_tile_wcs
# Bit o' margin (pixels)
margin = 20.
lo,hi = 0.5 - margin, 2048.5 + margin
poly1 = np.array([[lo,lo], [lo,hi], [hi,hi], [hi,lo]])
poly2 = np.zeros((4,2))
keep = []
for i,ilist in enumerate(I):
print 'Tile', i, W.coadd_id[i],
if ilist is None:
continue
wcs = get_coadd_tile_wcs(W.ra[i], W.dec[i])
ilist = np.array(ilist)
ok,x,y = wcs.radec2pixelxy(T.ra[ilist], T.dec[ilist])
# center of SDSS field within tile?
if np.any(ok * (x >= lo) * (x <= hi) * (y >= lo) * (y <= hi)):
print 'Center within tile'
keep.append(i)
continue
gotone = False
for ii in ilist:
m0,m1 = T.mu_start[ii], T.mu_end[ii]
n0,n1 = T.nu_start[ii], T.nu_end[ii]
node,incl = T.node[ii], T.incl[ii]
mu,nu = np.array([m0,m0,m1,m1]), np.array([n0,n1,n1,n0])
def one_tile(tile, opt, savepickle, ps, tiles, tiledir, tempoutdir, T=None, hdr=None):
bands = opt.bands
outfn = opt.output % (tile.coadd_id)
savewise_outfn = opt.save_wise_output % (tile.coadd_id)
sband = 'r'
bandnum = 'ugriz'.index(sband)
tt0 = Time()
print
print 'Coadd tile', tile.coadd_id
thisdir = get_unwise_tile_dir(tiledir, tile.coadd_id)
fn = os.path.join(thisdir, 'unwise-%s-w%i-img-m.fits' % (tile.coadd_id, bands[0]))
if os.path.exists(fn):
print 'Reading', fn
wcs = Tan(fn)
else:
print 'File', fn, 'does not exist; faking WCS'
from unwise_coadd import get_coadd_tile_wcs
wcs = get_coadd_tile_wcs(tile.ra, tile.dec)
r0,r1,d0,d1 = wcs.radec_bounds()
print 'RA,Dec bounds:', r0,r1,d0,d1
H,W = wcs.get_height(), wcs.get_width()
if T is None:
T = merge_tables([fits_table(fn, columns=[x.upper() for x in [
'chi2_psf_r', 'chi2_model_r', 'mag_psf_r',
#'mag_disk_r',
'mag_spheroid_r', 'spheroid_reff_world',
'spheroid_aspect_world', 'spheroid_theta_world',
#'disk_scale_world', 'disk_aspect_world',
#'disk_theta_world',
'alphamodel_j2000', 'deltamodel_j2000']],
column_map=dict(CHI2_PSF_R='chi2_psf',
CHI2_MODEL_R='chi2_model',
MAG_PSF_R='mag_psf',
MAG_SPHEROID_R='mag_spheroid_r',
))
for fn in
['DES_SNX3cat_000001.fits', 'DES_SNX3cat_000002.fits']]
)
T.mag_disk = np.zeros(len(T), np.float32) + 99.
print 'Read total of', len(T), 'DES sources'
ok,T.x,T.y = wcs.radec2pixelxy(T.alphamodel_j2000, T.deltamodel_j2000)
margin = int(60. * wcs.pixel_scale())
print 'Margin:', margin, 'pixels'
T.cut((T.x > -margin) * (T.x < (W+margin)) *
(T.y > -margin) * (T.y < (H+margin)))
print 'Cut to', len(T), 'in bounds'
if opt.photoObjsOnly:
return
print len(T), 'objects'
if len(T) == 0:
return
defaultflux = 1.
# hack
T.x = (T.x - 1.).astype(np.float32)
T.y = (T.y - 1.).astype(np.float32)
margin = 20.
I = np.flatnonzero((T.x >= -margin) * (T.x < W+margin) *
(T.y >= -margin) * (T.y < H+margin))
T.cut(I)
print 'Cut to margins: N objects:', len(T)
if len(T) == 0:
return
wanyband = wband = 'w'
classmap = {}
print 'Creating tractor sources...'
cat = get_se_modelfit_cat(T, bands=[wanyband])
print 'Created', len(T), 'sources'
assert(len(cat) == len(T))
pixscale = wcs.pixel_scale()
# crude intrinsic source radii, in pixels
sourcerad = np.zeros(len(cat))
for i in range(len(cat)):
src = cat[i]
if isinstance(src, PointSource):
continue
elif isinstance(src, HoggGalaxy):
sourcerad[i] = (src.nre * src.shape.re / pixscale)
elif isinstance(src, FixedCompositeGalaxy):
sourcerad[i] = max(src.shapeExp.re * ExpGalaxy.nre,
src.shapeDev.re * DevGalaxy.nre) / pixscale
print 'sourcerad range:', min(sourcerad), max(sourcerad)
# Find WISE-only catalog sources
wfn = os.path.join(tempoutdir, 'wise-sources-%s.fits' % (tile.coadd_id))
WISE = read_wise_sources(wfn, r0,r1,d0,d1, allwise=True)
for band in bands:
mag = WISE.get('w%impro' % band)
nm = NanoMaggies.magToNanomaggies(mag)
WISE.set('w%inm' % band, nm)
print 'Band', band, 'max WISE catalog flux:', max(nm)
print ' (min mag:', mag.min(), ')'
unmatched = np.ones(len(WISE), bool)
I,J,d = match_radec(WISE.ra, WISE.dec, T.ra, T.dec, 4./3600.)
unmatched[I] = False
UW = WISE[unmatched]
print 'Got', len(UW), 'unmatched WISE sources'
if opt.savewise:
fitwiseflux = {}
for band in bands:
fitwiseflux[band] = np.zeros(len(UW))
# Record WISE fluxes for catalog matches.
# (this provides decent initialization for 'minsb' approx.)
wiseflux = {}
for band in bands:
wiseflux[band] = np.zeros(len(T))
if len(I) == 0:
continue
# X[I] += Y[J] with duplicate I doesn't work.
#wiseflux[band][J] += WISE.get('w%inm' % band)[I]
lhs = wiseflux[band]
rhs = WISE.get('w%inm' % band)[I]
print 'Band', band, 'max matched WISE flux:', max(rhs)
for j,f in zip(J, rhs):
lhs[j] += f
ok,UW.x,UW.y = wcs.radec2pixelxy(UW.ra, UW.dec)
UW.x -= 1.
UW.y -= 1.
T.coadd_id = np.array([tile.coadd_id] * len(T))
inbounds = np.flatnonzero((T.x >= -0.5) * (T.x < W-0.5) *
(T.y >= -0.5) * (T.y < H-0.5))
print 'Before looping over bands:', Time()-tt0
for band in bands:
tb0 = Time()
print
print 'Coadd tile', tile.coadd_id
print 'Band', band
wband = 'w%i' % band
imfn = os.path.join(thisdir, 'unwise-%s-w%i-img-m.fits' % (tile.coadd_id, band))
ivfn = os.path.join(thisdir, 'unwise-%s-w%i-invvar-m.fits.gz' % (tile.coadd_id, band))
ppfn = os.path.join(thisdir, 'unwise-%s-w%i-std-m.fits.gz' % (tile.coadd_id, band))
nifn = os.path.join(thisdir, 'unwise-%s-w%i-n-m.fits.gz' % (tile.coadd_id, band))
print 'Reading', imfn
wcs = Tan(imfn)
r0,r1,d0,d1 = wcs.radec_bounds()
print 'RA,Dec bounds:', r0,r1,d0,d1
ra,dec = wcs.radec_center()
print 'Center:', ra,dec
img = fitsio.read(imfn)
print 'Reading', ivfn
invvar = fitsio.read(ivfn)
print 'Reading', ppfn
pp = fitsio.read(ppfn)
print 'Reading', nifn
nims = fitsio.read(nifn)
print 'Median # ims:', np.median(nims)
good = (nims > 0)
invvar[np.logical_not(good)] = 0.
sig1 = 1./np.sqrt(np.median(invvar[good]))
minsig = getattr(opt, 'minsig%i' % band)
minsb = sig1 * minsig
print 'Sigma1:', sig1, 'minsig', minsig, 'minsb', minsb
# Load the average PSF model (generated by wise_psf.py)
print 'Reading PSF from', opt.psffn
P = fits_table(opt.psffn, hdu=band)
psf = GaussianMixturePSF(P.amp, P.mean, P.var)
# Render the PSF profile for figuring out source radii for
# approximation purposes.
R = 100
psf.radius = R
pat = psf.getPointSourcePatch(0., 0.)
assert(pat.x0 == pat.y0)
assert(pat.x0 == -R)
psfprofile = pat.patch[R, R:]
#print 'PSF profile:', psfprofile
# Reset default flux based on min radius
defaultflux = minsb / psfprofile[opt.minradius]
print 'Setting default flux', defaultflux
# Set WISE source radii based on flux
UW.rad = np.zeros(len(UW), int)
wnm = UW.get('w%inm' % band)
for r,pro in enumerate(psfprofile):
flux = minsb / pro
UW.rad[wnm > flux] = r
UW.rad = np.maximum(UW.rad + 1, 3)
# Set SDSS fluxes based on WISE catalog matches.
wf = wiseflux[band]
I = np.flatnonzero(wf > defaultflux)
wfi = wf[I]
print 'Initializing', len(I), 'fluxes based on catalog matches'
for i,flux in zip(I, wf[I]):
assert(np.isfinite(flux))
cat[i].getBrightness().setBand(wanyband, flux)
# Set SDSS radii based on WISE flux
rad = np.zeros(len(I), int)
for r,pro in enumerate(psfprofile):
flux = minsb / pro
rad[wfi > flux] = r
srad2 = np.zeros(len(cat), int)
srad2[I] = rad
del rad
# Set radii
for i in range(len(cat)):
src = cat[i]
# set fluxes
b = src.getBrightness()
if b.getBand(wanyband) <= defaultflux:
b.setBand(wanyband, defaultflux)
R = max([opt.minradius, sourcerad[i], srad2[i]])
# ?? This is used to select which sources are in-range
sourcerad[i] = R
if isinstance(src, PointSource):
src.fixedRadius = R
src.minradius = opt.minradius
elif (isinstance(src, HoggGalaxy) or
isinstance(src, FixedCompositeGalaxy)):
src.halfsize = R
# We used to dice the image into blocks/cells...
fullIV = np.zeros(len(cat))
fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp']
fitstats = dict([(k, np.zeros(len(cat))) for k in fskeys])
twcs = ConstantFitsWcs(wcs)
sky = 0.
tsky = ConstantSky(sky)
if ps:
tag = '%s W%i' % (tile.coadd_id, band)
plt.clf()
n,b,p = plt.hist(img.ravel(), bins=100,
range=(-10*sig1, 20*sig1), log=True,
histtype='step', color='b')
mx = max(n)
plt.ylim(0.1, mx)
plt.xlim(-10*sig1, 20*sig1)
plt.axvline(sky, color='r')
plt.title('%s: Pixel histogram' % tag)
ps.savefig()
if savepickle:
mods = []
cats = []
# SDSS and WISE source margins beyond the image margins ( + source radii )
smargin = 1
wmargin = 1
tim = Image(data=img, invvar=invvar, psf=psf, wcs=twcs,
sky=tsky, photocal=LinearPhotoCal(1., band=wanyband),
name='Coadd %s W%i' % (tile.coadd_id, band))
# Relevant SDSS sources:
m = smargin + sourcerad
I = np.flatnonzero(((T.x+m) >= -0.5) * ((T.x-m) < (W-0.5)) *
((T.y+m) >= -0.5) * ((T.y-m) < (H-0.5)))
inbox = ((T.x[I] >= -0.5) * (T.x[I] < (W-0.5)) *
(T.y[I] >= -0.5) * (T.y[I] < (H-0.5)))
# Inside this cell
srci = I[inbox]
# In the margin
margi = I[np.logical_not(inbox)]
# sources in the ROI box
subcat = [cat[i] for i in srci]
# include *copies* of sources in the margins
# (that way we automatically don't save the results)
subcat.extend([cat[i].copy() for i in margi])
assert(len(subcat) == len(I))
# add WISE-only sources in the expanded region
m = wmargin + UW.rad
J = np.flatnonzero(((UW.x+m) >= -0.5) * ((UW.x-m) < (W-0.5)) *
((UW.y+m) >= -0.5) * ((UW.y-m) < (H-0.5)))
if opt.savewise:
jinbox = ((UW.x[J] >= -0.5) * (UW.x[J] < (W-0.5)) *
(UW.y[J] >= -0.5) * (UW.y[J] < (H-0.5)))
uwcat = []
wnm = UW.get('w%inm' % band)
nomag = 0
for ji,j in enumerate(J):
if not np.isfinite(wnm[j]):
nomag += 1
continue
ptsrc = PointSource(RaDecPos(UW.ra[j], UW.dec[j]),
NanoMaggies(**{wanyband: wnm[j]}))
ptsrc.radius = UW.rad[j]
subcat.append(ptsrc)
if opt.savewise:
if jinbox[ji]:
uwcat.append((j, ptsrc))
print 'WISE-only:', nomag, 'of', len(J), 'had invalid mags'
print 'Sources:', len(srci), 'in the box,', len(I)-len(srci), 'in the margins, and', len(J), 'WISE-only'
print 'Creating a Tractor with image', tim.shape, 'and', len(subcat), 'sources'
tractor = Tractor([tim], subcat)
tractor.disable_cache()
print 'Running forced photometry...'
t0 = Time()
tractor.freezeParamsRecursive('*')
if opt.sky:
tractor.thawPathsTo('sky')
print 'Initial sky values:'
for tim in tractor.getImages():
print tim.getSky()
tractor.thawPathsTo(wanyband)
wantims = (savepickle or (ps is not None) or opt.save_fits)
R = tractor.optimize_forced_photometry(
minsb=minsb, mindlnp=1., sky=opt.sky, minFlux=None,
fitstats=True, fitstat_extras=[('pronexp', [nims])],
variance=True, shared_params=False,
use_ceres=opt.ceres, BW=opt.ceresblock, BH=opt.ceresblock,
wantims=wantims, negfluxval=0.1*sig1)
print 'That took', Time()-t0
if wantims:
ims0 = R.ims0
ims1 = R.ims1
IV,fs = R.IV, R.fitstats
if opt.sky:
print 'Fit sky values:'
for tim in tractor.getImages():
print tim.getSky()
if opt.savewise:
for (j,src) in uwcat:
fitwiseflux[band][j] = src.getBrightness().getBand(wanyband)
if opt.save_fits:
(dat,mod,ie,chi,roi) = ims1[0]
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' % tag, dat, clobber=True)
fitsio.write('%s-mod.fits' % tag, mod, clobber=True)
fitsio.write('%s-chi.fits' % tag, chi, clobber=True)
if ps:
tag = '%s W%i' % (tile.coadd_id, band)
(dat,mod,ie,chi,roi) = ims1[0]
plt.clf()
plt.imshow(dat, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3*sig1, vmax=10*sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
# plt.clf()
# plt.imshow(1./ie, interpolation='nearest', origin='lower',
# cmap='gray', vmin=0, vmax=10*sig1)
# plt.colorbar()
# plt.title('%s: sigma' % tag)
# ps.savefig()
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3*sig1, vmax=10*sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi, interpolation='nearest', origin='lower',
cmap='gray', vmin=-5, vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
# plt.clf()
# plt.imshow(np.round(chi), interpolation='nearest', origin='lower',
# cmap='jet', vmin=-5, vmax=+5)
# plt.colorbar()
# plt.title('Chi')
# ps.savefig()
plt.clf()
plt.imshow(chi, interpolation='nearest', origin='lower',
cmap='gray', vmin=-20, vmax=+20)
plt.colorbar()
plt.title('%s: chi 2' % tag)
ps.savefig()
plt.clf()
n,b,p = plt.hist(chi.ravel(), bins=100,
range=(-10, 10), log=True,
histtype='step', color='b')
mx = max(n)
plt.ylim(0.1, mx)
plt.axvline(0, color='r')
plt.title('%s: chi' % tag)
ps.savefig()
# fn = ps.basefn + '-chi.fits'
# fitsio.write(fn, chi, clobber=True)
# print 'Wrote', fn
if savepickle:
if ims1 is None:
mod = None
else:
im,mod,ie,chi,roi = ims1[0]
mods.append(mod)
cats.append((
srci, margi, UW.x[J], UW.y[J],
T.x[srci], T.y[srci], T.x[margi], T.y[margi],
[src.copy() for src in cat],
[src.copy() for src in subcat]))
if len(srci):
# Save fit stats
fullIV[srci] = IV[:len(srci)]
for k in fskeys:
x = getattr(fs, k)
fitstats[k][srci] = np.array(x)
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = fullIV
T.set(wband + '_nanomaggies', nm.astype(np.float32))
T.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32))
dnm = np.zeros(len(nm_ivar), np.float32)
okiv = (nm_ivar > 0)
dnm[okiv] = (1./np.sqrt(nm_ivar[okiv])).astype(np.float32)
okflux = (nm > 0)
mag = np.zeros(len(nm), np.float32)
mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])).astype(np.float32)
dmag = np.zeros(len(nm), np.float32)
ok = (okiv * okflux)
dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok])).astype(np.float32)
mag[np.logical_not(okflux)] = np.nan
dmag[np.logical_not(ok)] = np.nan
T.set(wband + '_mag', mag)
T.set(wband + '_mag_err', dmag)
for k in fskeys:
T.set(wband + '_' + k, fitstats[k].astype(np.float32))
if ps:
I,J,d = match_radec(WISE.ra, WISE.dec, T.ra, T.dec, 4./3600.)
plt.clf()
lo,cathi = 10,18
if band == 3:
lo,cathi = 8, 13
elif band == 4:
#lo,cathi = 4.5, 10.5
lo,cathi = 4.5, 12
loghist(WISE.get('w%impro'%band)[I], T.get(wband+'_mag')[J],
range=((lo,cathi),(lo,cathi)), bins=200)
plt.xlabel('WISE W%i mag' % band)
plt.ylabel('Tractor W%i mag' % band)
plt.title('WISE catalog vs Tractor forced photometry')
plt.axis([cathi,lo,cathi,lo])
ps.savefig()
print 'Tile', tile.coadd_id, 'band', wband, 'took', Time()-tb0
T.cut(inbounds)
T.delete_column('psfflux')
T.delete_column('cmodelflux')
T.delete_column('devflux')
T.delete_column('expflux')
T.treated_as_pointsource = T.treated_as_pointsource.astype(np.uint8)
T.pointsource = T.pointsource.astype(np.uint8)
T.writeto(outfn, header=hdr)
print 'Wrote', outfn
if savepickle:
fn = opt.output % (tile.coadd_id)
fn = fn.replace('.fits','.pickle')
pickle_to_file((mods, cats, T, sourcerad), fn)
print 'Pickled', fn
print 'Tile', tile.coadd_id, 'took', Time()-tt0
