def write_rot_plots(
indstart,
nproc,
out_basedir='/scratch1/scratchdirs/ameisner/unwise_psf_plots',
band=1):
atlas = fitsio.read('~/unwise/pro/allsky-atlas.fits')
ntile = len(atlas)
indend = min(indstart + nproc, ntile)
atlas = atlas[indstart:indend]
for i, row in enumerate(atlas):
coadd_id = str(row['coadd_id'])
print i, coadd_id
rot = get_unwise_psf(band, coadd_id)
plt.imshow(rot,
cmap='gray',
interpolation='nearest',
norm=LogNorm(vmin=0.1, vmax=1000000),
origin='lower')
plt.xticks([])
plt.yticks([])
plt.title('W' + str(band) + ', ' + coadd_id)
outdir = out_basedir + '/' + coadd_id[0:3]
outname = outdir + '/' + coadd_id + '.png'
if not os.path.exists(outdir): os.mkdir(outdir)
plt.savefig(outname, dpi=250, bbox_inches='tight')
plt.cla()
def all_coadd_id():
atlas = fitsio.read('~/unwise/pro/allsky-atlas.fits')
for i, row in enumerate(atlas):
print i
rot = get_unwise_psf(None, str(row['coadd_id']))
print rot.shape
def test_frames():
frames = fitsio.read(
'/global/projecta/projectdirs/cosmo/work/wise/outputs/merge/neo4/fulldepth/000/0000m016/unwise-0000m016-w1-frames.fits'
)
psf = unwise_psf.get_unwise_psf(1, '0000m016', frames=frames)
return psf
def test_tr_w2():
# http://legacysurvey.org/viewer?ra=163.9380&dec=33.5105&zoom=10&layer=unwise-neo3
frames = fitsio.read(
'/global/projecta/projectdirs/cosmo/work/wise/outputs/merge/neo4/e008/163/1637p333/unwise-1637p333-w2-frames.fits'
)
psf = unwise_psf.get_unwise_psf(2, '0000m016', frames=frames)
return psf
def test_poles():
# http://legacysurvey.org/viewer?ra=97.7563&dec=-66.8662&zoom=8&layer=unwise-neo3
# what coadd_id does this correspond to ?
frames = fitsio.read(
'/global/projecta/projectdirs/cosmo/work/wise/outputs/merge/neo4/fulldepth/096/0964m667/unwise-0964m667-w2-frames.fits'
)
psf = unwise_psf.get_unwise_psf(2, '0964m667', frames=frames)
return psf
def wise_psf(band, coadd_id):
# psf noise: ~roughly 0.1 count in outskirts of W1 and W2
if band >= 3:
raise ValueError('Need to stare at W3+ PSF more!')
psfnoise = 0.1
stamp = unwise_psf.get_unwise_psf(band, coadd_id)
edges = numpy.concatenate(
[stamp[0, 1:-1], stamp[-1, 1:-1], stamp[1:-1, 0], stamp[1:-1, -1]])
if band == 1:
medval = numpy.median(edges[edges != 0]) / 2
elif band == 2:
medval = numpy.median(edges[edges != 0]) / 4
else:
medval = 0.
stamp[stamp == 0] = medval
stamp -= medval
# stamp = numpy.clip(stamp, -numpy.median(edges[edges != 0]), numpy.inf)
# print(numpy.median(edges[edges != 0]))
# print(numpy.min(stamp))
from scipy import signal
stamp[stamp < 0] = 0.
# suppress spurious warnings in signal.wiener
olderr = numpy.seterr(invalid='ignore', divide='ignore')
stamp = signal.wiener(stamp, 11, psfnoise)
numpy.seterr(**olderr)
# taper linearly over outer 60 pixels?
stampszo2 = stamp.shape[0] // 2
xx, yy = numpy.mgrid[-stampszo2:stampszo2 + 1, -stampszo2:stampszo2 + 1]
edgedist = numpy.clip(stampszo2 - numpy.abs(xx), 0,
stampszo2 - numpy.abs(yy))
stamp = stamp * numpy.clip(edgedist / 60., stamp < 10, 1)
stamp = stamp / numpy.sum(stamp)
psf = psfmod.SimplePSF(stamp)
from functools import partial
psf.fitfun = partial(psfmod.wise_psf_fit, psfstamp=stamp)
return psf
def unwise_forcedphot(cat,
tiles,
bands=[1, 2, 3, 4],
roiradecbox=None,
unwise_dir='.',
use_ceres=True,
ceres_block=8,
save_fits=False,
get_models=False,
ps=None,
psf_broadening=None,
pixelized_psf=False):
'''
Given a list of tractor sources *cat*
and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
runs forced photometry, returning a FITS table the same length as *cat*.
'''
# # Severely limit sizes of models
for src in cat:
if isinstance(src, PointSource):
src.fixedRadius = 20
else:
src.halfsize = 20
wantims = ((ps is not None) or save_fits or get_models)
wanyband = 'w'
if get_models:
models = {}
fskeys = [
'prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp'
]
Nsrcs = len(cat)
phot = fits_table()
phot.tile = np.array([' '] * Nsrcs)
ra = np.array([src.getPosition().ra for src in cat])
dec = np.array([src.getPosition().dec for src in cat])
for band in bands:
print('Photometering WISE band', band)
wband = 'w%i' % band
# The tiles have some overlap, so for each source, keep the
# fit in the tile whose center is closest to the source.
tiledists = np.empty(Nsrcs)
tiledists[:] = 1e100
flux_invvars = np.zeros(Nsrcs, np.float32)
fitstats = dict([(k, np.zeros(Nsrcs, np.float32)) for k in fskeys])
nexp = np.zeros(Nsrcs, np.int16)
mjd = np.zeros(Nsrcs, np.float64)
for tile in tiles:
print('Reading tile', tile.coadd_id)
tim = get_unwise_tractor_image(unwise_dir,
tile.coadd_id,
band,
bandname=wanyband,
roiradecbox=roiradecbox)
if tim is None:
print('Actually, no overlap with tile', tile.coadd_id)
continue
if pixelized_psf:
import unwise_psf
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id)
print('PSF postage stamp', psfimg.shape, 'sum', psfimg.sum())
from tractor.psf import PixelizedPSF
psfimg /= psfimg.sum()
tim.psf = PixelizedPSF(psfimg)
print('### HACK ### normalized PSF to 1.0')
print('Set PSF to', tim.psf)
if False:
ph, pw = psfimg.shape
px, py = np.meshgrid(np.arange(ph), np.arange(pw))
cx = np.sum(psfimg * px)
cy = np.sum(psfimg * py)
print('PSF center of mass: %.2f, %.2f' % (cx, cy))
for sz in range(1, 11):
middle = pw // 2
sub = (slice(middle - sz, middle + sz + 1),
slice(middle - sz, middle + sz + 1))
cx = np.sum((psfimg * px)[sub]) / np.sum(psfimg[sub])
cy = np.sum((psfimg * py)[sub]) / np.sum(psfimg[sub])
print('Size', sz,
': PSF center of mass: %.2f, %.2f' % (cx, cy))
import fitsio
fitsio.write('psfimg-%s-w%i.fits' % (tile.coadd_id, band),
psfimg,
clobber=True)
if psf_broadening is not None and not pixelized_psf:
# psf_broadening is a factor by which the PSF FWHMs
# should be scaled; the PSF is a little wider
# post-reactivation.
psf = tim.getPsf()
from tractor import GaussianMixturePSF
if isinstance(psf, GaussianMixturePSF):
#
print('Broadening PSF: from', psf)
p0 = psf.getParams()
#print('Params:', p0)
pnames = psf.getParamNames()
#print('Param names:', pnames)
p1 = [
p * psf_broadening**2 if 'var' in name else p
for (p, name) in zip(p0, pnames)
]
#print('Broadened:', p1)
psf.setParams(p1)
print('Broadened PSF:', psf)
else:
print(
'WARNING: cannot apply psf_broadening to WISE PSF of type',
type(psf))
print('Read image with shape', tim.shape)
# Select sources in play.
wcs = tim.wcs.wcs
H, W = tim.shape
ok, x, y = wcs.radec2pixelxy(ra, dec)
x = (x - 1.).astype(np.float32)
y = (y - 1.).astype(np.float32)
margin = 10.
I = np.flatnonzero((x >= -margin) * (x < W + margin) *
(y >= -margin) * (y < H + margin))
print(len(I), 'within the image + margin')
inbox = ((x[I] >= -0.5) * (x[I] < (W - 0.5)) * (y[I] >= -0.5) *
(y[I] < (H - 0.5)))
print(sum(inbox), 'strictly within the image')
# Compute L_inf distance to (full) tile center.
tilewcs = unwise_tile_wcs(tile.ra, tile.dec)
cx, cy = tilewcs.crpix
ok, tx, ty = tilewcs.radec2pixelxy(ra[I], dec[I])
td = np.maximum(np.abs(tx - cx), np.abs(ty - cy))
closest = (td < tiledists[I])
tiledists[I[closest]] = td[closest]
keep = inbox * closest
# Source indices (in the full "cat") to keep (the fit values for)
srci = I[keep]
if not len(srci):
print('No sources to be kept; skipping.')
continue
phot.tile[srci] = tile.coadd_id
nexp[srci] = tim.nuims[
np.clip(np.round(y[srci]).astype(int), 0, H - 1),
np.clip(np.round(x[srci]).astype(int), 0, W - 1)]
# Source indices in the margins
margi = I[np.logical_not(keep)]
# sources in the box -- at the start of the subcat list.
subcat = [cat[i] for i in srci]
# include *copies* of sources in the margins
# (that way we automatically don't save the results)
subcat.extend([cat[i].copy() for i in margi])
assert (len(subcat) == len(I))
# FIXME -- set source radii, ...?
minsb = 0.
fitsky = False
# Look in image and set radius based on peak height??
tractor = Tractor([tim], subcat)
if use_ceres:
from tractor.ceres_optimizer import CeresOptimizer
tractor.optimizer = CeresOptimizer(BW=ceres_block,
BH=ceres_block)
tractor.freezeParamsRecursive('*')
tractor.thawPathsTo(wanyband)
kwa = dict(fitstat_extras=[('pronexp', [tim.nims])])
t0 = Time()
R = tractor.optimize_forced_photometry(minsb=minsb,
mindlnp=1.,
sky=fitsky,
fitstats=True,
variance=True,
shared_params=False,
wantims=wantims,
**kwa)
print('unWISE forced photometry took', Time() - t0)
if use_ceres:
term = R.ceres_status['termination']
print('Ceres termination status:', term)
# Running out of memory can cause failure to converge
# and term status = 2.
# Fail completely in this case.
if term != 0:
raise RuntimeError('Ceres terminated with status %i' %
term)
if wantims:
ims0 = R.ims0
ims1 = R.ims1
IV, fs = R.IV, R.fitstats
if save_fits:
import fitsio
(dat, mod, ie, chi, roi) = ims1[0]
wcshdr = fitsio.FITSHDR()
tim.wcs.wcs.add_to_header(wcshdr)
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' % tag,
dat,
clobber=True,
header=wcshdr)
fitsio.write('%s-mod.fits' % tag,
mod,
clobber=True,
header=wcshdr)
fitsio.write('%s-chi.fits' % tag,
chi,
clobber=True,
header=wcshdr)
if get_models:
(dat, mod, ie, chi, roi) = ims1[0]
models[(tile.coadd_id, band)] = (mod, tim.roi)
if ps:
tag = '%s W%i' % (tile.coadd_id, band)
(dat, mod, ie, chi, roi) = ims1[0]
sig1 = tim.sig1
plt.clf()
plt.imshow(dat,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=10 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
plt.clf()
plt.imshow(mod,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=10 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-5,
vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
# Save results for this tile.
# the "keep" sources are at the beginning of the "subcat" list
flux_invvars[srci] = IV[:len(srci)].astype(np.float32)
if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
mjd[srci] = (tim.mjdmin + tim.mjdmax) / 2.
if fs is None:
continue
for k in fskeys:
x = getattr(fs, k)
# fitstats are returned only for un-frozen sources
fitstats[k][srci] = np.array(x).astype(np.float32)[:len(srci)]
# Note, this is *outside* the loop over tiles.
# The fluxes are saved in the source objects, and will be set based on
# the 'tiledists' logic above.
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = flux_invvars
# Sources out of bounds, eg, never change from their default
# (1-sigma or whatever) initial fluxes. Zero them out instead.
nm[nm_ivar == 0] = 0.
phot.set(wband + '_nanomaggies', nm.astype(np.float32))
phot.set(wband + '_nanomaggies_ivar', nm_ivar)
dnm = np.zeros(len(nm_ivar), np.float32)
okiv = (nm_ivar > 0)
dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32)
okflux = (nm > 0)
mag = np.zeros(len(nm), np.float32)
mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])).astype(
np.float32)
dmag = np.zeros(len(nm), np.float32)
ok = (okiv * okflux)
dmag[ok] = (np.abs(
(-2.5 / np.log(10.)) * dnm[ok] / nm[ok])).astype(np.float32)
mag[np.logical_not(okflux)] = np.nan
dmag[np.logical_not(ok)] = np.nan
phot.set(wband + '_mag', mag)
phot.set(wband + '_mag_err', dmag)
for k in fskeys:
phot.set(wband + '_' + k, fitstats[k])
phot.set(wband + '_nexp', nexp)
if not np.all(mjd == 0):
phot.set(wband + '_mjd', mjd)
if get_models:
return phot, models
return phot
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None,
use_ceres=True, ceres_block=8,
save_fits=False, get_models=False, ps=None,
psf_broadening=None,
pixelized_psf=False,
get_masks=None,
move_crpix=False,
modelsky_dir=None):
'''
Given a list of tractor sources *cat*
and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
runs forced photometry, returning a FITS table the same length as *cat*.
*get_masks*: the WCS to resample mask bits into.
'''
from tractor import NanoMaggies, PointSource, Tractor, ExpGalaxy, DevGalaxy, FixedCompositeGalaxy
if not pixelized_psf and psf_broadening is None:
# PSF broadening in post-reactivation data, by band.
# Newer version from Aaron's email to decam-chatter, 2018-06-14.
broadening = { 1: 1.0405, 2: 1.0346, 3: None, 4: None }
psf_broadening = broadening[band]
if False:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('wise-forced-w%i' % band)
plots = (ps is not None)
if plots:
import pylab as plt
wantims = (plots or save_fits or get_models)
wanyband = 'w'
if get_models:
models = {}
wband = 'w%i' % band
fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix',
'pronexp']
Nsrcs = len(cat)
phot = fits_table()
# Filled in based on unique tile overlap
phot.wise_coadd_id = np.array([' '] * Nsrcs)
phot.set(wband + '_psfdepth', np.zeros(len(phot), np.float32))
ra = np.array([src.getPosition().ra for src in cat])
dec = np.array([src.getPosition().dec for src in cat])
nexp = np.zeros(Nsrcs, np.int16)
mjd = np.zeros(Nsrcs, np.float64)
central_flux = np.zeros(Nsrcs, np.float32)
fitstats = {}
tims = []
if get_masks:
mh,mw = get_masks.shape
maskmap = np.zeros((mh,mw), np.uint32)
for tile in tiles:
print('Reading WISE tile', tile.coadd_id, 'band', band)
tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band,
bandname=wanyband, roiradecbox=roiradecbox)
if tim is None:
print('Actually, no overlap with tile', tile.coadd_id)
continue
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data' % tag)
ps.savefig()
plt.clf()
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
range=(-5,10), bins=100)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.title(tag)
ps.savefig()
if move_crpix and band in [1, 2]:
realwcs = tim.wcs.wcs
x,y = realwcs.crpix
tile_crpix = tile.get('crpix_w%i' % band)
dx = tile_crpix[0] - 1024.5
dy = tile_crpix[1] - 1024.5
realwcs.set_crpix(x+dx, y+dy)
#print('CRPIX', x,y, 'shift by', dx,dy, 'to', realwcs.crpix)
if modelsky_dir and band in [1, 2]:
fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band))
if not os.path.exists(fn):
raise RuntimeError('WARNING: does not exist:', fn)
x0,x1,y0,y1 = tim.roi
bg = fitsio.FITS(fn)[2][y0:y1, x0:x1]
#print('Read background map:', bg.shape, bg.dtype, 'vs image', tim.shape)
if plots:
plt.clf()
plt.subplot(1,2,1)
plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=5 * sig1)
plt.subplot(1,2,2)
plt.imshow(bg, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=5 * sig1)
tag = '%s W%i' % (tile.coadd_id, band)
plt.suptitle(tag)
ps.savefig()
plt.clf()
ha = dict(range=(-5,10), bins=100, histtype='step')
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
color='b', label='Original', **ha)
plt.hist(((tim.getImage()-bg) * tim.inverr)[tim.inverr > 0].ravel(),
color='g', label='Minus Background', **ha)
plt.axvline(0, color='k', alpha=0.5)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.legend()
plt.title(tag + ': background')
ps.savefig()
# Actually subtract the background!
tim.data -= bg
# Floor the per-pixel variances
if band in [1,2]:
# in Vega nanomaggies per pixel
floor_sigma = {1: 0.5, 2: 2.0}
with np.errstate(divide='ignore'):
new_ie = 1. / np.hypot(1./tim.inverr, floor_sigma[band])
new_ie[tim.inverr == 0] = 0.
if plots:
plt.clf()
plt.plot((1. / tim.inverr[tim.inverr>0]).ravel(), (1./new_ie[tim.inverr>0]).ravel(), 'b.')
plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band))
plt.xlabel('original')
plt.ylabel('floored')
ps.savefig()
tim.inverr = new_ie
# Read mask file?
if get_masks:
from astrometry.util.resample import resample_with_wcs, OverlapError
# unwise_dir can be a colon-separated list of paths
tilemask = None
for d in tile.unwise_dir.split(':'):
fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id,
'unwise-%s-msk.fits.gz' % tile.coadd_id)
if os.path.exists(fn):
print('Reading unWISE mask file', fn)
x0,x1,y0,y1 = tim.roi
tilemask = fitsio.FITS(fn)[0][y0:y1,x0:x1]
break
if tilemask is None:
print('unWISE mask file for tile', tile.coadd_id, 'does not exist')
else:
try:
tanwcs = tim.wcs.wcs
assert(tanwcs.shape == tilemask.shape)
Yo,Xo,Yi,Xi,_ = resample_with_wcs(get_masks, tanwcs, intType=np.int16)
# Only deal with mask pixels that are set.
I, = np.nonzero(tilemask[Yi,Xi] > 0)
# Trim to unique area for this tile
rr,dd = get_masks.pixelxy2radec(Yo[I]+1, Xo[I]+1)
good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
I = I[good]
maskmap[Yo[I],Xo[I]] = tilemask[Yi[I], Xi[I]]
except OverlapError:
# Shouldn't happen by this point
print('No overlap between WISE tile', tile.coadd_id, 'and brick')
# The tiles have some overlap, so zero out pixels outside the
# tile's unique area.
th,tw = tim.shape
xx,yy = np.meshgrid(np.arange(tw), np.arange(th))
rr,dd = tim.wcs.wcs.pixelxy2radec(xx+1, yy+1)
unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
#print(np.sum(unique), 'of', (th*tw), 'pixels in this tile are unique')
tim.inverr[unique == False] = 0.
del xx,yy,rr,dd,unique
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage() * (tim.inverr > 0),
interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data (unique)' % tag)
ps.savefig()
if pixelized_psf:
import unwise_psf
if (band == 1) or (band == 2):
# we only have updated PSFs for W1 and W2
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id,
modelname='neo4_unwisecat')
else:
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id)
if band == 4:
# oversample (the unwise_psf models are at native W4 5.5"/pix,
# while the unWISE coadds are made at 2.75"/pix.
ph,pw = psfimg.shape
subpsf = np.zeros((ph*2-1, pw*2-1), np.float32)
from astrometry.util.util import lanczos3_interpolate
xx,yy = np.meshgrid(np.arange(0., pw-0.51, 0.5, dtype=np.float32),
np.arange(0., ph-0.51, 0.5, dtype=np.float32))
xx = xx.ravel()
yy = yy.ravel()
ix = xx.astype(np.int32)
iy = yy.astype(np.int32)
dx = (xx - ix).astype(np.float32)
dy = (yy - iy).astype(np.float32)
psfimg = psfimg.astype(np.float32)
rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg])
if plots:
plt.clf()
plt.imshow(psfimg, interpolation='nearest', origin='lower')
plt.title('Original PSF model')
ps.savefig()
plt.clf()
plt.imshow(subpsf, interpolation='nearest', origin='lower')
plt.title('Subsampled PSF model')
ps.savefig()
psfimg = subpsf
del xx, yy, ix, iy, dx, dy
from tractor.psf import PixelizedPSF
psfimg /= psfimg.sum()
fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0}
psfimg *= fluxrescales[band]
tim.psf = PixelizedPSF(psfimg)
if psf_broadening is not None and not pixelized_psf:
# psf_broadening is a factor by which the PSF FWHMs
# should be scaled; the PSF is a little wider
# post-reactivation.
psf = tim.getPsf()
from tractor import GaussianMixturePSF
if isinstance(psf, GaussianMixturePSF):
#
print('Broadening PSF: from', psf)
p0 = psf.getParams()
pnames = psf.getParamNames()
p1 = [p * psf_broadening**2 if 'var' in name else p
for (p, name) in zip(p0, pnames)]
psf.setParams(p1)
print('Broadened PSF:', psf)
else:
print('WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf))
wcs = tim.wcs.wcs
ok,x,y = wcs.radec2pixelxy(ra, dec)
x = np.round(x - 1.).astype(int)
y = np.round(y - 1.).astype(int)
good = (x >= 0) * (x < tw) * (y >= 0) * (y < th)
# Which sources are in this brick's unique area?
usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
I, = np.nonzero(good * usrc)
nexp[I] = tim.nuims[y[I], x[I]]
if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
mjd[I] = (tim.mjdmin + tim.mjdmax) / 2.
phot.wise_coadd_id[I] = tile.coadd_id
central_flux[I] = tim.getImage()[y[I], x[I]]
del x,y,good,usrc
# PSF norm for depth
psf = tim.getPsf()
h,w = tim.shape
patch = psf.getPointSourcePatch(h//2, w//2).patch
psfnorm = np.sqrt(np.sum(patch**2))
# To handle zero-depth, we return 1/nanomaggies^2 units rather than mags.
psfdepth = 1. / (tim.sig1 / psfnorm)**2
phot.get(wband + '_psfdepth')[I] = psfdepth
tim.tile = tile
tims.append(tim)
if plots:
plt.clf()
mn,mx = 0.1, 20000
plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100,
range=(np.log10(mn), np.log10(mx)))
logt = np.arange(0, 5)
plt.xticks(logt, ['%i' % i for i in 10.**logt])
plt.title('Central fluxes (W%i)' % band)
plt.axvline(np.log10(20000), color='k')
plt.axvline(np.log10(1000), color='k')
ps.savefig()
# Eddie's non-secret recipe:
#- central pixel <= 1000: 19x19 pix box size
#- central pixel in 1000 - 20000: 59x59 box size
#- central pixel > 20000 or saturated: 149x149 box size
#- object near "bright star": 299x299 box size
nbig = nmedium = nsmall = 0
for src,cflux in zip(cat, central_flux):
if cflux > 20000:
R = 100
nbig += 1
elif cflux > 1000:
R = 30
nmedium += 1
else:
R = 15
nsmall += 1
if isinstance(src, PointSource):
src.fixedRadius = R
else:
### FIXME -- sizes for galaxies..... can we set PSF size separately?
galrad = 0
# RexGalaxy is a subclass of ExpGalaxy
if isinstance(src, (ExpGalaxy, DevGalaxy)):
galrad = src.shape.re
elif isinstance(src, FixedCompositeGalaxy):
galrad = max(src.shapeExp.re, src.shapeDev.re)
pixscale = 2.75
src.halfsize = int(np.hypot(R, galrad * 5 / pixscale))
#print('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small')
minsb = 0.
fitsky = False
tractor = Tractor(tims, cat)
if use_ceres:
from tractor.ceres_optimizer import CeresOptimizer
tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block)
tractor.freezeParamsRecursive('*')
tractor.thawPathsTo(wanyband)
kwa = dict(fitstat_extras=[('pronexp', [tim.nims for tim in tims])])
t0 = Time()
R = tractor.optimize_forced_photometry(
minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True,
variance=True, shared_params=False,
wantims=wantims, **kwa)
print('unWISE forced photometry took', Time() - t0)
if use_ceres:
term = R.ceres_status['termination']
# Running out of memory can cause failure to converge
# and term status = 2.
# Fail completely in this case.
if term != 0:
print('Ceres termination status:', term)
raise RuntimeError(
'Ceres terminated with status %i' % term)
if wantims:
ims1 = R.ims1
flux_invvars = R.IV
if R.fitstats is not None:
for k in fskeys:
x = getattr(R.fitstats, k)
fitstats[k] = np.array(x).astype(np.float32)
if save_fits:
for i,tim in enumerate(tims):
tile = tim.tile
(dat, mod, ie, chi, roi) = ims1[i]
wcshdr = fitsio.FITSHDR()
tim.wcs.wcs.add_to_header(wcshdr)
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' %
tag, dat, clobber=True, header=wcshdr)
fitsio.write('%s-mod.fits' % tag, mod,
clobber=True, header=wcshdr)
fitsio.write('%s-chi.fits' % tag, chi,
clobber=True, header=wcshdr)
if plots:
# Create models for just the brightest sources
bright_cat = [src for src in cat
if src.getBrightness().getBand(wanyband) > 1000]
print('Bright soures:', len(bright_cat))
btr = Tractor(tims, bright_cat)
for tim in tims:
mod = btr.getModelImage(tim)
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
sig1 = tim.sig1
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: bright-star models' % tag)
ps.savefig()
if get_models:
for i,tim in enumerate(tims):
tile = tim.tile
(dat, mod, ie, chi, roi) = ims1[i]
models[(tile.coadd_id, band)] = (mod, dat, ie, tim.roi, tim.wcs.wcs)
if plots:
for i,tim in enumerate(tims):
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
(dat, mod, ie, chi, roi) = ims1[i]
sig1 = tim.sig1
plt.clf()
plt.imshow(dat, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi, interpolation='nearest', origin='lower',
cmap='gray', vmin=-5, vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = flux_invvars
# Sources out of bounds, eg, never change from their default
# (1-sigma or whatever) initial fluxes. Zero them out instead.
nm[nm_ivar == 0] = 0.
phot.set(wband + '_nanomaggies', nm.astype(np.float32))
phot.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32))
dnm = np.zeros(len(nm_ivar), np.float32)
okiv = (nm_ivar > 0)
dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32)
okflux = (nm > 0)
mag = np.zeros(len(nm), np.float32)
mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])
).astype(np.float32)
dmag = np.zeros(len(nm), np.float32)
ok = (okiv * okflux)
dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok])
).astype(np.float32)
mag[np.logical_not(okflux)] = np.nan
dmag[np.logical_not(ok)] = np.nan
phot.set(wband + '_mag', mag)
phot.set(wband + '_mag_err', dmag)
for k in fskeys:
phot.set(wband + '_' + k, fitstats[k])
phot.set(wband + '_nexp', nexp)
if not np.all(mjd == 0):
phot.set(wband + '_mjd', mjd)
rtn = wphotduck()
rtn.phot = phot
rtn.models = None
rtn.maskmap = None
if get_models:
rtn.models = models
if get_masks:
rtn.maskmap = maskmap
return rtn
def work_(coadd_id, coadds, out):
"""
coadd_id: ID (name/tile name) of the coadd, such as 1194p212
coadds: Pandas DataFrame describing coadd metadata, such as epochs, bands, dates
out: directory where to write results
"""
coadds = coadds.copy()
w2_meta = coadds.loc[(coadd_id, slice(None), 2), :]
# Plan synthetic planets
# Pick positions and fluxes
# TODO: Need to re-add epochal offsets from orbit simulation
mov_x = []
mov_y = []
mov_flx = []
mjdstart = None
for i, meta in w2_meta.iterrows():
if mjdstart is None:
mjddiff = 0
else:
mjddiff = meta["MJDMEAN"] - mjdstart
xs = []
ys = []
flxs = []
random.seed(0)
for j in xrange(10): # Number of synths to create
# Pick pixel positions x, y, and pick flux
px = random.uniform(0 + 10, 2048 - 10)
py = random.uniform(0 + 10, 2048 - 10)
flx = (random.uniform(50, 400) / 6.)
# TODO: re-add orbit/parallax motion
xs.append(px)
ys.append(py)
flxs.append(flx)
mov_x.append(xs)
mov_y.append(ys)
mov_flx.append(flxs)
# Store x,y,flx for use when coadding
coadds.loc[w2_meta.index, "SYNTH_X"] = pd.Series(mov_x, w2_meta.index)
coadds.loc[w2_meta.index, "SYNTH_Y"] = pd.Series(mov_y, w2_meta.index)
coadds.loc[w2_meta.index, "SYNTH_FLUX"] = pd.Series(mov_flx, w2_meta.index)
# Subset metadata for this coadd, epoch 0 only
w1_meta = coadds.loc[(coadd_id, 0, 1), :]
w2_meta = coadds.loc[(coadd_id, 0, 2), :]
# Download coadds and add synths
# (only doing W2, but using same func to grab w1 coadd)
w1_hdr, w1_im = add_synths(w1_meta)
w2_hdr, w2_im = add_synths(w2_meta)
# Convert to fits format
w1_fits = make_fits_image(w1_hdr, w1_im)
w2_fits = make_fits_image(w2_hdr, w2_im)
# Write out resulting files
# (only doing W2)
open("%s/%s_w2_synths.fits" % (out, coadd_id), "wb").write(w2_fits)
# Extract sources & check missed fakes
def check_f_b(meta, srcs):
xs = meta["SYNTH_X"][:]
ys = meta["SYNTH_Y"][:]
flxs = meta["SYNTH_FLUX"][:]
sources = []
miss_xs = []
miss_ys = []
miss_flxs = []
for i in xrange(len(xs)):
x, y, flx = xs[i], ys[i], flxs[i]
# Find detection within 3 pixel width square centered on synth
tmp = srcs[((abs(srcs["XWIN_IMAGE"] - (x + 1)) < 1) &
(abs(srcs["YWIN_IMAGE"] - (y + 1)) < 1))].to_pandas()
if tmp.shape[0] == 0:
miss_xs.append(x)
miss_ys.append(y)
miss_flxs.append(flx)
continue
tmp = tmp.iloc[[0]].copy()
tmp["SYNTH_X"] = x
tmp["SYNTH_Y"] = y
tmp["SYNTH_FLUX"] = flx
sources.append(tmp)
tps = None
if len(sources) > 0:
tps = pd.concat(sources, ignore_index=True)
return tps, pd.DataFrame({
"SYNTH_X": miss_xs,
"SYNTH_Y": miss_ys,
"SYNTH_FLUX": miss_flxs,
})
# Get PSFs from unwise_psf
# TODO: Still need to fiure out whether to pass W2 PSF
# for both W1 and W2
w1_psf = unwise_psf.get_unwise_psf(1, coadd_id)
w2_psf = unwise_psf.get_unwise_psf(2, coadd_id)
# Try crowdsource
# First, make a PSF
#w2_psf_vp = cs_psf.VariablePixelizedPSF(cs_psf.central_stamp(w2_psf),normalize=-1)
# TODO: need to decide whether this is the right mechanism for creating a PSF, and
# whether to twiddle any knobs (normalize?)
w2_psf_s = cs_psf.SimplePSF(w2_psf)
# Then, call fit_im with the image and PSF
# Need to pass in a weight or crowdsource will crash.
# Should recommend considering a weight default = 1
coadd_id, epoch, band = w2_meta.name
invvar = aif.open(get_invvar(coadd_id, epoch, band))[0].data
# Also edited code to ignore a None "dq" when writing the flags column
# need to figure out what that's all about
# These results (x, y, flux, model, psf) have changed since the crowdsource
# code that described fit_im's usage was written. Now, "x" contains the
# source table, along with x, y, and so on
# TODO: Figure out correct weight to use. coverage maps?
x, y, flux, model, psf = crowdsource.fit_im(w2_im, w2_psf_s, weight=invvar)
# Convert that source table to a dataframe for easy csv-ing
df = pd.DataFrame(x)
# Write to disk
df.to_csv("%s/%s_cs_sources.csv" % (out, coadd_id), index=False)
# Convert PSFs to fits for sextractor
# Following sewpy's code, but there's some parameters we've left empty. Those may
# be important.
# TODO: what should they be?
w1_psf_fits = make_fits_psf(w1_psf)
w2_psf_fits = make_fits_psf(w2_psf)
# Run sextractor
sources = run_extract_sources(w1_fits,
w2_fits,
w1_psf=w1_psf_fits,
w2_psf=w2_psf_fits)
# Identify detected vs undetected synths from sextractor results
# TODO: do this for crowdsource, too.
tps, fns = check_f_b(w2_meta, sources)
if tps is not None:
tps.to_csv("%s/%s_se_tps.csv" % (out, coadd_id), index=False)
fns.to_csv("%s/%s_se_fns.csv" % (out, coadd_id), index=False)
sources.write("%s/%s_se_sources" % (out, coadd_id),
overwrite=True,
format="ascii")