def run_forced_phot(cat,
tim,
ceres=True,
derivs=False,
agn=False,
do_forced=True,
do_apphot=True,
get_model=False,
ps=None,
timing=False,
fixed_also=False,
ceres_threads=1):
'''
fixed_also: if derivs=True, also run without derivatives and report
that flux too?
'''
if timing:
tlast = Time()
if ps is not None:
import pylab as plt
opti = None
forced_kwargs = {}
if ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
try:
opti = CeresOptimizer(BW=B, BH=B, threads=ceres_threads)
except:
if ceres_threads > 1:
raise RuntimeError(
'ceres_threads requested but not supported by tractor.ceres version'
)
opti = CeresOptimizer(BW=B, BH=B)
#forced_kwargs.update(verbose=True)
# nsize = 0
for src in cat:
# Limit sizes of huge models
# from tractor.galaxy import ProfileGalaxy
# if isinstance(src, ProfileGalaxy):
# px,py = tim.wcs.positionToPixel(src.getPosition())
# h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval)
# MAXHALF = 128
# if h > MAXHALF:
# #print('halfsize', h,'for',src,'-> setting to',MAXHALF)
# nsize += 1
# src.halfsize = MAXHALF
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
#print('Limited the size of', nsize, 'large galaxy models')
if derivs:
realsrcs = []
derivsrcs = []
Iderivs = []
for i, src in enumerate(cat):
from tractor import PointSource
realsrcs.append(src)
if not isinstance(src, PointSource):
continue
Iderivs.append(i)
brightness_dra = src.getBrightness().copy()
brightness_ddec = src.getBrightness().copy()
brightness_dra.setParams(np.zeros(brightness_dra.numberOfParams()))
brightness_ddec.setParams(
np.zeros(brightness_ddec.numberOfParams()))
brightness_dra.freezeAllBut(tim.band)
brightness_ddec.freezeAllBut(tim.band)
dsrc = SourceDerivatives(src, [brightness_dra, brightness_ddec],
tim, ps)
derivsrcs.append(dsrc)
Iderivs = np.array(Iderivs)
if fixed_also:
pass
else:
# For convenience, put all the real sources at the front of
# the list, so we can pull the IVs off the front of the list.
cat = realsrcs + derivsrcs
if agn:
from tractor.galaxy import ExpGalaxy, DevGalaxy, FixedCompositeGalaxy
from tractor import PointSource
from legacypipe.survey import SimpleGalaxy, RexGalaxy
realsrcs = []
agnsrcs = []
iagn = []
for i, src in enumerate(cat):
realsrcs.append(src)
## ??
if isinstance(src, (SimpleGalaxy, RexGalaxy)):
#print('Skipping SIMP or REX:', src)
continue
if isinstance(src, (ExpGalaxy, DevGalaxy, FixedCompositeGalaxy)):
iagn.append(i)
bright = src.getBrightness().copy()
bright.setParams(np.zeros(bright.numberOfParams()))
bright.freezeAllBut(tim.band)
agn = PointSource(src.pos, bright)
agn.freezeAllBut('brightness')
#print('Adding "agn"', agn, 'to', src)
#print('agn params:', agn.getParamNames())
agnsrcs.append(src)
iagn = np.array(iagn)
cat = realsrcs + agnsrcs
print('Added AGN to', len(iagn), 'galaxies')
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
disable_galaxy_cache()
F = fits_table()
if do_forced:
if timing and (derivs or agn):
t = Time()
print('Setting up:', t - tlast)
tlast = t
if derivs:
if fixed_also:
print('Forced photom with fixed positions:')
R = tr.optimize_forced_photometry(variance=True,
fitstats=False,
shared_params=False,
priors=False,
**forced_kwargs)
F.flux_fixed = np.array([
src.getBrightness().getFlux(tim.band) for src in cat
]).astype(np.float32)
N = len(cat)
F.flux_fixed_ivar = R.IV[:N].astype(np.float32)
if timing:
t = Time()
print('Forced photom with fixed positions finished:',
t - tlast)
tlast = t
cat = realsrcs + derivsrcs
tr.setCatalog(Catalog(*cat))
print('Forced photom with position derivatives:')
if ps is None and not get_model:
forced_kwargs.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True,
fitstats=True,
shared_params=False,
priors=False,
**forced_kwargs)
if ps is not None or get_model:
(data, mod, ie, chi, roi) = R.ims1[0]
if ps is not None:
ima = dict(vmin=-2. * tim.sig1,
vmax=5. * tim.sig1,
interpolation='nearest',
origin='lower',
cmap='gray')
imchi = dict(interpolation='nearest',
origin='lower',
vmin=-5,
vmax=5,
cmap='RdBu')
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
if derivs:
trx = Tractor([tim], realsrcs)
trx.freezeParam('images')
modx = trx.getModelImage(0)
chix = (data - modx) * tim.getInvError()
plt.clf()
plt.imshow(modx, **ima)
plt.title('Model without derivatives: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chix, **imchi)
plt.title('Chi without derivatives: %s' % tim.name)
ps.savefig()
if derivs or agn:
cat = realsrcs
N = len(cat)
F.flux = np.array([
src.getBrightness().getFlux(tim.band) for src in cat
]).astype(np.float32)
F.flux_ivar = R.IV[:N].astype(np.float32)
F.fracflux = R.fitstats.profracflux[:N].astype(np.float32)
F.rchisq = R.fitstats.prochi2[:N].astype(np.float32)
try:
F.fracmasked = R.fitstats.promasked[:N].astype(np.float32)
except:
print(
'No "fracmasked" available (only in recent Tractor versions)')
if derivs:
F.flux_dra = np.zeros(len(F), np.float32)
F.flux_ddec = np.zeros(len(F), np.float32)
F.flux_dra[Iderivs] = np.array(
[src.getParams()[0] for src in derivsrcs]).astype(np.float32)
F.flux_ddec[Iderivs] = np.array(
[src.getParams()[1] for src in derivsrcs]).astype(np.float32)
F.flux_dra_ivar = np.zeros(len(F), np.float32)
F.flux_ddec_ivar = np.zeros(len(F), np.float32)
F.flux_dra_ivar[Iderivs] = R.IV[N::2].astype(np.float32)
F.flux_ddec_ivar[Iderivs] = R.IV[N + 1::2].astype(np.float32)
if agn:
F.flux_agn = np.zeros(len(F), np.float32)
F.flux_agn_ivar = np.zeros(len(F), np.float32)
F.flux_agn[iagn] = np.array(
[src.getParams()[0] for src in agnsrcs])
F.flux_agn_ivar[iagn] = R.IV[N:].astype(np.float32)
if timing:
t = Time()
print('Forced photom:', t - tlast)
tlast = t
if do_apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack(
[tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
#print('Apertures:', apertures, 'pixels')
#print('apxy shape', apxy.shape) # --> (2,N)
# The aperture photometry routine doesn't like pixel positions outside the image
H, W = img.shape
Iap = np.flatnonzero((apxy[0, :] >= 0) * (apxy[1, :] >= 0) *
(apxy[0, :] <= W - 1) * (apxy[1, :] <= H - 1))
print('Aperture photometry for', len(Iap), 'of', len(apxy[0, :]),
'sources within image bounds')
for rad in apertures:
aper = photutils.CircularAperture(apxy[:, Iap], rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = np.zeros((len(F), len(apertures)), np.float32)
F.apflux[Iap, :] = ap.astype(np.float32)
apimgerr = np.vstack(apimgerr).T
apiv = np.zeros(apimgerr.shape, np.float32)
apiv[apimgerr != 0] = 1. / apimgerr[apimgerr != 0]**2
F.apflux_ivar = np.zeros((len(F), len(apertures)), np.float32)
F.apflux_ivar[Iap, :] = apiv
if timing:
print('Aperture photom:', Time() - tlast)
if get_model:
return F, mod
return F
mods2 = []
resids2 = []
chis2 = []
stars.xfit = stars.xx.copy()
stars.yfit = stars.yy.copy()
alphas = [0.1, 0.3, 1.0]
for i,tr in enumerate(tractors):
print tr
src = tr.catalog[0]
print 'Initial position:', src.pos
x,y = src.pos.x, src.pos.y
tr.freezeParam('images')
tr.printThawedParams()
for step in range(50):
dlnp,X,alpha = tr.optimize(priors=False, shared_params=False,
alphas=alphas)
print 'dlnp', dlnp
print 'pos', src.pos.x, src.pos.y
print 'Delta position:', src.pos.x - x, src.pos.y - y
#if dlnp < 0.1:
if dlnp == 0.:
break
print 'Final position:', src.pos
pos = src.getPosition()
stars.xfit[i] = stars.x0[i] + pos.x
stars.yfit[i] = stars.y0[i] + pos.y
def psf_residuals(expnum,
ccdname,
stampsize=35,
nstar=30,
magrange=(13, 17),
verbose=0,
splinesky=False):
# Set the debugging level.
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
pngprefix = 'qapsf-{}-{}'.format(expnum, ccdname)
# Gather all the info we need about this CCD.
decals = Decals()
ccd = decals.find_ccds(expnum=expnum, ccdname=ccdname)[0]
band = ccd.filter
ps1band = dict(g=0, r=1, i=2, z=3, Y=4)
print('Band {}'.format(band))
#scales = dict(g=0.0066, r=0.01, z=0.025)
#vmin, vmax = np.arcsinh(-1), np.arcsinh(100)
#print(scales[band])
im = decals.get_image_object(ccd)
iminfo = im.get_image_info()
H, W = iminfo['dims']
wcs = im.get_wcs()
# Choose a uniformly selected subset of PS1 stars on this CCD.
ps1 = ps1cat(ccdwcs=wcs)
cat = ps1.get_stars(band=band, magrange=magrange)
rand = np.random.RandomState(seed=expnum * ccd.ccdnum)
these = rand.choice(len(cat) - 1, nstar, replace=False)
#these = rand.random_integers(0,len(cat)-1,nstar)
cat = cat[these]
cat = cat[np.argsort(cat.median[:, ps1band[band]])] # sort by magnitude
#print(cat.nmag_ok)
get_tim_kwargs = dict(const2psf=True, splinesky=splinesky)
# Make a QAplot of the positions of all the stars.
tim = im.get_tractor_image(**get_tim_kwargs)
img = tim.getImage()
#img = tim.getImage()/scales[band]
fig = plt.figure(figsize=(5, 10))
ax = fig.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
#ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest',
# origin='lower',vmin=vmax,vmax=vmax)
ax.imshow(img, **tim.ima)
ax.axis('off')
ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.format(
band, expnum, ccdname, ccd.airmass, ccd.seeing))
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ax.text(xpos,
ypos,
'{:2d}'.format(istar + 1),
color='red',
horizontalalignment='left')
circ = plt.Circle((xpos, ypos), radius=30, color='g', fill=False, lw=1)
ax.add_patch(circ)
#radec = wcs.radec_bounds()
#ax.scatter(cat.ra,cat.dec)
#ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998])
#ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015])
#ax.set_xlabel('$RA\ (deg)$',fontsize=18)
#ax.set_ylabel('$Dec\ (deg)$',fontsize=18)
fig.savefig(pngprefix + '-ccd.png', bbox_inches='tight')
# Initialize the many-stamp QAplot
ncols = 3
nrows = np.ceil(nstar / ncols).astype('int')
inchperstamp = 2.0
fig = plt.figure(figsize=(inchperstamp * 3 * ncols, inchperstamp * nrows))
irow = 0
icol = 0
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
mag = ps1star.median[ps1band[band]] # r-band
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ix, iy = int(xpos), int(ypos)
# create a little tractor Image object around the star
slc = (slice(max(iy - stampsize, 0), min(iy + stampsize + 1, H)),
slice(max(ix - stampsize, 0), min(ix + stampsize + 1, W)))
# The PSF model 'const2Psf' is the one used in DR1: a 2-component
# Gaussian fit to PsfEx instantiated in the image center.
tim = im.get_tractor_image(slc=slc, **get_tim_kwargs)
stamp = tim.getImage()
ivarstamp = tim.getInvvar()
# Initialize a tractor PointSource from PS1 measurements
flux = NanoMaggies.magToNanomaggies(mag)
star = PointSource(RaDecPos(ra, dec), NanoMaggies(**{band: flux}))
# Fit just the source RA,Dec,flux.
tractor = Tractor([tim], [star])
tractor.freezeParam('images')
print('2-component MOG:', tim.psf)
tractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_mog = tractor.getModelImage(0)
chi2_mog = -2.0 * tractor.getLogLikelihood()
mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
# Now change the PSF model to a pixelized PSF model from PsfEx instantiated
# at this place in the image.
psf = PixelizedPsfEx(im.psffn)
tim.psf = psf.constantPsfAt(xpos, ypos)
#print('PSF model:', tim.psf)
#tractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_psfex = tractor.getModelImage(0)
chi2_psfex = -2.0 * tractor.getLogLikelihood()
mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
#mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95])
sig = np.std((stamp - model_psfex)[ivarstamp > 0])
mn, mx = [-2.0 * sig, 5 * sig]
# Generate a QAplot.
if (istar > 0) and (istar % (ncols) == 0):
irow = irow + 1
icol = 3 * istar - 3 * ncols * irow
#print(istar, irow, icol, icol+1, icol+2)
ax1 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol),
aspect='equal')
ax1.axis('off')
#ax1.imshow(stamp, **tim.ima)
ax1.imshow(stamp,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax1.text(0.1,
0.9,
'{:2d}'.format(istar + 1),
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax1.transAxes)
ax2 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 1),
aspect='equal')
ax2.axis('off')
#ax2.imshow(stamp-model_mog, **tim.ima)
ax2.imshow(stamp - model_mog,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax2.text(0.1,
0.9,
'MoG',
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax2.transAxes)
ax2.text(0.08,
0.08,
'{:.3f}'.format(mag_mog),
color='white',
horizontalalignment='left',
verticalalignment='bottom',
transform=ax2.transAxes)
#ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14)
#ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex))
ax3 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 2),
aspect='equal')
ax3.axis('off')
#ax3.imshow(stamp-model_psfex, **tim.ima)
ax3.imshow(stamp - model_psfex,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax3.text(0.1,
0.9,
'PSFEx',
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax3.transAxes)
ax3.text(0.08,
0.08,
'{:.3f}'.format(mag_psfex),
color='white',
horizontalalignment='left',
verticalalignment='bottom',
transform=ax3.transAxes)
if istar == (nstar - 1):
break
fig.savefig(pngprefix + '-stargrid.png', bbox_inches='tight')
#disable_galaxy_cache()
tractor.cache = Cache(100)
set_galaxy_cache_size(100)
labels.append((len(mem), 'tractor'))
mem.append(memuse())
mod = tractor.getModelImage(im)
labels.append((len(mem), 'model'))
mem.append(memuse())
track('mod')
tractor.freezeParam('images')
for x in range(10):
#for x in range(2):
tractor.optimize(alphas=[1e-3, 1e-2, 0.1, 1])
labels.append((len(mem), 'opt %i' % x))
mem.append(memuse())
nun = gc.collect()
print nun, 'unreachable objects'
track('opt')
if False:
for x in range(10):
p0 = np.array(tractor.getParams())
ss = np.array(tractor.getStepSizes())
def psf_residuals(expnum,ccdname,stampsize=35,nstar=30,
magrange=(13,17),verbose=0):
# Set the debugging level.
if verbose==0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)
pngprefix = 'qapsf-{}-{}'.format(expnum,ccdname)
# Gather all the info we need about this CCD.
decals = Decals()
ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0]
band = ccd.filter
ps1band = dict(g=0,r=1,i=2,z=3,Y=4)
print('Band {}'.format(band))
#scales = dict(g=0.0066, r=0.01, z=0.025)
#vmin, vmax = np.arcsinh(-1), np.arcsinh(100)
#print(scales[band])
im = DecamImage(decals,ccd)
iminfo = im.get_image_info()
H,W = iminfo['dims']
wcs = im.get_wcs()
# Choose a uniformly selected subset of PS1 stars on this CCD.
ps1 = ps1cat(ccdwcs=wcs)
cat = ps1.get_stars(band=band,magrange=magrange)
rand = np.random.RandomState(seed=expnum*ccd.ccdnum)
these = rand.choice(len(cat)-1,nstar,replace=False)
#these = rand.random_integers(0,len(cat)-1,nstar)
cat = cat[these]
cat = cat[np.argsort(cat.median[:,ps1band[band]])] # sort by magnitude
#print(cat.nmag_ok)
# Make a QAplot of the positions of all the stars.
tim = im.get_tractor_image(const2psf=True)
img = tim.getImage()
#img = tim.getImage()/scales[band]
fig = plt.figure(figsize=(5,10))
ax = fig.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
#ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest',
# origin='lower',vmin=vmax,vmax=vmax)
ax.imshow(img, **tim.ima)
ax.axis('off')
ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.
format(band,expnum,ccdname,ccd.airmass,ccd.seeing))
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ax.text(xpos,ypos,'{:2d}'.format(istar+1),color='red',
horizontalalignment='left')
circ = plt.Circle((xpos,ypos),radius=30,color='g',fill=False,lw=1)
ax.add_patch(circ)
#radec = wcs.radec_bounds()
#ax.scatter(cat.ra,cat.dec)
#ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998])
#ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015])
#ax.set_xlabel('$RA\ (deg)$',fontsize=18)
#ax.set_ylabel('$Dec\ (deg)$',fontsize=18)
fig.savefig(pngprefix+'-ccd.png',bbox_inches='tight')
# Initialize the many-stamp QAplot
ncols = 3
nrows = np.ceil(nstar/ncols).astype('int')
inchperstamp = 2.0
fig = plt.figure(figsize=(inchperstamp*3*ncols,inchperstamp*nrows))
irow = 0
icol = 0
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
mag = ps1star.median[ps1band[band]] # r-band
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ix,iy = int(xpos), int(ypos)
# create a little tractor Image object around the star
slc = (slice(max(iy-stampsize, 0), min(iy+stampsize+1, H)),
slice(max(ix-stampsize, 0), min(ix+stampsize+1, W)))
# The PSF model 'const2Psf' is the one used in DR1: a 2-component
# Gaussian fit to PsfEx instantiated in the image center.
tim = im.get_tractor_image(slc=slc, const2psf=True)
stamp = tim.getImage()
ivarstamp = tim.getInvvar()
# Initialize a tractor PointSource from PS1 measurements
flux = NanoMaggies.magToNanomaggies(mag)
star = PointSource(RaDecPos(ra,dec), NanoMaggies(**{band: flux}))
# Fit just the source RA,Dec,flux.
tractor = Tractor([tim], [star])
tractor.freezeParam('images')
print('2-component MOG:', tim.psf)
tractor.printThawedParams()
for step in range(50):
dlnp,X,alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_mog = tractor.getModelImage(0)
chi2_mog = -2.0*tractor.getLogLikelihood()
mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
# Now change the PSF model to a pixelized PSF model from PsfEx instantiated
# at this place in the image.
psf = PixelizedPsfEx(im.psffn)
tim.psf = psf.constantPsfAt(xpos, ypos)
#print('PSF model:', tim.psf)
#tractor.printThawedParams()
for step in range(50):
dlnp,X,alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_psfex = tractor.getModelImage(0)
chi2_psfex = -2.0*tractor.getLogLikelihood()
mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
#mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95])
sig = np.std((stamp-model_psfex)[ivarstamp>0])
mn, mx = [-2.0*sig,5*sig]
# Generate a QAplot.
if (istar>0) and (istar%(ncols)==0):
irow = irow+1
icol = 3*istar - 3*ncols*irow
#print(istar, irow, icol, icol+1, icol+2)
ax1 = plt.subplot2grid((nrows,3*ncols), (irow,icol), aspect='equal')
ax1.axis('off')
#ax1.imshow(stamp, **tim.ima)
ax1.imshow(stamp,cmap='gray',interpolation='nearest',
origin='lower',vmin=mn,vmax=mx)
ax1.text(0.1,0.9,'{:2d}'.format(istar+1),color='white',
horizontalalignment='left',verticalalignment='top',
transform=ax1.transAxes)
ax2 = plt.subplot2grid((nrows,3*ncols), (irow,icol+1), aspect='equal')
ax2.axis('off')
#ax2.imshow(stamp-model_mog, **tim.ima)
ax2.imshow(stamp-model_mog,cmap='gray',interpolation='nearest',
origin='lower',vmin=mn,vmax=mx)
ax2.text(0.1,0.9,'MoG',color='white',
horizontalalignment='left',verticalalignment='top',
transform=ax2.transAxes)
ax2.text(0.08,0.08,'{:.3f}'.format(mag_mog),color='white',
horizontalalignment='left',verticalalignment='bottom',
transform=ax2.transAxes)
#ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14)
#ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex))
ax3 = plt.subplot2grid((nrows,3*ncols), (irow,icol+2), aspect='equal')
ax3.axis('off')
#ax3.imshow(stamp-model_psfex, **tim.ima)
ax3.imshow(stamp-model_psfex,cmap='gray',interpolation='nearest',
origin='lower',vmin=mn,vmax=mx)
ax3.text(0.1,0.9,'PSFEx',color='white',
horizontalalignment='left',verticalalignment='top',
transform=ax3.transAxes)
ax3.text(0.08,0.08,'{:.3f}'.format(mag_psfex),color='white',
horizontalalignment='left',verticalalignment='bottom',
transform=ax3.transAxes)
if istar==(nstar-1):
break
fig.savefig(pngprefix+'-stargrid.png',bbox_inches='tight')
def main():
# Where are the data?
datadir = os.path.join(os.path.dirname(__file__), 'data-decam')
name = 'decam-520206-S16'
imagefn = os.path.join(datadir, '%s-image-sub.fits' % name)
invvarfn = os.path.join(datadir, '%s-invvar-sub.fits' % name)
psfexfn = os.path.join(datadir, '%s-psfex.fits' % name)
catfn = os.path.join(datadir, 'tractor-1816p325-sub.fits')
# Read the image and inverse-variance maps.
image = fitsio.read(imagefn)
invvar = fitsio.read(invvarfn)
# The DECam inverse-variance maps are unfortunately corrupted
# by fpack, causing zeros to become negative. Fix those.
invvar[invvar < np.median(invvar) * 0.1] = 0.
H, W = image.shape
print('Subimage size:', image.shape)
# For the PSF model, we need to know what subimage region this is:
subimage_offset = (35, 1465)
# We also need the calibrated zeropoint.
zeropoint = 24.7787
# What filter was this image taken in? (z)
prim_header = fitsio.read_header(imagefn)
band = prim_header['FILTER'].strip()[0]
print('Band:', band)
# These DECam images were calibrated so that the zeropoints need
# an exposure-time factor, so add that in.
exptime = prim_header['EXPTIME']
zeropoint += 2.5 * np.log10(exptime)
# Read the PsfEx model file
psf = PixelizedPsfEx(psfexfn)
# Instantiate a constant pixelized PSF at the image center
# (of the subimage)
x0, y0 = subimage_offset
psf = psf.constantPsfAt(x0 + W / 2., y0 + H / 2.)
# Load the WCS model from the header
# We convert from the RA---TPV type to RA---SIP
header = fitsio.read_header(imagefn, ext=1)
wcsobj = wcs_pv2sip_hdr(header, stepsize=10)
# We'll just use a rough sky estimate...
skyval = np.median(image)
# Create the Tractor Image (tim).
tim = Image(data=image,
invvar=invvar,
psf=psf,
wcs=ConstantFitsWcs(wcsobj),
sky=ConstantSky(skyval),
photocal=LinearPhotoCal(
NanoMaggies.zeropointToScale(zeropoint), band=band))
# Read the official DECaLS DR3 catalog -- it has only two sources in this subimage.
catalog = fits_table(catfn)
print('Read', len(catalog), 'sources')
print('Source types:', catalog.type)
# Create Tractor sources corresponding to these two catalog
# entries.
# In DECaLS, the "SIMP" type is a round Exponential galaxy with a
# fixed 0.45" radius, but we'll treat it as a general Exp galaxy.
sources = []
for c in catalog:
# Create a "position" object given the catalog RA,Dec
position = RaDecPos(c.ra, c.dec)
# Create a "brightness" object; in the catalog, the fluxes are
# stored in a [ugrizY] array, so pull out the right index
band_index = 'ugrizY'.index(band)
flux = c.decam_flux[band_index]
brightness = NanoMaggies(**{band: flux})
# Depending on the source classification in the catalog, pull
# out different fields for the galaxy shape, and for the
# galaxy type. The DECaLS catalogs, conveniently, store
# galaxy shapes as (radius, e1, e2) ellipses.
if c.type.strip() == 'DEV':
shape = EllipseE(c.shapedev_r, c.shapedev_e1, c.shapedev_e2)
galclass = DevGalaxy
elif c.type.strip() == 'SIMP':
shape = EllipseE(c.shapeexp_r, c.shapeexp_e1, c.shapeexp_e2)
galclass = ExpGalaxy
else:
assert (False)
# Create the tractor galaxy object
source = galclass(position, brightness, shape)
print('Created', source)
sources.append(source)
# Create the Tractor object -- a list of tractor Images and a list of tractor sources.
tractor = Tractor([tim], sources)
# Render the initial model image.
print('Getting initial model...')
mod = tractor.getModelImage(0)
make_plot(tim, mod, 'Initial Scene', 'mod0.png')
# Instantiate a new source at the location of the unmodelled peak.
print('Adding new source...')
# Find the peak very naively...
ipeak = np.argmax((image - mod) * tim.inverr)
iy, ix = np.unravel_index(ipeak, tim.shape)
print('Residual peak at', ix, iy)
# Compute the RA,Dec location of the peak...
radec = tim.getWcs().pixelToPosition(ix, iy)
print('RA,Dec', radec)
# Try modelling it as a point source.
# We'll initialize the brightness arbitrarily to 1 nanomaggy (= mag 22.5)
brightness = NanoMaggies(**{band: 1.})
source = PointSource(radec, brightness)
# Add it to the catalog!
tractor.catalog.append(source)
# Render the new model image with this source added.
mod = tractor.getModelImage(0)
make_plot(tim, mod, 'New Source (Before Fit)', 'mod1.png')
print('Fitting new source...')
# Now we're going to fit for the properties of the new source we
# added.
# We don't want to fit for any of the image calibration properties:
tractor.freezeParam('images')
# And we don't (yet) want to fit the existing sources. The new
# source is index number 2, so freeze everything else in the catalog.
tractor.catalog.freezeAllBut(2)
print('Fitting parameters:')
tractor.printThawedParams()
# Do the actual optimization:
tractor.optimize_loop()
mod = tractor.getModelImage(0)
make_plot(tim, mod, 'New Source Fit', 'mod2.png')
print('Fitting sources simultaneously...')
# Now let's unfreeze all the sources and fit them simultaneously.
tractor.catalog.thawAllParams()
tractor.printThawedParams()
tractor.optimize_loop()
mod = tractor.getModelImage(0)
make_plot(tim, mod, 'Simultaneous Fit', 'mod3.png')
def main():
'''
This function generates the plots in the paper.
Some files and directories are assumed to exist in the current directory:
* WISE atlas tiles, from http://unwise.me/data/allsky-atlas.fits
* unwise-neo1-coadds, from http://unwise.me/data/neo1/
* unwise-neo1-coadds-half, unwise-neo1-coadds-quarter: directories
'''
# First, create the WCS into which we want to render
# degrees width to render in galactic coords
# |l| < 60
# |b| < 30
width = 120
# ~2 arcmin per pixel
W = int(width * 60.) / 2
H = W/2
zoom = 360. / width
wcs = anwcs_create_hammer_aitoff(0., 0., zoom, W, H, 0)
# Select WISE tiles that overlap. This atlas table is available
# from http://unwise.me/data/allsky-atlas.fits
# Select WISE tiles that overlap.
T = fits_table('allsky-atlas.fits')
print(len(T), 'tiles total')
T.ll,T.bb = radectolb(T.ra, T.dec)
I = np.flatnonzero(np.logical_or(T.ll < width+1,
T.ll > (360-width-1)) *
(T.bb > -width/2-1) * (T.bb < width/2+1))
T.cut(I)
print(len(I), 'tiles in L,B range')
# Create a coadd for each WISE band
lbpat = 'unwise-neo1-w%i-lb.fits'
imgs = []
for band in [1,2]:
outfn = lbpat % (band)
if os.path.exists(outfn):
print('Exists:', outfn)
img = fitsio.read(outfn)
imgs.append(img)
continue
coimg = np.zeros((H,W), np.float32)
conimg = np.zeros((H,W), np.float32)
for i,brick in enumerate(T.coadd_id):
# We downsample by 2, twice, just to make repeat runs a
# little faster.
# unWISE
fn = os.path.join('unwise-neo1-coadds', brick[:3], brick,
'unwise-%s-w%i-img-u.fits' % (brick, band))
qfn = os.path.join('unwise-neo1-coadds-quarter',
'unwise-%s-w%i.fits' % (brick, band))
hfn = os.path.join('unwise-neo1-coadds-half',
'unwise-%s-w%i.fits' % (brick, band))
if not os.path.exists(qfn):
if not os.path.exists(hfn):
print('Reading', fn)
halfsize(fn, hfn)
halfsize(hfn, qfn)
fn = qfn
print('Reading', fn)
img = fitsio.read(fn)
bwcs = Tan(fn, 0)
bh,bw = img.shape
# Coadd each unWISE pixel into the nearest target pixel.
xx,yy = np.meshgrid(np.arange(bw), np.arange(bh))
rr,dd = bwcs.pixelxy2radec(xx, yy)
ll,bb = radectolb(rr.ravel(), dd.ravel())
ll = ll.reshape(rr.shape)
bb = bb.reshape(rr.shape)
ok,ox,oy = wcs.radec2pixelxy(ll, bb)
ox = np.round(ox - 1).astype(int)
oy = np.round(oy - 1).astype(int)
K = (ox >= 0) * (ox < W) * (oy >= 0) * (oy < H) * ok
#print('ok:', np.unique(ok), 'x', ox.min(), ox.max(), 'y', oy.min(), oy.max())
assert(np.all(np.isfinite(img)))
if np.sum(K) == 0:
# no overlap
print('No overlap')
continue
np.add.at( coimg, (oy[K], ox[K]), img[K])
np.add.at(conimg, (oy[K], ox[K]), 1)
img = coimg / np.maximum(conimg, 1)
# Hack -- write and then read FITS WCS header.
fn = 'wiselb.wcs'
wcs.writeto(fn)
hdr = fitsio.read_header(fn)
hdr['CTYPE1'] = 'GLON-AIT'
hdr['CTYPE2'] = 'GLAT-AIT'
fitsio.write(outfn, img, header=hdr, clobber=True)
fitsio.write(outfn.replace('.fits', '-n.fits'), conimg,
header=hdr, clobber=True)
imgs.append(img)
w1,w2 = imgs
# Get/confirm L,B bounds...
H,W = w1.shape
print('Image size', W, 'x', H)
ok,l1,b1 = wcs.pixelxy2radec(1, (H+1)/2.)
ok,l2,b2 = wcs.pixelxy2radec(W, (H+1)/2.)
ok,l3,b3 = wcs.pixelxy2radec((W+1)/2., 1)
ok,l4,b4 = wcs.pixelxy2radec((W+1)/2., H)
print('L,B', (l1,b1), (l2,b2), (l3,b3), (l4,b4))
llo,lhi = l2,l1+360
blo,bhi = b3,b4
# Set plot sizes
plt.figure(1, figsize=(10,5))
plt.subplots_adjust(left=0.1, right=0.95, bottom=0.1, top=0.95)
plt.figure(2, figsize=(5,5))
plt.subplots_adjust(left=0.11, right=0.96, bottom=0.1, top=0.95)
suffix = '.pdf'
rgb = wise_rgb(w1, w2)
xlo,ylo = 0,0
plt.figure(1)
plt.clf()
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo, ylo, lticks=[60,30,0,330,300], bticks=[-30,-15,0,15,30])
plt.savefig('xbulge-00' + suffix)
# Compute the median of each row as a crude way of suppressing the
# Galactic plane
medy1 = np.median(w1, axis=1)
medy2 = np.median(w2, axis=1)
rgb = wise_rgb(w1 - medy1[:,np.newaxis],
w2 - medy2[:,np.newaxis])
# Zoom in a bit for Galactic plane subtracted version
lhi,llo,blo,bhi = 40, 320, -20, 20
okxy = np.array([wcs.radec2pixelxy(l,b) for l,b in [
(llo, blo), (llo, bhi), (lhi, blo), (lhi, bhi)]])
xlo = int(np.floor(min(okxy[:,-2])))
xhi = int(np.ceil (max(okxy[:,-2])))
ylo = int(np.floor(min(okxy[:,-1])))
yhi = int(np.ceil (max(okxy[:,-1])))
plt.clf()
plt.imshow(rgb[ylo:yhi, xlo:xhi, :],origin='lower', interpolation='nearest')
#lbticks(wcs, xlo, ylo, lticks=[40,20,0,340,320], bticks=[-20,-10,0,10,20])
lbticks(wcs, xlo, ylo, lticks=[30,15,0,345,330], bticks=[-20,-10,0,10,20])
plt.savefig('xbulge-01' + suffix)
# Zoom in on the core
lhi,llo,blo,bhi = 15, 345, -15, 15
ok,x1,y1 = wcs.radec2pixelxy(llo, blo)
ok,x2,y2 = wcs.radec2pixelxy(llo, bhi)
ok,x3,y3 = wcs.radec2pixelxy(lhi, blo)
ok,x4,y4 = wcs.radec2pixelxy(lhi, bhi)
xlo = int(np.floor(min(x1,x2,x3,x4)))
xhi = int(np.ceil (max(x1,x2,x3,x4)))
ylo = int(np.floor(min(y1,y2,y3,y4)))
yhi = int(np.ceil (max(y1,y2,y3,y4)))
print('xlo,ylo', xlo, ylo)
w1 = w1[ylo:yhi, xlo:xhi]
w2 = w2[ylo:yhi, xlo:xhi]
plt.figure(2)
# Apply color cut
w1mag = -2.5*(np.log10(w1) - 9.)
w2mag = -2.5*(np.log10(w2) - 9.)
cc = w1mag - w2mag
goodcolor = np.isfinite(cc)
mlo,mhi = np.percentile(cc[goodcolor], [5,95])
print('W1 - W2 color masks:', mlo,mhi)
mask = goodcolor * (cc > mlo) * (cc < mhi)
plt.clf()
rgb = wise_rgb(w1, w2)
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo,ylo)
plt.title('Data')
plt.savefig('xbulge-fit-data' + suffix)
plt.clf()
rgb = wise_rgb(w1 * mask, w2 * mask)
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo,ylo)
plt.title('Data (masked)')
plt.savefig('xbulge-fit-masked' + suffix)
ie = mask.astype(np.float32)
from tractor import (Image, NCircularGaussianPSF, LinearPhotoCal, Tractor,
PixPos, Fluxes)
from tractor.galaxy import ExpGalaxy, GalaxyShape
# Create Tractor images
tim1 = Image(data=w1 * mask, inverr=ie,
psf=NCircularGaussianPSF([1.],[1.]),
photocal=LinearPhotoCal(1., 'w1'))
tim2 = Image(data=w2 * mask, inverr=ie,
psf=NCircularGaussianPSF([1.],[1.]),
photocal=LinearPhotoCal(1., 'w2'))
H,W = w1.shape
gal = ExpGalaxy(PixPos(W/2, H/2), Fluxes(w1=w1.sum(), w2=w2.sum()),
GalaxyShape(200, 0.4, 90.))
tractor = Tractor([tim1, tim2],[gal])
# fitsio.write('data-w1.fits', w1 * mask, clobber=True)
# fitsio.write('data-w2.fits', w2 * mask, clobber=True)
# fitsio.write('mask.fits', mask.astype(np.uint8), clobber=True)
# Optimize galaxy model
tractor.freezeParam('images')
for step in range(50):
dlnp,x,alpha = tractor.optimize()
print('dlnp', dlnp)
print('x', x)
print('alpha', alpha)
print('Galaxy', gal)
if dlnp == 0:
break
# Get galaxy model images, compute residuals
mod1 = tractor.getModelImage(0)
resid1 = w1 - mod1
mod2 = tractor.getModelImage(1)
resid2 = w2 - mod2
rgb = wise_rgb(mod1, mod2)
plt.clf()
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo,ylo)
plt.title('Model')
plt.savefig('xbulge-fit-model' + suffix)
rgb = resid_rgb(resid1, resid2)
plt.clf()
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo,ylo)
plt.title('Residuals')
plt.savefig('xbulge-fit-resid' + suffix)
rgb = resid_rgb(resid1*mask, resid2*mask)
plt.clf()
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo,ylo)
plt.title('Residuals (masked)')
plt.savefig('xbulge-fit-residmasked' + suffix)
# fitsio.write('resid1.fits', resid1, clobber=True)
# fitsio.write('resid2.fits', resid2, clobber=True)
# Compute median-smoothed residuals
fr1 = np.zeros_like(resid1)
fr2 = np.zeros_like(resid2)
median_smooth(resid1, np.logical_not(mask), 25, fr1)
median_smooth(resid2, np.logical_not(mask), 25, fr2)
rgb = resid_rgb(fr1, fr2)
plt.clf()
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo,ylo)
plt.title('Residuals (smoothed)')
plt.savefig('xbulge-fit-smooth2' + suffix)
def main(decals=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for decals.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if decals is None:
decals = Decals()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from decals-ccds.fits table
T = decals.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(np.array([f.strip() == opt.filename for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert(len(T) == 1)
im = decals.get_image_object(T[0])
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True)
print('Got tim:', tim)
if opt.catfn in ['DR1', 'DR2']:
if opt.catalog_path is None:
opt.catalog_path = opt.catfn.lower()
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, decals=decals)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3],
'tractor-%s.fits' % b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
TT.append(T)
T = merge_tables(TT)
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T
T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T
cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE)
# print('Got cat:', cat)
print('Forced photom...')
opti = None
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T))
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap
ap = 1./(np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap
if opt.forced:
kwa = {}
if opt.plots is None:
kwa.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True, fitstats=True,
shared_params=False, **kwa)
if opt.plots:
(data,mod,ie,chi,roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([src.getBrightness().getFlux(tim.band)
for src in cat]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2 .astype(np.float32)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, decals.decals_dir)
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
fname = os.path.join(d2, d1, fname)
print('Trimmed filename to', fname)
#version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file'))
version_hdr.add_record(dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext'))
version_hdr.add_record(dict(name='CAMERA', value='DECam', comment='Dark Energy Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {'mjd':'sec', 'exptime':'sec', 'flux':'nanomaggy',
'flux_ivar':'1/nanomaggy^2'}
columns = F.get_columns()
for i,col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
print('Finished forced phot:', Time()-t0)
return 0
else:
rtxt = 'X'
ab = 1. - t.ell_gim2d
if ab >= 0 and ab < 1:
abtxt = '%.2f' % ab
else:
abtxt = 'X'
plt.text(t.xx,
t.yy,
'T%i, r %s ab %s' % (t.type, rtxt, abtxt),
color='r')
plt.axis(ax)
ps.savefig()
tractor.freezeParam('images')
for src in srcs:
src.freezeAllBut('brightness')
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
tractor.optimizer = opti
print('Forced phot...')
kwa = {}
R = tractor.optimize_forced_photometry(shared_params=False,
variance=True,
**kwa)
print('R:', R, dir(R))
def galex_coadds(onegal,
galaxy=None,
radius_mosaic=30,
radius_mask=None,
pixscale=1.5,
ref_pixscale=0.262,
output_dir=None,
galex_dir=None,
log=None,
centrals=True,
verbose=False):
'''Generate custom GALEX cutouts.
radius_mosaic and radius_mask in arcsec
pixscale: GALEX pixel scale in arcsec/pixel.
'''
import fitsio
import matplotlib.pyplot as plt
from astrometry.libkd.spherematch import match_radec
from astrometry.util.resample import resample_with_wcs, OverlapError
from tractor import (Tractor, NanoMaggies, Image, LinearPhotoCal,
NCircularGaussianPSF, ConstantFitsWcs, ConstantSky)
from legacypipe.survey import imsave_jpeg
from legacypipe.catalog import read_fits_catalog
if galaxy is None:
galaxy = 'galaxy'
if galex_dir is None:
galex_dir = os.environ.get('GALEX_DIR')
if output_dir is None:
output_dir = '.'
if radius_mask is None:
radius_mask = radius_mosaic
radius_search = 5.0 # [arcsec]
else:
radius_search = radius_mask
W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels]
targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0,
-pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W),
float(H))
# Read the custom Tractor catalog
tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy))
if not os.path.isfile(tractorfile):
print('Missing Tractor catalog {}'.format(tractorfile))
return 0
cat = fits_table(tractorfile)
print('Read {} sources from {}'.format(len(cat), tractorfile),
flush=True,
file=log)
keep = np.ones(len(cat)).astype(bool)
if centrals:
# Find the large central galaxy and mask out (ignore) all the models
# which are within its elliptical mask.
# This algorithm will have to change for mosaics not centered on large
# galaxies, e.g., in galaxy groups.
m1, m2, d12 = match_radec(cat.ra,
cat.dec,
onegal['RA'],
onegal['DEC'],
radius_search / 3600.0,
nearest=False)
if len(m1) == 0:
print('No central galaxies found at the central coordinates!',
flush=True,
file=log)
else:
pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions
for mm in m1:
morphtype = cat.type[mm].strip()
if morphtype == 'EXP' or morphtype == 'COMP':
e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[
mm], cat.shapeexp_r[mm] # [arcsec]
elif morphtype == 'DEV' or morphtype == 'COMP':
e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[
mm], cat.shapedev_r[mm] # [arcsec]
else:
r50 = None
if r50:
majoraxis = r50 * 5 / pixscale # [pixels]
ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2)
these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis,
ba * majoraxis,
np.radians(phi),
cat.bx * pixfactor,
cat.by * pixfactor)
if np.sum(these) > 0:
#keep[these] = False
pass
print('Hack!')
keep[mm] = False
#srcs = read_fits_catalog(cat)
#_srcs = np.array(srcs)[~keep].tolist()
#mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0)
#import matplotlib.pyplot as plt
##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png')
#plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png')
#pdb.set_trace()
srcs = read_fits_catalog(cat)
for src in srcs:
src.freezeAllBut('brightness')
#srcs_nocentral = np.array(srcs)[keep].tolist()
# Find all overlapping GALEX tiles and then read the tims.
galex_tiles = _read_galex_tiles(targetwcs,
galex_dir,
log=log,
verbose=verbose)
gbands = ['n', 'f']
nicegbands = ['NUV', 'FUV']
zps = dict(n=20.08, f=18.82)
coimgs, comods, coresids, coimgs_central, comods_nocentral = [], [], [], [], []
for niceband, band in zip(nicegbands, gbands):
J = np.flatnonzero(galex_tiles.get('has_' + band))
print(len(J), 'GALEX tiles have coverage in band', band)
coimg = np.zeros((H, W), np.float32)
comod = np.zeros((H, W), np.float32)
cowt = np.zeros((H, W), np.float32)
comod_nocentral = np.zeros((H, W), np.float32)
for src in srcs:
src.setBrightness(NanoMaggies(**{band: 1}))
for j in J:
brick = galex_tiles[j]
fn = os.path.join(
galex_dir, brick.tilename.strip(),
'%s-%sd-intbgsub.fits.gz' % (brick.brickname, band))
#print(fn)
gwcs = Tan(*[
float(f) for f in [
brick.crval1, brick.crval2, brick.crpix1, brick.crpix2,
brick.cdelt1, 0., 0., brick.cdelt2, 3840., 3840.
]
])
img = fitsio.read(fn)
#print('Read', img.shape)
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, gwcs, [], 3)
except OverlapError:
continue
K = np.flatnonzero(img[Yi, Xi] != 0.)
if len(K) == 0:
continue
Yo, Xo, Yi, Xi = Yo[K], Xo[K], Yi[K], Xi[K]
wt = brick.get(band + 'exptime')
coimg[Yo, Xo] += wt * img[Yi, Xi]
cowt[Yo, Xo] += wt
x0, x1, y0, y1 = min(Xi), max(Xi), min(Yi), max(Yi)
subwcs = gwcs.get_subimage(x0, y0, x1 - x0 + 1, y1 - y0 + 1)
twcs = ConstantFitsWcs(subwcs)
timg = img[y0:y1 + 1, x0:x1 + 1]
tie = np.ones_like(timg) ## HACK!
#hdr = fitsio.read_header(fn)
#zp = hdr['']
zp = zps[band]
photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp),
band=band)
tsky = ConstantSky(0.0)
# HACK -- circular Gaussian PSF of fixed size...
# in arcsec
#fwhms = dict(NUV=6.0, FUV=6.0)
# -> sigma in pixels
#sig = fwhms[band] / 2.35 / twcs.pixel_scale()
sig = 6.0 / np.sqrt(8 * np.log(2)) / twcs.pixel_scale()
tpsf = NCircularGaussianPSF([sig], [1.])
tim = Image(data=timg,
inverr=tie,
psf=tpsf,
wcs=twcs,
sky=tsky,
photocal=photocal,
name='GALEX ' + band + brick.brickname)
## Build the model image with and without the central galaxy model.
tractor = Tractor([tim], srcs)
mod = tractor.getModelImage(0)
tractor.freezeParam('images')
tractor.optimize_forced_photometry(priors=False,
shared_params=False)
mod = tractor.getModelImage(0)
srcs_nocentral = np.array(srcs)[keep].tolist()
#srcs_nocentral = np.array(srcs)[nocentral].tolist()
tractor_nocentral = Tractor([tim], srcs_nocentral)
mod_nocentral = tractor_nocentral.getModelImage(0)
comod[Yo, Xo] += wt * mod[Yi - y0, Xi - x0]
comod_nocentral[Yo, Xo] += wt * mod_nocentral[Yi - y0, Xi - x0]
coimg /= np.maximum(cowt, 1e-18)
comod /= np.maximum(cowt, 1e-18)
comod_nocentral /= np.maximum(cowt, 1e-18)
coresid = coimg - comod
# Subtract the model image which excludes the central (comod_nocentral)
# from the data (coimg) to isolate the light of the central
# (coimg_central).
coimg_central = coimg - comod_nocentral
coimgs.append(coimg)
comods.append(comod)
coresids.append(coresid)
comods_nocentral.append(comod_nocentral)
coimgs_central.append(coimg_central)
# Write out the final images with and without the central, making sure
# to apply the zeropoint to go from counts/s to AB nanomaggies.
# https://asd.gsfc.nasa.gov/archive/galex/FAQ/counts_background.html
for thisimg, imtype in zip((coimg, comod, comod_nocentral),
('image', 'model', 'model-nocentral')):
fitsfile = os.path.join(
output_dir, '{}-{}-{}.fits'.format(galaxy, imtype, niceband))
if verbose:
print('Writing {}'.format(fitsfile))
fitsio.write(fitsfile,
thisimg * 10**(-0.4 * (zp - 22.5)),
clobber=True)
# Build a color mosaic (but note that the images here are in units of
# background-subtracted counts/s).
#_galex_rgb = _galex_rgb_moustakas
#_galex_rgb = _galex_rgb_dstn
_galex_rgb = _galex_rgb_official
for imgs, imtype in zip(
(coimgs, comods, coresids, comods_nocentral, coimgs_central),
('image', 'model', 'resid', 'model-nocentral', 'image-central')):
rgb = _galex_rgb(imgs)
jpgfile = os.path.join(output_dir,
'{}-{}-FUVNUV.jpg'.format(galaxy, imtype))
if verbose:
print('Writing {}'.format(jpgfile))
imsave_jpeg(jpgfile, rgb, origin='lower')
return 1
def main(decals=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for decals.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if decals is None:
decals = Decals()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from decals-ccds.fits table
T = decals.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(
np.array([f.strip() == opt.filename
for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert (len(T) == 1)
im = decals.get_image_object(T[0])
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True)
print('Got tim:', tim)
if opt.catfn in ['DR1', 'DR2']:
if opt.catalog_path is None:
opt.catalog_path = opt.catfn.lower()
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, decals=decals)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3],
'tractor-%s.fits' % b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) *
(yy >= -margin) * (yy <= (H + margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
TT.append(T)
T = merge_tables(TT)
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T
T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T
cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE)
# print('Got cat:', cat)
print('Forced photom...')
opti = None
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T))
ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack(
[tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap
ap = 1. / (np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap
if opt.forced:
kwa = {}
if opt.plots is None:
kwa.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True,
fitstats=True,
shared_params=False,
**kwa)
if opt.plots:
(data, mod, ie, chi, roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest',
origin='lower',
vmin=-5,
vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([
src.getBrightness().getFlux(tim.band) for src in cat
]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2.astype(np.float32)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, decals.decals_dir)
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
fname = os.path.join(d2, d1, fname)
print('Trimmed filename to', fname)
#version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file'))
version_hdr.add_record(
dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file'))
version_hdr.add_record(
dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext'))
version_hdr.add_record(
dict(name='CAMERA', value='DECam', comment='Dark Energy Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='EXPOSURE',
value='decam-%s-%s' % (im.expnum, im.ccdname),
comment='Name of this image'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {
'mjd': 'sec',
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2'
}
columns = F.get_columns()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
print('Finished forced phot:', Time() - t0)
return 0
