def get_ps1_chip_image(filename, offset=(0, 0), npix=None):
filename_base = filename[:-5]
im = pyfits.getdata(filename_base + '.fits')
wt = pyfits.getdata(filename_base + '.wt.fits')
mk = pyfits.getdata(filename_base + '.mk.fits')
psffile = filename_base[:-3] + '.psf'
psffp = tempfile.NamedTemporaryFile()
psffp.close()
subprocess.call([
"dannyVizPSF",
str(im.shape[0]),
str(im.shape[1]),
str(im.shape[0] / 2),
str(im.shape[1] / 2), "51", "51", psffile, psffp.name
])
psfstamp = pyfits.getdata(psffp.name)
smffn = get_smf_filename(os.path.basename(filename_base)[:11])
im[mk != 0] = 0
wt[mk != 0] = 0
if npix is None:
npix = im.shape
im = im[offset[0]:offset[0] + npix[0], offset[1]:offset[1] + npix[1]]
wt = wt[offset[0]:offset[0] + npix[0], offset[1]:offset[1] + npix[1]]
mk = mk[offset[0]:offset[0] + npix[0], offset[1]:offset[1] + npix[1]]
invvar = (1. / (wt + (wt == 0))) * (wt != 0)
hsmf = pyfits.getheader(smffn)
zp = hsmf['MAG_ZP']
hchip = pyfits.getheader(filename_base + '.fits', 1)
chip = hchip['FPPOS'].upper().strip()
filterid = 'PS1_' + hsmf['filterid'][0]
tpsf = tractor.GaussianMixturePSF.fromStamp(psfstamp, N=3)
twcs = wcs_ps1(smffn, chip, offset=offset)
tsky = tractor.ConstantSky(0)
photocal = tractor.MagsPhotoCal(
filterid, zp + 2.5 * numpy.log10(hchip['exptime']) + 0.5)
tim = tractor.Image(data=im,
invvar=invvar,
psf=tpsf,
wcs=twcs,
photocal=photocal,
sky=tsky,
name='PS1 %s %s' % (filterid, filename_base))
tim.zr = [-100, 100]
tim.extent = [0, im.shape[0], 0, im.shape[1]]
return tim
def get_galnorm(seeing, pixsc):
from tractor.patch import ModelMask
S = 32
W,H = S*2+1, S*2+1
psf_sigma = seeing / 2.35 / pixsc
tim = tractor.Image(data=np.zeros((H,W)), inverr=np.ones((H,W)),
psf=tractor.NCircularGaussianPSF([psf_sigma], [1.]),
wcs=tractor.NullWCS(pixscale=pixsc))
gal = SimpleGalaxy(tractor.PixPos(S,S), tractor.Flux(1.))
mm = ModelMask(0, 0, W, H)
galmod = gal.getModelPatch(tim, modelMask=mm).patch
galmod = np.maximum(0, galmod)
galmod /= galmod.sum()
return np.sqrt(np.sum(galmod**2))
def fit_general_gaussian(self,
img,
sig1,
xi,
yi,
fluxi,
psf_r=15,
ps=None):
import tractor
H, W = img.shape
ix = int(np.round(xi))
iy = int(np.round(yi))
xlo = max(0, ix - psf_r)
xhi = min(W, ix + psf_r + 1)
ylo = max(0, iy - psf_r)
yhi = min(H, iy + psf_r + 1)
xx, yy = np.meshgrid(np.arange(xlo, xhi), np.arange(ylo, yhi))
r2 = (xx - xi)**2 + (yy - yi)**2
keep = (r2 < psf_r**2)
pix = img[ylo:yhi, xlo:xhi].copy()
ie = np.zeros_like(pix)
ie[keep] = 1. / sig1
psf = tractor.NCircularGaussianPSF([4.], [1.])
tim = tractor.Image(data=pix, inverr=ie, psf=psf)
src = tractor.PointSource(tractor.PixPos(xi - xlo, yi - ylo),
tractor.Flux(fluxi))
tr = tractor.Tractor([tim], [src])
src.pos.addGaussianPrior('x', 0., 1.)
doplot = (ps is not None)
if doplot:
mod0 = tr.getModelImage(0)
tim.freezeAllBut('psf')
psf.freezeAllBut('sigmas')
# print('Optimizing params:')
# tr.printThawedParams()
#print('Parameter step sizes:', tr.getStepSizes())
optargs = dict(priors=False, shared_params=False)
for step in range(50):
dlnp, x, alpha = tr.optimize(**optargs)
if dlnp == 0:
break
# Now fit only the PSF size
tr.freezeParam('catalog')
# print('Optimizing params:')
# tr.printThawedParams()
for step in range(50):
dlnp, x, alpha = tr.optimize(**optargs)
if dlnp == 0:
break
# fwhms.append(psf.sigmas[0] * 2.35 * self.pixscale)
if doplot:
mod1 = tr.getModelImage(0)
chi1 = tr.getChiImage(0)
# Now switch to a non-isotropic PSF
s = psf.sigmas[0]
#print('Isotropic fit sigma', s)
s = np.clip(s, 1., 5.)
tim.psf = tractor.GaussianMixturePSF(1., 0., 0., s**2, s**2, 0.)
#print('Optimizing params:')
#tr.printThawedParams()
try:
for step in range(50):
dlnp, x, alpha = tr.optimize(**optargs)
#print('PSF:', tim.psf)
if dlnp == 0:
break
except:
import traceback
print('Error during fitting PSF in a focus frame; not to worry')
traceback.print_exc()
print(
'(The above was just an error during fitting one star in a focus frame; not to worry.)'
)
# Don't need to re-fit source params because PSF ampl and mean
# can fit for flux and position.
if doplot:
mod2 = tr.getModelImage(0)
chi2 = tr.getChiImage(0)
kwa = dict(vmin=-3 * sig1, vmax=50 * sig1, cmap='gray')
plt.clf()
plt.subplot(2, 3, 1)
plt.title('Image')
dimshow(pix, ticks=False, **kwa)
plt.subplot(2, 3, 2)
plt.title('Initial model')
dimshow(mod0, ticks=False, **kwa)
plt.subplot(2, 3, 3)
plt.title('Isotropic model')
dimshow(mod1, ticks=False, **kwa)
plt.subplot(2, 3, 4)
plt.title('Final model')
dimshow(mod2, ticks=False, **kwa)
plt.subplot(2, 3, 5)
plt.title('Isotropic chi')
dimshow(chi1, vmin=-10, vmax=10, ticks=False)
plt.subplot(2, 3, 6)
plt.title('Final chi')
dimshow(chi2, vmin=-10, vmax=10, ticks=False)
plt.suptitle('PSF fit')
ps.savefig()
return tim.psf.getParams()[-3:]
def main():
# In LSST meas-deblend (on lsst6):
# python examples/suprimePlot.py --data ~dstn/lsst/ACT-data -v 126969 -c 5 --data-range -100 300 --roi 0 500 0 500 --psf psf.fits --image img.fits --sources srcs.fits
from optparse import OptionParser
import sys
parser = OptionParser(usage=('%prog <img> <psf> <srcs>'))
parser.add_option('-v',
'--verbose',
dest='verbose',
action='count',
default=0,
help='Make more verbose')
opt, args = parser.parse_args()
if len(args) != 3:
parser.print_help()
sys.exit(-1)
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
imgfn, psffn, srcfn = args
pimg = pyfits.open(imgfn)
if len(pimg) != 4:
print('Image must have 3 extensions')
sys.exit(-1)
img = pimg[1].data
mask = pimg[2].data
maskhdr = pimg[2].header
var = pimg[3].data
del pimg
print('var', var.shape)
#print var
print('mask', mask.shape)
#print mask
print('img', img.shape)
#print img
mask = mask.astype(np.int16)
for bit in range(16):
on = ((mask & (1 << bit)) != 0)
print('Bit', bit, 'has', np.sum(on), 'pixels set')
'''
MP_BAD = 0
Bit 0 has 2500 pixels set
MP_SAT = 1
Bit 1 has 5771 pixels set
MP_INTRP= 2
Bit 2 has 11269 pixels set
MP_CR = 3
Bit 3 has 136 pixels set
MP_EDGE = 4
Bit 4 has 11856 pixels set
HIERARCH MP_DETECTED = 5
Bit 5 has 37032 pixels set
'''
print('Mask header:', maskhdr)
maskplanes = {}
print('Mask planes:')
for card in maskhdr.ascardlist():
if not card.key.startswith('MP_'):
continue
print(card.value, card.key)
maskplanes[card.key[3:]] = card.value
print('Variance range:', var.min(), var.max())
print('Image median:', np.median(img.ravel()))
invvar = 1. / var
invvar[var == 0] = 0.
invvar[var < 0] = 0.
sig = np.sqrt(np.median(var))
H, W = img.shape
for k, v in maskplanes.items():
plt.clf()
I = ((mask & (1 << v)) != 0)
rgb = np.zeros((H, W, 3))
clipimg = np.clip((img - (-3. * sig)) / (13. * sig), 0, 1)
cimg = clipimg.copy()
cimg[I] = 1
rgb[:, :, 0] = cimg
cimg = clipimg.copy()
cimg[I] = 0
rgb[:, :, 1] = cimg
rgb[:, :, 2] = cimg
plt.imshow(rgb, interpolation='nearest', origin='lower')
plt.title(k)
plt.savefig('mask-%s.png' % k.lower())
badmask = sum([(1 << maskplanes[k])
for k in ['BAD', 'SAT', 'INTRP', 'CR']])
# HACK -- left EDGE sucks
badmask += (1 << maskplanes['EDGE'])
#badmask = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3)
#badmask |= (1 << 4)
print('Masking out: 0x%x' % badmask)
invvar[(mask & badmask) != 0] = 0.
assert (all(np.isfinite(img.ravel())))
assert (all(np.isfinite(invvar.ravel())))
psf = pyfits.open(psffn)[0].data
print('psf', psf.shape)
psf /= psf.sum()
from tractor.emfit import em_fit_2d
from tractor.fitpsf import em_init_params
# Create Gaussian mixture model PSF approximation.
S = psf.shape[0]
# number of Gaussian components
K = 3
w, mu, sig = em_init_params(K, None, None, None)
II = psf.copy()
II /= II.sum()
# HIDEOUS HACK
II = np.maximum(II, 0)
print('Multi-Gaussian PSF fit...')
xm, ym = -(S / 2), -(S / 2)
em_fit_2d(II, xm, ym, w, mu, sig)
print('w,mu,sig', w, mu, sig)
mypsf = tractor.GaussianMixturePSF(w, mu, sig)
P = mypsf.getPointSourcePatch(S / 2, S / 2)
mn, mx = psf.min(), psf.max()
ima = dict(interpolation='nearest', origin='lower', vmin=mn, vmax=mx)
plt.clf()
plt.subplot(1, 2, 1)
plt.imshow(psf, **ima)
plt.subplot(1, 2, 2)
pimg = np.zeros_like(psf)
P.addTo(pimg)
plt.imshow(pimg, **ima)
plt.savefig('psf.png')
sig = np.sqrt(np.median(var))
plt.clf()
plt.hist(img.ravel(), 100, range=(-3. * sig, 3. * sig))
plt.savefig('imghist.png')
srcs = fits_table(srcfn)
print('Initial:', len(srcs), 'sources')
# Trim sources with x=0 or y=0
srcs = srcs[(srcs.x != 0) * (srcs.y != 0)]
print('Trim on x,y:', len(srcs), 'sources left')
# Zero out nans & infs
for c in ['theta', 'a', 'b']:
I = np.logical_not(np.isfinite(srcs.get(c)))
srcs.get(c)[I] = 0.
# Set sources with flux=NaN to something more sensible...
I = np.logical_not(np.isfinite(srcs.flux))
srcs.flux[I] = 1.
# Sort sources by flux.
srcs = srcs[np.argsort(-srcs.flux)]
# Trim sources that are way outside the image.
margin = 8. * np.maximum(srcs.a, srcs.b)
H, W = img.shape
srcs = srcs[(srcs.x > -margin) * (srcs.y > -margin) *
(srcs.x < (W + margin) * (srcs.y < (H + margin)))]
print('Trim out-of-bounds:', len(srcs), 'sources left')
wcs = tractor.FitsWcs(Sip(imgfn, 1))
#wcs = tractor.NullWCS()
timg = tractor.Image(data=img,
invvar=invvar,
psf=mypsf,
wcs=wcs,
sky=tractor.ConstantSky(0.),
photocal=tractor.NullPhotoCal(),
name='image')
inverr = timg.getInvError()
assert (all(np.isfinite(inverr.ravel())))
tsrcs = []
for s in srcs:
#pos = tractor.PixPos(s.x, s.y)
pos = tractor.RaDecPos(s.ra, s.dec)
if s.a > 0 and s.b > 0:
eflux = tractor.Flux(s.flux / 2.)
dflux = tractor.Flux(s.flux / 2.)
re, ab, phi = s.a, s.b / s.a, 90. - s.theta
eshape = gal.GalaxyShape(re, ab, phi)
dshape = gal.GalaxyShape(re, ab, phi)
print('Fluxes', eflux, dflux)
tsrc = gal.CompositeGalaxy(pos, eflux, eshape, dflux, dshape)
else:
flux = tractor.Flux(s.flux)
print('Flux', flux)
tsrc = tractor.PointSource(pos, flux)
tsrcs.append(tsrc)
chug = tractor.Tractor([timg])
for src in tsrcs:
if chug.getModelPatch(timg, src) is None:
print('Dropping non-overlapping source:', src)
continue
chug.addSource(src)
print('Kept a total of', len(chug.catalog), 'sources')
ima = dict(interpolation='nearest',
origin='lower',
vmin=-3. * sig,
vmax=10. * sig)
chia = dict(interpolation='nearest', origin='lower', vmin=-5., vmax=5.)
plt.clf()
plt.imshow(img, **ima)
plt.colorbar()
plt.savefig('img.png')
plt.clf()
plt.imshow(invvar, interpolation='nearest', origin='lower')
plt.colorbar()
plt.savefig('invvar.png')
mod = chug.getModelImages()[0]
plt.clf()
plt.imshow(mod, **ima)
plt.colorbar()
plt.savefig('mod-0.png')
chi = chug.getChiImage(0)
plt.clf()
plt.imshow(chi, **chia)
plt.colorbar()
plt.savefig('chi-0.png')
for step in range(5):
cat = chug.getCatalog()
for src in cat:
if chug.getModelPatch(timg, src) is None:
print('Dropping non-overlapping source:', src)
chug.removeSource(src)
print('Kept a total of', len(chug.catalog), 'sources')
#cat = chug.getCatalog()
#for i,src in enumerate([]):
#for i,src in enumerate(chug.getCatalog()):
#for i in range(len(cat)):
i = 0
while i < len(cat):
src = cat[i]
#print 'Step', i
#for j,s in enumerate(cat):
# x,y = timg.getWcs().positionToPixel(s, s.getPosition())
# print ' ',
# if j == i:
# print '*',
# print '(%6.1f, %6.1f)'%(x,y), s
print('Optimizing source', i, 'of', len(cat))
x, y = timg.getWcs().positionToPixel(src.getPosition(), src)
print('(%6.1f, %6.1f)' % (x, y), src)
# pre = src.getModelPatch(timg)
s1 = str(src)
print('src1 ', s1)
dlnp1, X, a = chug.optimizeCatalogFluxes(srcs=[src])
s2 = str(src)
dlnp2, X, a = chug.optimizeCatalogAtFixedComplexityStep(srcs=[src],
sky=False)
s3 = str(src)
#post = src.getModelPatch(timg)
print('src1 ', s1)
print('src2 ', s2)
print('src3 ', s3)
print('dlnp', dlnp1, dlnp2)
if chug.getModelPatch(timg, src) is None:
print('After optimizing, no overlap!')
print('Removing source', src)
chug.removeSource(src)
i -= 1
i += 1
# plt.clf()
# plt.subplot(2,2,1)
# img = timg.getImage()
# (x0,x1,y0,y1) = pre.getExtent()
# plt.imshow(img, **ima)
# ax = plt.axis()
# plt.plot([x0,x0,x1,x1,x0], [y0,y1,y1,y0,y0], 'k-', lw=2)
# plt.axis(ax)
# plt.subplot(2,2,3)
# plt.imshow(pre.getImage(), **ima)
# plt.subplot(2,2,4)
# plt.imshow(post.getImage(), **ima)
# plt.savefig('prepost-s%i-s%03i.png' % (step, i))
#
# mod = chug.getModelImages()[0]
# plt.clf()
# plt.imshow(mod, **ima)
# plt.colorbar()
# plt.savefig('mod-s%i-s%03i.png' % (step, i))
# chi = chug.getChiImage(0)
# plt.clf()
# plt.imshow(chi, **chia)
# plt.colorbar()
# plt.savefig('chi-s%i-s%03i.png' % (step, i))
#dlnp,x,a = chug.optimizeCatalogFluxes()
#print 'fluxes: dlnp', dlnp
#dlnp,x,a = chug.optimizeCatalogAtFixedComplexityStep()
#print 'opt: dlnp', dlnp
mod = chug.getModelImages()[0]
plt.clf()
plt.imshow(mod, **ima)
plt.colorbar()
plt.savefig('mod-%i.png' % (step + 1))
chi = chug.getChiImage(0)
plt.clf()
plt.imshow(chi, **chia)
plt.colorbar()
plt.savefig('chi-%i.png' % (step + 1))
return
for step in range(5):
chug.optimizeCatalogFluxes()
mod = chug.getModelImages()[0]
plt.clf()
plt.imshow(mod, **ima)
plt.colorbar()
plt.savefig('mod-s%i.png' % step)
chi = chug.getChiImage(0)
plt.clf()
plt.imshow(chi, **chia)
plt.colorbar()
plt.savefig('chi-s%i.png' % step)
'''
if __name__ == '__main__':
H, W = 100, 100
pixscale = 0.262
ra, dec = 40., 10.
psf_sigma = 1.4 # pixels
v = psf_sigma**2
ps = pixscale / 3600.
wcs = Tan(ra, dec, W / 2. + 0.5, H / 2. + 0.5, -ps, 0., 0., ps, float(W),
float(H))
tim = tractor.Image(data=np.zeros((H, W), np.float32),
inverr=np.ones((H, W), np.float32),
psf=tractor.GaussianMixturePSF(1., 0., 0., v, v, 0.),
wcs=tractor.ConstantFitsWcs(wcs))
src = RexGalaxy(tractor.RaDecPos(ra, dec), tractor.Flux(100.),
LogRadius(0.))
tr = tractor.Tractor([tim], [src])
mod = tr.getModelImage(0)
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower')
plt.savefig('rex.png')
# add noise with std 1.
noisy = mod + np.random.normal(size=mod.shape)
# make that the tim's data
tim.data = noisy
def run(self, ps=None, focus=False, momentsize=5, n_fwhm=100):
import pylab as plt
from astrometry.util.plotutils import dimshow, plothist
from legacyanalysis.ps1cat import ps1cat
import photutils
import tractor
fn = self.fn
ext = self.ext
pixsc = self.pixscale
F = fitsio.FITS(fn)
primhdr = F[0].read_header()
self.primhdr = primhdr
img, hdr = self.read_raw(F, ext)
self.hdr = hdr
# pre sky-sub
mn, mx = np.percentile(img.ravel(), [25, 98])
self.imgkwa = dict(vmin=mn, vmax=mx, cmap='gray')
if self.debug and ps is not None:
plt.clf()
dimshow(img, **self.imgkwa)
plt.title('Raw image')
ps.savefig()
M = 200
plt.clf()
plt.subplot(2, 2, 1)
dimshow(img[-M:, :M], ticks=False, **self.imgkwa)
plt.subplot(2, 2, 2)
dimshow(img[-M:, -M:], ticks=False, **self.imgkwa)
plt.subplot(2, 2, 3)
dimshow(img[:M, :M], ticks=False, **self.imgkwa)
plt.subplot(2, 2, 4)
dimshow(img[:M, -M:], ticks=False, **self.imgkwa)
plt.suptitle('Raw image corners')
ps.savefig()
img, trim_x0, trim_y0 = self.trim_edges(img)
fullH, fullW = img.shape
if self.debug and ps is not None:
plt.clf()
dimshow(img, **self.imgkwa)
plt.title('Trimmed image')
ps.savefig()
M = 200
plt.clf()
plt.subplot(2, 2, 1)
dimshow(img[-M:, :M], ticks=False, **self.imgkwa)
plt.subplot(2, 2, 2)
dimshow(img[-M:, -M:], ticks=False, **self.imgkwa)
plt.subplot(2, 2, 3)
dimshow(img[:M, :M], ticks=False, **self.imgkwa)
plt.subplot(2, 2, 4)
dimshow(img[:M, -M:], ticks=False, **self.imgkwa)
plt.suptitle('Trimmed corners')
ps.savefig()
band = self.get_band(primhdr)
exptime = primhdr['EXPTIME']
airmass = primhdr['AIRMASS']
print('Band', band, 'Exptime', exptime, 'Airmass', airmass)
zp0 = self.nom.zeropoint(band, ext=self.ext)
sky0 = self.nom.sky(band)
kx = self.nom.fiducial_exptime(band).k_co
# Find the sky value and noise level
sky, sig1 = self.get_sky_and_sigma(img)
sky1 = np.median(sky)
skybr = -2.5 * np.log10(sky1 / pixsc / pixsc / exptime) + zp0
print('Sky brightness: %8.3f mag/arcsec^2' % skybr)
print('Fiducial: %8.3f mag/arcsec^2' % sky0)
img -= sky
self.remove_sky_gradients(img)
# Post sky-sub
mn, mx = np.percentile(img.ravel(), [25, 98])
self.imgkwa = dict(vmin=mn, vmax=mx, cmap='gray')
if ps is not None:
plt.clf()
dimshow(img, **self.imgkwa)
plt.title('Sky-sub image: %s-%s' % (os.path.basename(fn).replace(
'.fits', '').replace('.fz', ''), ext))
plt.colorbar()
ps.savefig()
# Read WCS header and compute boresight
wcs = self.get_wcs(hdr)
ra_ccd, dec_ccd = wcs.pixelxy2radec((fullW + 1) / 2., (fullH + 1) / 2.)
# Detect stars
psfsig = self.nominal_fwhm / 2.35
detsn = self.detection_map(img, sig1, psfsig, ps)
slices = self.detect_sources(detsn, self.det_thresh, ps)
print(len(slices), 'sources detected')
if len(slices) < 20:
slices = self.detect_sources(detsn, 10., ps)
print(len(slices), 'sources detected')
ndetected = len(slices)
camera = primhdr.get('INSTRUME', '').strip().lower()
# -> "decam" / "mosaic3"
meas = dict(band=band,
airmass=airmass,
skybright=skybr,
pixscale=pixsc,
primhdr=primhdr,
hdr=hdr,
wcs=wcs,
ra_ccd=ra_ccd,
dec_ccd=dec_ccd,
extension=ext,
camera=camera,
ndetected=ndetected)
if ndetected == 0:
print('NO SOURCES DETECTED')
return meas
xx, yy = [], []
fx, fy = [], []
mx2, my2, mxy = [], [], []
wmx2, wmy2, wmxy = [], [], []
# "Peak" region to centroid
P = momentsize
H, W = img.shape
for i, slc in enumerate(slices):
y0 = slc[0].start
x0 = slc[1].start
subimg = detsn[slc]
imax = np.argmax(subimg)
y, x = np.unravel_index(imax, subimg.shape)
if (x0 + x) < P or (x0 + x) > W - 1 - P or (y0 + y) < P or (
y0 + y) > H - 1 - P:
#print('Skipping edge peak', x0+x, y0+y)
continue
xx.append(x0 + x)
yy.append(y0 + y)
pkarea = detsn[y0 + y - P:y0 + y + P + 1,
x0 + x - P:x0 + x + P + 1]
from scipy.ndimage.measurements import center_of_mass
cy, cx = center_of_mass(pkarea)
#print('Center of mass', cx,cy)
fx.append(x0 + x - P + cx)
fy.append(y0 + y - P + cy)
#print('x,y', x0+x, y0+y, 'vs centroid', x0+x-P+cx, y0+y-P+cy)
### HACK -- measure source ellipticity
# go back to the image (not detection map)
#subimg = img[slc]
subimg = img[y0 + y - P:y0 + y + P + 1,
x0 + x - P:x0 + x + P + 1].copy()
subimg /= subimg.sum()
ph, pw = subimg.shape
px, py = np.meshgrid(np.arange(pw), np.arange(ph))
mx2.append(np.sum(subimg * (px - cx)**2))
my2.append(np.sum(subimg * (py - cy)**2))
mxy.append(np.sum(subimg * (px - cx) * (py - cy)))
# Gaussian windowed version
s = 1.
wimg = subimg * np.exp(-0.5 * ((px - cx)**2 + (py - cy)**2) / s**2)
wimg /= np.sum(wimg)
wmx2.append(np.sum(wimg * (px - cx)**2))
wmy2.append(np.sum(wimg * (py - cy)**2))
wmxy.append(np.sum(wimg * (px - cx) * (py - cy)))
mx2 = np.array(mx2)
my2 = np.array(my2)
mxy = np.array(mxy)
wmx2 = np.array(wmx2)
wmy2 = np.array(wmy2)
wmxy = np.array(wmxy)
# semi-major/minor axes and position angle
theta = np.rad2deg(np.arctan2(2 * mxy, mx2 - my2) / 2.)
theta = np.abs(theta) * np.sign(mxy)
s = np.sqrt(((mx2 - my2) / 2.)**2 + mxy**2)
a = np.sqrt((mx2 + my2) / 2. + s)
b = np.sqrt((mx2 + my2) / 2. - s)
ell = 1. - b / a
wtheta = np.rad2deg(np.arctan2(2 * wmxy, wmx2 - wmy2) / 2.)
wtheta = np.abs(wtheta) * np.sign(wmxy)
ws = np.sqrt(((wmx2 - wmy2) / 2.)**2 + wmxy**2)
wa = np.sqrt((wmx2 + wmy2) / 2. + ws)
wb = np.sqrt((wmx2 + wmy2) / 2. - ws)
well = 1. - wb / wa
fx = np.array(fx)
fy = np.array(fy)
xx = np.array(xx)
yy = np.array(yy)
if ps is not None:
plt.clf()
dimshow(detsn, vmin=-3, vmax=50, cmap='gray')
ax = plt.axis()
plt.plot(fx, fy, 'go', mec='g', mfc='none', ms=10)
plt.colorbar()
plt.title('Detected sources')
plt.axis(ax)
ps.savefig()
# show centroids too
# plt.plot(xx, yy, 'go', mec='g', mfc='none', ms=8)
# plt.axis(ax)
# ps.savefig()
# if ps is not None:
# plt.clf()
# plt.subplot(2,1,1)
# mx = np.percentile(np.append(mx2,my2), 99)
# ha = dict(histtype='step', range=(0,mx), bins=50)
# plt.hist(mx2, color='b', label='mx2', **ha)
# plt.hist(my2, color='r', label='my2', **ha)
# plt.hist(mxy, color='g', label='mxy', **ha)
# plt.legend()
# plt.xlim(0,mx)
# plt.subplot(2,1,2)
# mx = np.percentile(np.append(wmx2,wmy2), 99)
# ha = dict(histtype='step', range=(0,mx), bins=50, lw=3, alpha=0.3)
# plt.hist(wmx2, color='b', label='wx2', **ha)
# plt.hist(wmy2, color='r', label='wy2', **ha)
# plt.hist(wmxy, color='g', label='wxy', **ha)
# plt.legend()
# plt.xlim(0,mx)
# plt.suptitle('Source moments')
# ps.savefig()
#
# #mx = np.percentile(np.abs(np.append(mxy,wmxy)), 99)
# plt.clf()
# plt.subplot(2,1,1)
# ha = dict(histtype='step', range=(0,1), bins=50)
# plt.hist(ell, color='g', label='ell', **ha)
# plt.hist(well, color='g', lw=3, alpha=0.3, label='windowed ell', **ha)
# plt.legend()
# plt.subplot(2,1,2)
# ha = dict(histtype='step', range=(-90,90), bins=50)
# plt.hist(theta, color='g', label='theta', **ha)
# plt.hist(wtheta, color='g', lw=3, alpha=0.3,
# label='windowed theta', **ha)
# plt.xlim(-90,90)
# plt.legend()
# plt.suptitle('Source ellipticities & angles')
# ps.savefig()
# Cut down to stars whose centroids are within 1 pixel of their peaks...
#keep = (np.hypot(fx - xx, fy - yy) < 2)
#print(sum(keep), 'of', len(keep), 'stars have centroids within 2 of peaks')
#print('mean dx', np.mean(fx-xx), 'dy', np.mean(fy-yy), 'pixels')
#assert(float(sum(keep)) / len(keep) > 0.9)
#fx = fx[keep]
#fy = fy[keep]
apxy = np.vstack((fx, fy)).T
ap = []
aprad_pix = self.aprad / pixsc
aper = photutils.CircularAperture(apxy, aprad_pix)
p = photutils.aperture_photometry(img, aper)
apflux = p.field('aperture_sum')
# Manual aperture photometry to get clipped means in sky annulus
sky_inner_r, sky_outer_r = [r / pixsc for r in self.skyrad]
sky = []
for xi, yi in zip(fx, fy):
ix = int(np.round(xi))
iy = int(np.round(yi))
skyR = int(np.ceil(sky_outer_r))
xlo = max(0, ix - skyR)
xhi = min(W, ix + skyR + 1)
ylo = max(0, iy - skyR)
yhi = min(H, iy + skyR + 1)
xx, yy = np.meshgrid(np.arange(xlo, xhi), np.arange(ylo, yhi))
r2 = (xx - xi)**2 + (yy - yi)**2
inannulus = ((r2 >= sky_inner_r**2) * (r2 < sky_outer_r**2))
skypix = img[ylo:yhi, xlo:xhi][inannulus]
#print('ylo,yhi, xlo,xhi', ylo,yhi, xlo,xhi, 'img subshape', img[ylo:yhi, xlo:xhi].shape, 'inann shape', inannulus.shape)
s, nil = sensible_sigmaclip(skypix)
sky.append(s)
sky = np.array(sky)
apflux2 = apflux - sky * (np.pi * aprad_pix**2)
good = (apflux2 > 0) * (apflux > 0)
apflux = apflux[good]
apflux2 = apflux2[good]
fx = fx[good]
fy = fy[good]
# Read in the PS1 catalog, and keep those within 0.25 deg of CCD center
# and those with main sequence colors
pscat = ps1cat(ccdwcs=wcs)
stars = pscat.get_stars()
#print('Got PS1 stars:', len(stars))
# we add the color term later
ps1band = ps1cat.ps1band[band]
stars.mag = stars.median[:, ps1band]
ok, px, py = wcs.radec2pixelxy(stars.ra, stars.dec)
px -= 1
py -= 1
if ps is not None:
#kwa = dict(vmin=-3*sig1, vmax=50*sig1, cmap='gray')
# Add to the 'detected sources' plot
# mn,mx = np.percentile(img.ravel(), [50,99])
# kwa = dict(vmin=mn, vmax=mx, cmap='gray')
# plt.clf()
# dimshow(img, **kwa)
ax = plt.axis()
#plt.plot(fx, fy, 'go', mec='g', mfc='none', ms=10)
K = np.argsort(stars.mag)
plt.plot(px[K[:10]] - trim_x0,
py[K[:10]] - trim_y0,
'o',
mec='m',
mfc='none',
ms=12,
mew=2)
plt.plot(px[K[10:]] - trim_x0,
py[K[10:]] - trim_y0,
'o',
mec='m',
mfc='none',
ms=8)
plt.axis(ax)
plt.title('PS1 stars')
#plt.colorbar()
ps.savefig()
# we trimmed the image before running detection; re-add that margin
fullx = fx + trim_x0
fully = fy + trim_y0
# Match PS1 to our detections, find offset
radius = self.maxshift / pixsc
I, J, dx, dy = self.match_ps1_stars(px, py, fullx, fully, radius,
stars)
print(len(I), 'spatial matches with large radius', self.maxshift,
'arcsec,', radius, 'pix')
bins = 2 * int(np.ceil(radius))
#print('Histogramming with', bins, 'bins')
histo, xe, ye = np.histogram2d(dx,
dy,
bins=bins,
range=((-radius, radius), (-radius,
radius)))
# smooth histogram before finding peak -- fuzzy matching
from scipy.ndimage.filters import gaussian_filter
histo = gaussian_filter(histo, 1.)
histo = histo.T
mx = np.argmax(histo)
my, mx = np.unravel_index(mx, histo.shape)
shiftx = (xe[mx] + xe[mx + 1]) / 2.
shifty = (ye[my] + ye[my + 1]) / 2.
if ps is not None:
plt.clf()
plothist(dx, dy, range=((-radius, radius), (-radius, radius)))
plt.xlabel('dx (pixels)')
plt.ylabel('dy (pixels)')
plt.title('Offsets to PS1 stars')
ax = plt.axis()
plt.axhline(0, color='b')
plt.axvline(0, color='b')
plt.plot(shiftx, shifty, 'o', mec='m', mfc='none', ms=15, mew=3)
plt.axis(ax)
ps.savefig()
# Refine with smaller search radius
radius2 = 3. / pixsc
I, J, dx, dy = self.match_ps1_stars(px, py, fullx + shiftx,
fully + shifty, radius2, stars)
print(len(J), 'matches to PS1 with small radius', 3, 'arcsec')
shiftx2 = np.median(dx)
shifty2 = np.median(dy)
#print('Stage-1 shift', shiftx, shifty)
#print('Stage-2 shift', shiftx2, shifty2)
sx = shiftx + shiftx2
sy = shifty + shifty2
print('Astrometric shift (%.0f, %.0f) pixels' % (sx, sy))
if self.debug and ps is not None:
plt.clf()
plothist(dx, dy, range=((-radius2, radius2), (-radius2, radius2)))
plt.xlabel('dx (pixels)')
plt.ylabel('dy (pixels)')
plt.title('Offsets to PS1 stars')
ax = plt.axis()
plt.axhline(0, color='b')
plt.axvline(0, color='b')
plt.plot(shiftx2, shifty2, 'o', mec='m', mfc='none', ms=15, mew=3)
plt.axis(ax)
ps.savefig()
if ps is not None:
mn, mx = np.percentile(img.ravel(), [50, 99])
kwa2 = dict(vmin=mn, vmax=mx, cmap='gray')
plt.clf()
dimshow(img, **kwa2)
ax = plt.axis()
plt.plot(fx[J], fy[J], 'go', mec='g', mfc='none', ms=10, mew=2)
plt.plot(px[I] - sx - trim_x0,
py[I] - sy - trim_y0,
'm+',
ms=10,
mew=2)
plt.axis(ax)
plt.title('Matched PS1 stars')
plt.colorbar()
ps.savefig()
plt.clf()
dimshow(img, **kwa2)
ax = plt.axis()
plt.plot(fx[J], fy[J], 'go', mec='g', mfc='none', ms=10, mew=2)
K = np.argsort(stars.mag)
plt.plot(px[K[:10]] - sx - trim_x0,
py[K[:10]] - sy - trim_y0,
'o',
mec='m',
mfc='none',
ms=12,
mew=2)
plt.plot(px[K[10:]] - sx - trim_x0,
py[K[10:]] - sy - trim_y0,
'o',
mec='m',
mfc='none',
ms=8,
mew=2)
plt.axis(ax)
plt.title('All PS1 stars')
plt.colorbar()
ps.savefig()
# Now cut to just *stars* with good colors
stars.gicolor = stars.median[:, 0] - stars.median[:, 2]
keep = (stars.gicolor > 0.4) * (stars.gicolor < 2.7)
stars.cut(keep)
if len(stars) == 0:
print('No overlap or too few stars in PS1')
return None
px = px[keep]
py = py[keep]
# Re-match
I, J, dx, dy = self.match_ps1_stars(px, py, fullx + sx, fully + sy,
radius2, stars)
print('Cut to', len(stars), 'PS1 stars with good colors; matched',
len(I))
nmatched = len(I)
meas.update(dx=sx, dy=sy, nmatched=nmatched)
if focus:
meas.update(img=img,
hdr=hdr,
primhdr=primhdr,
fx=fx,
fy=fy,
px=px - trim_x0 - sx,
py=py - trim_y0 - sy,
sig1=sig1,
stars=stars,
moments=(mx2, my2, mxy, theta, a, b, ell),
wmoments=(wmx2, wmy2, wmxy, wtheta, wa, wb, well),
apflux=apflux,
apflux2=apflux2)
return meas
#print('Mean astrometric shift (arcsec): delta-ra=', -np.mean(dy)*0.263, 'delta-dec=', np.mean(dx)*0.263)
# Compute photometric offset compared to PS1
# as the PS1 minus observed mags
colorterm = self.colorterm_ps1_to_observed(stars.median, band)
stars.mag += colorterm
ps1mag = stars.mag[I]
if False and ps is not None:
plt.clf()
plt.semilogy(ps1mag, apflux2[J], 'b.')
plt.xlabel('PS1 mag')
plt.ylabel('DECam ap flux (with sky sub)')
ps.savefig()
plt.clf()
plt.semilogy(ps1mag, apflux[J], 'b.')
plt.xlabel('PS1 mag')
plt.ylabel('DECam ap flux (no sky sub)')
ps.savefig()
apmag2 = -2.5 * np.log10(apflux2) + zp0 + 2.5 * np.log10(exptime)
apmag = -2.5 * np.log10(apflux) + zp0 + 2.5 * np.log10(exptime)
if ps is not None:
plt.clf()
plt.plot(ps1mag, apmag[J], 'b.', label='No sky sub')
plt.plot(ps1mag, apmag2[J], 'r.', label='Sky sub')
# ax = plt.axis()
# mn = min(ax[0], ax[2])
# mx = max(ax[1], ax[3])
# plt.plot([mn,mx], [mn,mx], 'k-', alpha=0.1)
# plt.axis(ax)
plt.xlabel('PS1 mag')
plt.ylabel('DECam ap mag')
plt.legend(loc='upper left')
plt.title('Zeropoint')
ps.savefig()
dm = ps1mag - apmag[J]
dmag, dsig = sensible_sigmaclip(dm, nsigma=2.5)
print('Mag offset: %8.3f' % dmag)
print('Scatter: %8.3f' % dsig)
if not np.isfinite(dmag) or not np.isfinite(dsig):
print('FAILED TO GET ZEROPOINT!')
meas.update(zp=None)
return meas
from scipy.stats import sigmaclip
goodpix, lo, hi = sigmaclip(dm, low=3, high=3)
dmagmed = np.median(goodpix)
print(len(goodpix), 'stars used for zeropoint median')
print('Using median zeropoint:')
zp_med = zp0 + dmagmed
trans_med = 10.**(-0.4 * (zp0 - zp_med - kx * (airmass - 1.)))
print('Zeropoint %6.3f' % zp_med)
print('Transparency: %.3f' % trans_med)
dm = ps1mag - apmag2[J]
dmag2, dsig2 = sensible_sigmaclip(dm, nsigma=2.5)
#print('Sky-sub mag offset', dmag2)
#print('Scatter', dsig2)
if ps is not None:
plt.clf()
plt.plot(ps1mag,
apmag[J] + dmag - ps1mag,
'b.',
label='No sky sub')
plt.plot(ps1mag, apmag2[J] + dmag2 - ps1mag, 'r.', label='Sky sub')
plt.xlabel('PS1 mag')
plt.ylabel('DECam ap mag - PS1 mag')
plt.legend(loc='upper left')
plt.ylim(-0.25, 0.25)
plt.axhline(0, color='k', alpha=0.25)
plt.title('Zeropoint')
ps.savefig()
zp_obs = zp0 + dmag
transparency = 10.**(-0.4 * (zp0 - zp_obs - kx * (airmass - 1.)))
meas.update(zp=zp_obs, transparency=transparency)
print('Zeropoint %6.3f' % zp_obs)
print('Fiducial %6.3f' % zp0)
print('Transparency: %.3f' % transparency)
# print('Using sky-subtracted values:')
# zp_sky = zp0 + dmag2
# trans_sky = 10.**(-0.4 * (zp0 - zp_sky - kx * (airmass - 1.)))
# print('Zeropoint %6.3f' % zp_sky)
# print('Transparency: %.3f' % trans_sky)
fwhms = []
psf_r = 15
if n_fwhm not in [0, None]:
Jf = J[:n_fwhm]
for i, (xi, yi, fluxi) in enumerate(zip(fx[Jf], fy[Jf], apflux[Jf])):
#print('Fitting source', i, 'of', len(Jf))
ix = int(np.round(xi))
iy = int(np.round(yi))
xlo = max(0, ix - psf_r)
xhi = min(W, ix + psf_r + 1)
ylo = max(0, iy - psf_r)
yhi = min(H, iy + psf_r + 1)
xx, yy = np.meshgrid(np.arange(xlo, xhi), np.arange(ylo, yhi))
r2 = (xx - xi)**2 + (yy - yi)**2
keep = (r2 < psf_r**2)
pix = img[ylo:yhi, xlo:xhi].copy()
ie = np.zeros_like(pix)
ie[keep] = 1. / sig1
#print('fitting source at', ix,iy)
#print('number of active pixels:', np.sum(ie > 0), 'shape', ie.shape)
psf = tractor.NCircularGaussianPSF([4.], [1.])
tim = tractor.Image(data=pix, inverr=ie, psf=psf)
src = tractor.PointSource(tractor.PixPos(xi - xlo, yi - ylo),
tractor.Flux(fluxi))
tr = tractor.Tractor([tim], [src])
#print('Posterior before prior:', tr.getLogProb())
src.pos.addGaussianPrior('x', 0., 1.)
#print('Posterior after prior:', tr.getLogProb())
doplot = (i < 5) * (ps is not None)
if doplot:
mod0 = tr.getModelImage(0)
tim.freezeAllBut('psf')
psf.freezeAllBut('sigmas')
# print('Optimizing params:')
# tr.printThawedParams()
#print('Parameter step sizes:', tr.getStepSizes())
optargs = dict(priors=False, shared_params=False)
for step in range(50):
dlnp, x, alpha = tr.optimize(**optargs)
#print('dlnp', dlnp)
#print('src', src)
#print('psf', psf)
if dlnp == 0:
break
# Now fit only the PSF size
tr.freezeParam('catalog')
# print('Optimizing params:')
# tr.printThawedParams()
for step in range(50):
dlnp, x, alpha = tr.optimize(**optargs)
#print('dlnp', dlnp)
#print('src', src)
#print('psf', psf)
if dlnp == 0:
break
fwhms.append(psf.sigmas[0] * 2.35 * pixsc)
if doplot:
mod1 = tr.getModelImage(0)
chi1 = tr.getChiImage(0)
plt.clf()
plt.subplot(2, 2, 1)
plt.title('Image')
dimshow(pix, **self.imgkwa)
plt.subplot(2, 2, 2)
plt.title('Initial model')
dimshow(mod0, **self.imgkwa)
plt.subplot(2, 2, 3)
plt.title('Final model')
dimshow(mod1, **self.imgkwa)
plt.subplot(2, 2, 4)
plt.title('Final chi')
dimshow(chi1, vmin=-10, vmax=10)
plt.suptitle('PSF fit')
ps.savefig()
fwhms = np.array(fwhms)
fwhm = np.median(fwhms)
print('Median FWHM: %.3f' % np.median(fwhms))
meas.update(seeing=fwhm)
if False and ps is not None:
lo, hi = np.percentile(fwhms, [5, 95])
lo -= 0.1
hi += 0.1
plt.clf()
plt.hist(fwhms, 25, range=(lo, hi), histtype='step', color='b')
plt.xlabel('FWHM (arcsec)')
ps.savefig()
if ps is not None:
plt.clf()
for i, (xi, yi) in enumerate(zip(fx[J], fy[J])[:50]):
ix = int(np.round(xi))
iy = int(np.round(yi))
xlo = max(0, ix - psf_r)
xhi = min(W, ix + psf_r + 1)
ylo = max(0, iy - psf_r)
yhi = min(H, iy + psf_r + 1)
pix = img[ylo:yhi, xlo:xhi]
slc = pix[iy - ylo, :].copy()
slc /= np.sum(slc)
p1 = plt.plot(slc, 'b-', alpha=0.2)
slc = pix[:, ix - xlo].copy()
slc /= np.sum(slc)
p2 = plt.plot(slc, 'r-', alpha=0.2)
ph, pw = pix.shape
cx, cy = pw / 2, ph / 2
if i == 0:
xx = np.linspace(0, pw, 300)
dx = xx[1] - xx[0]
sig = fwhm / pixsc / 2.35
yy = np.exp(-0.5 * (xx - cx)**2 / sig**2) # * np.sum(pix)
yy /= (np.sum(yy) * dx)
p3 = plt.plot(xx, yy, 'k-', zorder=20)
#plt.ylim(-0.2, 1.0)
plt.legend([p1[0], p2[0], p3[0]],
['image slice (y)', 'image slice (x)', 'fit'])
plt.title('PSF fit')
ps.savefig()
return meas
def main():
# I read a DESI DR8 target catalog, cut to ELGs, then took a narrow
# r-mag slice around the peak of that distribution;
# desi/target/catalogs/dr8/0.31.1/targets/main/resolve/targets-dr8-hp-1,5,11,50,55,60,83,84,86,89,91,98,119,155,158,174,186,187,190.fits')
# Then took the median g and z mags
# And opened the coadd invvar mags for a random brick in that footprint
# (0701p000) to find the median per-pixel invvars.
r = 23.0
g = 23.4
z = 22.2
# Selecting EXPs, the peak of the shapeexp_r was ~ 0.75".
r_e = 0.75
# Image properties:
giv = 77000.
riv = 27000.
ziv = 8000.
# PSF sizes were roughly equal, 1.16, 1.17, 1.19"
# -> sigma = 1.9 DECam pixels
psf_sigma = 1.9
H, W = 1000, 1000
seed = 42
np.random.seed(seed)
ra, dec = 70., 1.
brick = BrickDuck(ra, dec, 'custom-0700p010')
wcs = wcs_for_brick(brick, W=W, H=H)
bands = 'grz'
tims = []
for band, iv in zip(bands, [giv, riv, ziv]):
img = np.zeros((H, W), np.float32)
tiv = np.zeros_like(img) + iv
s = psf_sigma**2
psf = GaussianMixturePSF(1., 0., 0., s, s, 0.)
twcs = ConstantFitsWcs(wcs)
sky = ConstantSky(0.)
photocal = LinearPhotoCal(1., band=band)
tai = TAITime(None, mjd=55700.)
tim = tractor.Image(data=img,
invvar=tiv,
psf=psf,
wcs=twcs,
sky=sky,
photocal=photocal,
name='fake %s' % band,
time=tai)
tim.skyver = ('1', '1')
tim.psfver = ('1', '1')
tim.plver = '1'
tim.x0 = tim.y0 = 0
tim.subwcs = wcs
tim.psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
tim.galnorm = tim.psfnorm
tim.propid = '2020A-000'
tim.band = band
tim.dq = None
tim.sig1 = 1. / np.sqrt(iv)
tim.psf_sigma = psf_sigma
tim.primhdr = fitsio.FITSHDR()
tims.append(tim)
# Simulated catalog
gflux = NanoMaggies.magToNanomaggies(g)
rflux = NanoMaggies.magToNanomaggies(r)
zflux = NanoMaggies.magToNanomaggies(z)
box = 50
CX, CY = np.meshgrid(np.arange(box // 2, W, box),
np.arange(box // 2, H, box))
ny, nx = CX.shape
BA, PHI = np.meshgrid(np.linspace(0.1, 1.0, nx), np.linspace(0., 180., ny))
cat = []
for cx, cy, ba, phi in zip(CX.ravel(), CY.ravel(), BA.ravel(),
PHI.ravel()):
#print('x,y %.1f,%.1f, ba %.2f, phi %.2f' % (cx, cy, ba, phi))
r, d = wcs.pixelxy2radec(cx + 1, cy + 1)
src = ExpGalaxy(RaDecPos(r, d),
NanoMaggies(order=bands, g=gflux, r=rflux, z=zflux),
EllipseE.fromRAbPhi(r_e, ba, phi))
cat.append(src)
from legacypipe.catalog import prepare_fits_catalog
TT = fits_table()
TT.bx = CX.ravel()
TT.by = CY.ravel()
TT.ba = BA.ravel()
TT.phi = PHI.ravel()
tcat = tractor.Catalog(*cat)
T2 = prepare_fits_catalog(tcat, None, TT, bands, save_invvars=False)
T2.writeto('sim-cat.fits')
tr = Tractor(tims, cat)
for band, tim in zip(bands, tims):
mod = tr.getModelImage(tim)
mod += np.random.standard_normal(
size=tim.shape) * 1. / tim.getInvError()
fitsio.write('sim-%s.fits' % band, mod, clobber=True)
tim.data = mod
ccds = fits_table()
ccds.filter = np.array([f for f in bands])
ccds.ccd_cuts = np.zeros(len(ccds), np.int16)
ccds.imgfn = np.array([tim.name for tim in tims])
ccds.propid = np.array(['2020A-000'] * len(ccds))
ccds.fwhm = np.zeros(len(ccds), np.float32) + psf_sigma * 2.35
ccds.mjd_obs = np.zeros(len(ccds))
ccds.camera = np.array(['fake'] * len(ccds))
survey = FakeLegacySurvey(ccds, tims)
import logging
verbose = False
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
run_brick(None,
survey,
radec=(ra, dec),
width=W,
height=H,
do_calibs=False,
gaia_stars=False,
large_galaxies=False,
tycho_stars=False,
forceall=True,
outliers=False) #, stages=['image_coadds'])
def read_cfht_coadd(imgfn, weightfn, roi=None, radecroi=None, filtermap=None):
'''
Given filenames for CFHT coadd image and weight files, produce
a tractor.Image object.
*roi*: (x0,x1, y0,y1): a region-of-interest in pixel space;
returns the subimage [x0,x1), [y0,y1).
*radecroi*: (ra0, ra1, dec0, dec1): a region-of-interest in RA,Dec space;
returns the subimage bounding the given RA,Dec box [ra0,ra1], [dec0,dec1].
*filtermap*: dict, eg, { 'i.MP9701': 'i' }, to map from the FILTER header keyword to
a standard filter name.
'''
P = pyfits.open(imgfn)
print 'Read', P[0].data.shape, 'image'
img = P[0].data
imgheader = P[0].header
# WCS: the image file has a WCS header
# we should be able to do:
#twcs = tractor.FitsWcs(imgfn)
# ARGH! Memory issues reading the file; HACK: copy header...
f, tempfn = tempfile.mkstemp()
os.close(f)
pyfits.writeto(tempfn, None, header=imgheader, clobber=True)
twcs = tractor.FitsWcs(tempfn)
# Cut down to the region-of-interest, if given.
if roi is not None:
x0, x1, y0, y1 = roi
elif radecroi is not None:
ralo, rahi, declo, dechi = radecroi
xy = [
twcs.positionToPixel(tractor.RaDecPos(r, d))
for r, d in [(ralo, declo), (ralo, dechi), (rahi,
declo), (rahi, dechi)]
]
xy = np.array(xy)
x0, x1 = xy[:, 0].min(), xy[:, 0].max()
y0, y1 = xy[:, 1].min(), xy[:, 1].max()
print 'RA,Dec ROI', ralo, rahi, declo, dechi, 'becomes x,y ROI', x0, x1, y0, y1
# Clip to image size...
H, W = data.shape
x0 = max(0, min(x0, W - 1))
x1 = max(0, min(x1, W))
y0 = max(0, min(y0, H - 1))
y1 = max(0, min(y1, H))
print ' clipped to', x0, x1, y0, y1
else:
H, W = img.shape
x0, x1, y0, y1 = 0, W, 0, H
if roi is not None or radecroi is not None:
# Actually cut the pixels
img = img[y0:y1, x0:x1].copy()
# Also tell the WCS to apply an offset.
twcs.setX0Y0(x0, y0)
print 'Image:', img.shape
# HACK, tell the WCS how big the image is...
# (needed because of the previous HACK, copying the header)
twcs.wcs.set_imagesize(x1 - x0, y1 - y0)
print twcs
# Argh, this doesn't work: the files are .fz compressed
#P = pyfits.open(weightfn)
#weight = P[1].data[y0:y1, x0:x1]
# HACK: use "imcopy" to uncompress to a temp file!
#print 'Writing to temp file', tempfn
cmd = "imcopy '%s[%i:%i,%i:%i]' '!%s'" % (weightfn, x0 + 1, x1, y0 + 1, y1,
tempfn)
print 'running', cmd
os.system(cmd)
P = pyfits.open(tempfn)
weight = P[0].data
print 'Read', weight.shape, 'weight image'
# PSF model: FAKE IT for now
tpsf = tractor.GaussianMixturePSF(np.array([0.9, 0.1]), np.zeros((2, 2)),
np.array([1, 2]))
# SKY level: assume zero
#sky = np.median(img)
#print 'Image median value:', sky
sky = 0.
tsky = tractor.ConstantSky(sky)
# Photometric calibration: the FITS header says:
'''
FILTER = 'r.MP9601' / Filter
PHOTZP = 30.000 / photometric zeropoint
COMMENT AB magnitude = -2.5 * log10(flux) + PHOTZP
COMMENT r.MP9601=r_SDSS-0.024*(g_SDSS-r_SDSS)
'''
# Grab the filter name, and apply the filtermap (if given)
filter = imgheader['FILTER']
if filtermap:
filter = filtermap.get(filter, filter)
zp = imgheader['PHOTZP']
# Simple photocal object
photocal = tractor.MagsPhotoCal(filter, zp)
# For plotting: find the approximate standard deviation
#print 'Median weight:', np.median(weight)
sigma1 = 1. / np.sqrt(np.median(weight))
zr = np.array([-3, 10]) * sigma1 + sky
name = 'CFHT ' + imgheader.get('OBJECT', '')
tim = tractor.Image(data=img,
invvar=weight,
psf=tpsf,
wcs=twcs,
sky=tsky,
photocal=photocal,
name=name,
zr=zr)
tim.extent = [x0, x1, y0, y1]
return tim
('Case 4: Different noise, different PSF', [(1., 2.), (0.5, 4.)]),
]:
print()
print(name)
print()
# We're simulating a single isolated point source
src = tractor.PointSource(tractor.PixPos(W // 2, H // 2),
tractor.Flux(trueflux))
# Produce tractor image objects with noise model, PSF model, etc
tims = []
for noise, psf_size in imageset:
tim = tractor.Image(np.zeros((H, W), np.float32),
inverr=np.ones((H, W), np.float32) * (1. / noise),
psf=tractor.NCircularGaussianPSF([psf_size], [1.]),
photocal=tractor.LinearPhotoCal(1.))
# Create noiseless model image (simulated image)
tr = tractor.Tractor([tim], [src])
mod = tr.getModelImage(0)
tim.data = mod
tims.append(tim)
# First we'll run without any noise added to the images, to get estimates of ideal performance.
# Run the Simultaneous Fitting method
tr = tractor.Tractor(tims, [src])
src.brightness.setParams([0.])
# Freeze the source position -- only fit for flux
src.freezeParam('pos')
# Freeze the image calibration parameters (PSF model, sky background, photometric calibration, etc)
def Deal(scifiles, varfiles, SURVEY='PS1', vb=False):
images = []
bands = []
epochs = []
total_mags = []
for scifile, varfile in zip(scifiles, varfiles):
name = scifile.replace('_sci.fits', '')
if vb:
print " "
print "Making Tractor image from " + name + "_*.fits:"
# Read in sci and wht images. Note assumptions about file format:
sci, invvar, hdr, total_flux = Read_in_data(scifile, varfile, vb)
if total_flux == 0.0:
print "No flux found in image from " + scifile
print "Skipping to next image!"
continue
# Initialize a PSF object (single Gaussian by default), first
# getting FWHM from somewhere. Start with FWHM a little small,
# then refine it:
if SURVEY == 'PS1':
FWHM = lenstractor.PS1_IQ(hdr)
elif SURVEY == 'KIDS':
FWHM = lenstractor.KIDS_IQ(hdr)
else:
Raise("Unrecognised survey %s" % SURVEY)
if vb: print " PSF FWHM =", FWHM, "pixels"
# MAGIC 0.7 shrinkage factor:
psf = Initial_PSF(0.7 * FWHM)
if vb: print psf
# Now get the photometric calibration from the image header.
if SURVEY == 'PS1':
band, photocal = lenstractor.PS1_photocal(hdr)
elif SURVEY == 'KIDS':
band, photocal = lenstractor.KIDS_photocal(hdr)
else:
Raise("Unrecognised survey %s" % SURVEY)
if vb: print photocal
bands.append(band)
if SURVEY == 'PS1':
epochs.append(lenstractor.PS1_epoch(hdr))
elif SURVEY == 'KIDS':
epochs.append(lenstractor.KIDS_epoch(hdr))
# Use photocal to return a total magnitude:
total_mag = photocal.countsToMag(total_flux)
if vb: print "Total brightness of image (mag):", total_mag
total_mags.append(total_mag)
# Set up sky to be varied:
median = np.median(sci[invvar > 0])
sky = tractor.ConstantSky(median)
delta = 0.1 * np.sqrt(1.0 / np.sum(invvar))
assert delta > 0
sky.stepsize = delta
if vb: print sky
# Get WCS from FITS header:
if SURVEY == 'PS1':
wcs = lenstractor.PS1WCS(hdr)
elif SURVEY == 'KIDS':
wcs = lenstractor.KIDSWCS(hdr)
if vb: print wcs
# Make a tractor Image object out of all this stuff, and add it to the array:
images.append(
tractor.Image(data=sci,
invvar=invvar,
name=name,
psf=psf,
wcs=wcs,
sky=sky,
photocal=photocal))
# Figure out the unique band names and epochs:
uniqbands = np.unique(np.array(bands))
if vb:
print " "
print "Read in", len(images), "image datasets"
print " in", len(uniqbands), "bands:", uniqbands
print " at", len(epochs), "epochs"
print " "
return images, np.array(total_mags), np.array(bands)
photcal = tractor.MagsPhotoCal(band, zpt)
csky_level_sig = photcal.brightnessToCounts(sky_level_sig)
## The rms of the noise in ADU.
## noise = galsim.PoissonNoise(rng, sky_level=sky_level_pixel)
## Gaussian approximation for large N.
## noise = galsim.GaussianNoise(rng, sigma=sky_level_sig)
## Rendered in counts.
noise = np.random.normal(loc=csky_level_sig,
scale=np.sqrt(csky_level_sig),
size=(H, W))
tim = tractor.Image(data=np.zeros((H, W), np.float32),
inverr=np.ones((H, W), np.float32),
psf=psf,
wcs=wcs,
photcal=photcal)
## _tr = tractor.Tractor([tim], [src])
## mod = _tr.getModelImage(0)
tim.data = tim.data + noise.data ## + mod.data
tims.append(tim)
cat = tractor.Catalog(src)
##
tr = tractor.Tractor(tims, cat)
# Evaluate likelihood.
def read_wise_level1b(basefn, radecroi=None, radecrad=None, filtermap=None,
nanomaggies=False, mask_gz=False, unc_gz=False,
sipwcs=False, constantInvvar=False,
roi=None,
zrsigs=[-3, 10],
):
if filtermap is None:
filtermap = {}
intfn = basefn + '-int-1b.fits'
maskfn = basefn + '-msk-1b.fits'
if mask_gz:
maskfn = maskfn + '.gz'
uncfn = basefn + '-unc-1b.fits'
if unc_gz:
uncfn = uncfn + '.gz'
logger.debug('intensity image %s' % intfn)
logger.debug('mask image %s' % maskfn)
logger.debug('uncertainty image %s' % uncfn)
if sipwcs:
wcs = Sip(intfn, 0)
twcs = tractor.ConstantFitsWcs(wcs)
else:
twcs = tractor.ConstantFitsWcs(intfn)
# Read enough of the image to get its size
Fint = fitsio.FITS(intfn)
H, W = Fint[0].get_info()['dims']
if radecrad is not None:
r, d, rad = radecrad
x, y = twcs.positionToPixel(tractor.RaDecPos(r, d))
pixrad = rad / (twcs.pixel_scale() / 3600.)
print('Tractor WCS:', twcs)
print('RA,Dec,rad', r, d, rad, 'becomes x,y,pixrad', x, y, pixrad)
roi = (x - pixrad, x + pixrad + 1, y - pixrad, y + pixrad + 1)
if radecroi is not None:
ralo, rahi, declo, dechi = radecroi
xy = [twcs.positionToPixel(tractor.RaDecPos(r, d))
for r, d in [(ralo, declo), (ralo, dechi), (rahi, declo), (rahi, dechi)]]
xy = np.array(xy)
x0, x1 = xy[:, 0].min(), xy[:, 0].max()
y0, y1 = xy[:, 1].min(), xy[:, 1].max()
print('RA,Dec ROI', ralo, rahi, declo, dechi,
'becomes x,y ROI', x0, x1, y0, y1)
roi = (x0, x1 + 1, y0, y1 + 1)
if roi is not None:
x0, x1, y0, y1 = roi
x0 = int(np.floor(x0))
x1 = int(np.ceil(x1))
y0 = int(np.floor(y0))
y1 = int(np.ceil(y1))
roi = (x0, x1, y0, y1)
# Clip to image size...
x0 = np.clip(x0, 0, W)
x1 = np.clip(x1, 0, W)
y0 = np.clip(y0, 0, H)
y1 = np.clip(y1, 0, H)
if x0 == x1 or y0 == y1:
print('ROI is empty')
return None
assert(x0 < x1)
assert(y0 < y1)
#roi = (x0,x1,y0,y1)
twcs.setX0Y0(x0, y0)
else:
x0, x1, y0, y1 = 0, W, 0, H
roi = (x0, x1, y0, y1)
ihdr = Fint[0].read_header()
data = Fint[0][y0:y1, x0:x1]
logger.debug('Read %s intensity' % (str(data.shape)))
band = ihdr['BAND']
F = fitsio.FITS(uncfn)
assert(F[0].get_info()['dims'] == [H, W])
unc = F[0][y0:y1, x0:x1]
F = fitsio.FITS(maskfn)
assert(F[0].get_info()['dims'] == [H, W])
mask = F[0][y0:y1, x0:x1]
# HACK -- circular Gaussian PSF of fixed size...
# in arcsec
fwhms = {1: 6.1, 2: 6.4, 3: 6.5, 4: 12.0}
# -> sigma in pixels
sig = fwhms[band] / 2.35 / twcs.pixel_scale()
# print 'PSF sigma', sig, 'pixels'
tpsf = tractor.NCircularGaussianPSF([sig], [1.])
filter = 'w%i' % band
if filtermap:
filter = filtermap.get(filter, filter)
zp = ihdr['MAGZP']
if nanomaggies:
photocal = tractor.LinearPhotoCal(tractor.NanoMaggies.zeropointToScale(zp),
band=filter)
else:
photocal = tractor.MagsPhotoCal(filter, zp)
# print 'Image median:', np.median(data)
# print 'unc median:', np.median(unc)
sky = np.median(data)
tsky = tractor.ConstantSky(sky)
name = 'WISE ' + ihdr['FRSETID'] + ' W%i' % band
# Mask bits, from
# http://wise2.ipac.caltech.edu/docs/release/allsky/expsup/sec4_4a.html#maskdef
# 0 from static mask: excessively noisy due to high dark current alone
# 1 from static mask: generally noisy [includes bit 0]
# 2 from static mask: dead or very low responsivity
# 3 from static mask: low responsivity or low dark current
# 4 from static mask: high responsivity or high dark current
# 5 from static mask: saturated anywhere in ramp
# 6 from static mask: high, uncertain, or unreliable non-linearity
# 7 from static mask: known broken hardware pixel or excessively noisy responsivity estimate [may include bit 1]
# 8 reserved
# 9 broken pixel or negative slope fit value (downlink value = 32767)
# 10 saturated in sample read 1 (down-link value = 32753)
# 11 saturated in sample read 2 (down-link value = 32754)
# 12 saturated in sample read 3 (down-link value = 32755)
# 13 saturated in sample read 4 (down-link value = 32756)
# 14 saturated in sample read 5 (down-link value = 32757)
# 15 saturated in sample read 6 (down-link value = 32758)
# 16 saturated in sample read 7 (down-link value = 32759)
# 17 saturated in sample read 8 (down-link value = 32760)
# 18 saturated in sample read 9 (down-link value = 32761)
# 19 reserved
# 20 reserved
# 21 new/transient bad pixel from dynamic masking
# 22 reserved
# 23 reserved
# 24 reserved
# 25 reserved
# 26 non-linearity correction unreliable
# 27 contains cosmic-ray or outlier that cannot be classified (from temporal outlier rejection in multi-frame pipeline)
# 28 contains positive or negative spike-outlier
# 29 reserved
# 30 reserved
# 31 not used: sign bit
goodmask = ((mask & sum([1 << bit for bit in [0, 1, 2, 3, 4, 5, 6, 7, 9,
10, 11, 12, 13, 14, 15, 16, 17, 18,
21, 26, 27, 28]])) == 0)
sigma1 = np.median(unc[goodmask])
zr = np.array(zrsigs) * sigma1 + sky
# constant
cinvvar = np.zeros_like(data)
cinvvar[goodmask] = 1. / (sigma1**2)
# varying
vinvvar = np.zeros_like(data)
vinvvar[goodmask] = 1. / (unc[goodmask])**2
bad = np.flatnonzero(np.logical_not(np.isfinite(vinvvar)))
if len(bad):
vinvvar.flat[bad] = 0.
cinvvar.flat[bad] = 0.
data.flat[bad] = sky
if constantInvvar:
invvar = cinvvar
else:
invvar = vinvvar
# avoid NaNs
data[np.logical_not(goodmask)] = sky
mjd = ihdr['MJD_OBS']
time = TAITime(None, mjd=mjd)
tim = tractor.Image(data=data, invvar=invvar, psf=tpsf, wcs=twcs,
sky=tsky, photocal=photocal, time=time, name=name, zr=zr,
domask=False)
tim.extent = [x0, x1, y0, y1]
tim.sigma1 = sigma1
#tim.roi = roi
# FIXME
tim.maskplane = mask
tim.uncplane = unc
tim.goodmask = goodmask
# carry both around for retrofitting
tim.vinvvar = vinvvar
tim.cinvvar = cinvvar
return tim
def read_wise_level3(basefn, radecroi=None, filtermap=None,
nanomaggies=False):
if filtermap is None:
filtermap = {}
intfn = basefn + '-int-3.fits'
uncfn = basefn + '-unc-3.fits'
print('intensity image', intfn)
print('uncertainty image', uncfn)
P = pyfits.open(intfn)
ihdr = P[0].header
data = P[0].data
print('Read', data.shape, 'intensity')
band = ihdr['BAND']
P = pyfits.open(uncfn)
unc = P[0].data
print('Read', unc.shape, 'uncertainty')
''' cov:
BAND = 1 / wavelength band number
WAVELEN = 3.368 / [microns] effective wavelength of band
COADDID = '3342p000_ab41' / atlas-image identifier
MAGZP = 20.5 / [mag] relative photometric zero point
MEDINT = 4.0289044380188 / [DN] median of intensity pixels
'''
''' int:
BUNIT = 'DN ' / image pixel units
CTYPE1 = 'RA---SIN' / Projection type for axis 1
CTYPE2 = 'DEC--SIN' / Projection type for axis 2
CRPIX1 = 2048.000000 / Axis 1 reference pixel at CRVAL1,CRVAL2
CRPIX2 = 2048.000000 / Axis 2 reference pixel at CRVAL1,CRVAL2
CDELT1 = -0.0003819444391411 / Axis 1 scale at CRPIX1,CRPIX2 (deg/pix)
CDELT2 = 0.0003819444391411 / Axis 2 scale at CRPIX1,CRPIX2 (deg/pix)
CROTA2 = 0.000000 / Image twist: +axis2 W of N, J2000.0 (deg)
'''
''' unc:
FILETYPE= '1-sigma uncertainty image' / product description
'''
twcs = tractor.WcslibWcs(intfn)
print('WCS', twcs)
# twcs.debug()
print('pixel scale', twcs.pixel_scale())
# HACK -- circular Gaussian PSF of fixed size...
# in arcsec
fwhms = {1: 6.1, 2: 6.4, 3: 6.5, 4: 12.0}
# -> sigma in pixels
sig = fwhms[band] / 2.35 / twcs.pixel_scale()
print('PSF sigma', sig, 'pixels')
tpsf = tractor.NCircularGaussianPSF([sig], [1.])
if radecroi is not None:
ralo, rahi, declo, dechi = radecroi
xy = [twcs.positionToPixel(tractor.RaDecPos(r, d))
for r, d in [(ralo, declo), (ralo, dechi), (rahi, declo), (rahi, dechi)]]
xy = np.array(xy)
x0, x1 = xy[:, 0].min(), xy[:, 0].max()
y0, y1 = xy[:, 1].min(), xy[:, 1].max()
print('RA,Dec ROI', ralo, rahi, declo, dechi,
'becomes x,y ROI', x0, x1, y0, y1)
# Clip to image size...
H, W = data.shape
x0 = max(0, min(x0, W - 1))
x1 = max(0, min(x1, W))
y0 = max(0, min(y0, H - 1))
y1 = max(0, min(y1, H))
print(' clipped to', x0, x1, y0, y1)
data = data[y0:y1, x0:x1]
unc = unc[y0:y1, x0:x1]
twcs.setX0Y0(x0, y0)
print('Cut data to', data.shape)
else:
H, W = data.shape
x0, x1, y0, y1 = 0, W, 0, H
filt = 'w%i' % band
if filtermap:
filt = filtermap.get(filt, filt)
zp = ihdr['MAGZP']
if nanomaggies:
photocal = tractor.LinearPhotoCal(tractor.NanoMaggies.zeropointToScale(zp),
band=filt)
else:
photocal = tractor.MagsPhotoCal(filt, zp)
print('Image median:', np.median(data))
print('unc median:', np.median(unc))
sky = np.median(data)
tsky = tractor.ConstantSky(sky)
sigma1 = np.median(unc)
zr = np.array([-3, 10]) * sigma1 + sky
name = 'WISE ' + ihdr['COADDID'] + ' W%i' % band
tim = tractor.Image(data=data, invvar=1. / (unc**2), psf=tpsf, wcs=twcs,
sky=tsky, photocal=photocal, name=name, zr=zr,
domask=False)
tim.extent = [x0, x1, y0, y1]
return tim