def wise_psf_plots():
# Plot Aaron's PSF models
for i in range(1, 5):
P = pyfits.open('psf%i.fits' % i)[0].data
print(P.min(), P.max())
P /= P.max()
plt.clf()
plt.imshow(np.log10(np.maximum(P, 1e-8)),
interpolation='nearest', origin='lower', vmax=0.01)
plt.colorbar()
plt.savefig('psf-w%i.png' % i)
plt.clf()
for i, y in enumerate([0, 500, 1000]):
for j, x in enumerate([0, 500, 1000]):
P = pyfits.open('psf-1-%i-%i.fits' % (x, y))[0].data
P /= P.max()
plt.subplot(3, 3, 3 * i + j + 1)
plt.imshow(np.log10(np.maximum(P, 1e-8)),
interpolation='nearest', origin='lower', vmax=0.01)
# plt.colorbar()
plt.savefig('psf-w1-xy.png')
psf = pyfits.open('psf%i.fits' % 1)[0].data
S = psf.shape[0]
# number of Gaussian components
for K in range(1, 6):
w, mu, sig = em_init_params(K, None, None, None)
II = psf.copy()
II /= II.sum()
II = np.maximum(II, 0)
xm, ym = -(S / 2), -(S / 2)
res = em_fit_2d(II, xm, ym, w, mu, sig)
print('em_fit_2d result:', res)
if res != 0:
raise RuntimeError('Failed to fit PSF')
print('w,mu,sig', w, mu, sig)
mypsf = GaussianMixturePSF(w, mu, sig)
mypsf.computeRadius()
#
mypsf.radius = S / 2
mod = mypsf.getPointSourcePatch(0., 0.)
mod = mod.patch
mod /= mod.sum()
plt.clf()
plt.subplot(1, 2, 1)
ima = dict(interpolation='nearest', origin='lower',
vmax=0.01 + np.log10(II.max()))
plt.imshow(np.log10(np.maximum(II, 1e-8)), **ima)
plt.subplot(1, 2, 2)
plt.imshow(np.log10(np.maximum(mod, 1e-8)), **ima)
plt.savefig('psf-k%i.png' % K)
def get_psf_model(band, pixpsf=False, xy=None, positive=True, cache=None):
if cache is None:
cache = {}
if xy is not None:
x,y = xy
gx = np.clip((int(x) / 100) * 100 + 50, 50, 950)
gy = np.clip((int(y) / 100) * 100 + 50, 50, 950)
assert(gx % 100 == 50)
assert(gy % 100 == 50)
xy = (gx, gy)
key = (band, pixpsf, xy, positive)
if key in cache:
return cache[key]
if xy is None:
psf = pyfits.open('wise-psf-w%i-500-500.fits' % band)[0].data
else:
# ASSUME existence of wise-psf/wise-psf-w%i-%i-%i.fits on a grid 50,150,...,950
fn = 'wise-psf/wise-psf-w%i-%03i-%03i.fits' % (band, gx, gy)
psf = pyfits.open(fn)[0].data
if pixpsf:
# print 'Read PSF image:', psf.shape, 'range', psf.min(), psf.max()
if positive:
psf = np.maximum(psf, 0.)
psf = PixelizedPSF(psf)
cache[key] = psf
return psf
S = psf.shape[0]
# number of Gaussian components
K = 3
w, mu, sig = em_init_params(K, None, None, None)
II = psf.copy()
II = np.maximum(II, 0)
II /= II.sum()
xm, ym = -(S / 2), -(S / 2)
res = em_fit_2d(II, xm, ym, w, mu, sig)
if res != 0:
raise RuntimeError('Failed to fit PSF')
print('W1 PSF:')
print(' w', w)
print(' mu', mu)
print(' sigma', sig)
psf = GaussianMixturePSF(w, mu, sig)
psf.computeRadius()
cache[key] = psf
return psf
def _fitParamGrid(self):
# number of MoG mixture components
K = self.K
w,mu,sig = em_init_params(K, None, None, None)
# all MoG fit parameters (we need to make them shaped (ny,nx)
# for spline fitting)
pp = []
# x,y coords at which we will evaluate the PSF.
YY = np.linspace(0, self.H, self.ny)
XX = np.linspace(0, self.W, self.nx)
# fit params at start of this row
px0 = None
for y in YY:
pprow = []
for ix,x in enumerate(XX):
# We start each row with the MoG fit parameters of the start of the
# previous row (to try to make the fit more continuous)
if ix == 0 and px0 is not None:
w,mu,sig = px0
im = self.instantiateAt(x, y)
PS = im.shape[0]
im /= im.sum()
im = np.maximum(im, 0)
xm,ym = -(PS/2), -(PS/2)
em_fit_2d(im, xm, ym, w, mu, sig)
print 'Fit w,mu,sig', w,mu,sig
if ix == 0:
px0 = w,mu,sig
params = np.hstack((w.ravel()[:-1],
mu.ravel(),
sig[:,0,0].ravel(),
sig[:,0,1].ravel(),
sig[:,1,1].ravel())).copy()
pprow.append(params)
pp.append(pprow)
pp = np.array(pp)
self.fitSavedData(pp, XX, YY)
if self.savesplinedata:
self.splinedata = (pp, XX, YY)
def main():
# In LSST meas-deblend (on lsst6):
# python examples/suprimePlot.py --data ~dstn/lsst/ACT-data -v 126969 -c 5 --data-range -100 300 --roi 0 500 0 500 --psf psf.fits --image img.fits --sources srcs.fits
from optparse import OptionParser
import sys
parser = OptionParser(usage=('%prog <img> <psf> <srcs>'))
parser.add_option('-v',
'--verbose',
dest='verbose',
action='count',
default=0,
help='Make more verbose')
opt, args = parser.parse_args()
if len(args) != 3:
parser.print_help()
sys.exit(-1)
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
imgfn, psffn, srcfn = args
pimg = pyfits.open(imgfn)
if len(pimg) != 4:
print('Image must have 3 extensions')
sys.exit(-1)
img = pimg[1].data
mask = pimg[2].data
maskhdr = pimg[2].header
var = pimg[3].data
del pimg
print('var', var.shape)
#print var
print('mask', mask.shape)
#print mask
print('img', img.shape)
#print img
mask = mask.astype(np.int16)
for bit in range(16):
on = ((mask & (1 << bit)) != 0)
print('Bit', bit, 'has', np.sum(on), 'pixels set')
'''
MP_BAD = 0
Bit 0 has 2500 pixels set
MP_SAT = 1
Bit 1 has 5771 pixels set
MP_INTRP= 2
Bit 2 has 11269 pixels set
MP_CR = 3
Bit 3 has 136 pixels set
MP_EDGE = 4
Bit 4 has 11856 pixels set
HIERARCH MP_DETECTED = 5
Bit 5 has 37032 pixels set
'''
print('Mask header:', maskhdr)
maskplanes = {}
print('Mask planes:')
for card in maskhdr.ascardlist():
if not card.key.startswith('MP_'):
continue
print(card.value, card.key)
maskplanes[card.key[3:]] = card.value
print('Variance range:', var.min(), var.max())
print('Image median:', np.median(img.ravel()))
invvar = 1. / var
invvar[var == 0] = 0.
invvar[var < 0] = 0.
sig = np.sqrt(np.median(var))
H, W = img.shape
for k, v in maskplanes.items():
plt.clf()
I = ((mask & (1 << v)) != 0)
rgb = np.zeros((H, W, 3))
clipimg = np.clip((img - (-3. * sig)) / (13. * sig), 0, 1)
cimg = clipimg.copy()
cimg[I] = 1
rgb[:, :, 0] = cimg
cimg = clipimg.copy()
cimg[I] = 0
rgb[:, :, 1] = cimg
rgb[:, :, 2] = cimg
plt.imshow(rgb, interpolation='nearest', origin='lower')
plt.title(k)
plt.savefig('mask-%s.png' % k.lower())
badmask = sum([(1 << maskplanes[k])
for k in ['BAD', 'SAT', 'INTRP', 'CR']])
# HACK -- left EDGE sucks
badmask += (1 << maskplanes['EDGE'])
#badmask = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3)
#badmask |= (1 << 4)
print('Masking out: 0x%x' % badmask)
invvar[(mask & badmask) != 0] = 0.
assert (all(np.isfinite(img.ravel())))
assert (all(np.isfinite(invvar.ravel())))
psf = pyfits.open(psffn)[0].data
print('psf', psf.shape)
psf /= psf.sum()
from tractor.emfit import em_fit_2d
from tractor.fitpsf import em_init_params
# Create Gaussian mixture model PSF approximation.
S = psf.shape[0]
# number of Gaussian components
K = 3
w, mu, sig = em_init_params(K, None, None, None)
II = psf.copy()
II /= II.sum()
# HIDEOUS HACK
II = np.maximum(II, 0)
print('Multi-Gaussian PSF fit...')
xm, ym = -(S / 2), -(S / 2)
em_fit_2d(II, xm, ym, w, mu, sig)
print('w,mu,sig', w, mu, sig)
mypsf = tractor.GaussianMixturePSF(w, mu, sig)
P = mypsf.getPointSourcePatch(S / 2, S / 2)
mn, mx = psf.min(), psf.max()
ima = dict(interpolation='nearest', origin='lower', vmin=mn, vmax=mx)
plt.clf()
plt.subplot(1, 2, 1)
plt.imshow(psf, **ima)
plt.subplot(1, 2, 2)
pimg = np.zeros_like(psf)
P.addTo(pimg)
plt.imshow(pimg, **ima)
plt.savefig('psf.png')
sig = np.sqrt(np.median(var))
plt.clf()
plt.hist(img.ravel(), 100, range=(-3. * sig, 3. * sig))
plt.savefig('imghist.png')
srcs = fits_table(srcfn)
print('Initial:', len(srcs), 'sources')
# Trim sources with x=0 or y=0
srcs = srcs[(srcs.x != 0) * (srcs.y != 0)]
print('Trim on x,y:', len(srcs), 'sources left')
# Zero out nans & infs
for c in ['theta', 'a', 'b']:
I = np.logical_not(np.isfinite(srcs.get(c)))
srcs.get(c)[I] = 0.
# Set sources with flux=NaN to something more sensible...
I = np.logical_not(np.isfinite(srcs.flux))
srcs.flux[I] = 1.
# Sort sources by flux.
srcs = srcs[np.argsort(-srcs.flux)]
# Trim sources that are way outside the image.
margin = 8. * np.maximum(srcs.a, srcs.b)
H, W = img.shape
srcs = srcs[(srcs.x > -margin) * (srcs.y > -margin) *
(srcs.x < (W + margin) * (srcs.y < (H + margin)))]
print('Trim out-of-bounds:', len(srcs), 'sources left')
wcs = tractor.FitsWcs(Sip(imgfn, 1))
#wcs = tractor.NullWCS()
timg = tractor.Image(data=img,
invvar=invvar,
psf=mypsf,
wcs=wcs,
sky=tractor.ConstantSky(0.),
photocal=tractor.NullPhotoCal(),
name='image')
inverr = timg.getInvError()
assert (all(np.isfinite(inverr.ravel())))
tsrcs = []
for s in srcs:
#pos = tractor.PixPos(s.x, s.y)
pos = tractor.RaDecPos(s.ra, s.dec)
if s.a > 0 and s.b > 0:
eflux = tractor.Flux(s.flux / 2.)
dflux = tractor.Flux(s.flux / 2.)
re, ab, phi = s.a, s.b / s.a, 90. - s.theta
eshape = gal.GalaxyShape(re, ab, phi)
dshape = gal.GalaxyShape(re, ab, phi)
print('Fluxes', eflux, dflux)
tsrc = gal.CompositeGalaxy(pos, eflux, eshape, dflux, dshape)
else:
flux = tractor.Flux(s.flux)
print('Flux', flux)
tsrc = tractor.PointSource(pos, flux)
tsrcs.append(tsrc)
chug = tractor.Tractor([timg])
for src in tsrcs:
if chug.getModelPatch(timg, src) is None:
print('Dropping non-overlapping source:', src)
continue
chug.addSource(src)
print('Kept a total of', len(chug.catalog), 'sources')
ima = dict(interpolation='nearest',
origin='lower',
vmin=-3. * sig,
vmax=10. * sig)
chia = dict(interpolation='nearest', origin='lower', vmin=-5., vmax=5.)
plt.clf()
plt.imshow(img, **ima)
plt.colorbar()
plt.savefig('img.png')
plt.clf()
plt.imshow(invvar, interpolation='nearest', origin='lower')
plt.colorbar()
plt.savefig('invvar.png')
mod = chug.getModelImages()[0]
plt.clf()
plt.imshow(mod, **ima)
plt.colorbar()
plt.savefig('mod-0.png')
chi = chug.getChiImage(0)
plt.clf()
plt.imshow(chi, **chia)
plt.colorbar()
plt.savefig('chi-0.png')
for step in range(5):
cat = chug.getCatalog()
for src in cat:
if chug.getModelPatch(timg, src) is None:
print('Dropping non-overlapping source:', src)
chug.removeSource(src)
print('Kept a total of', len(chug.catalog), 'sources')
#cat = chug.getCatalog()
#for i,src in enumerate([]):
#for i,src in enumerate(chug.getCatalog()):
#for i in range(len(cat)):
i = 0
while i < len(cat):
src = cat[i]
#print 'Step', i
#for j,s in enumerate(cat):
# x,y = timg.getWcs().positionToPixel(s, s.getPosition())
# print ' ',
# if j == i:
# print '*',
# print '(%6.1f, %6.1f)'%(x,y), s
print('Optimizing source', i, 'of', len(cat))
x, y = timg.getWcs().positionToPixel(src.getPosition(), src)
print('(%6.1f, %6.1f)' % (x, y), src)
# pre = src.getModelPatch(timg)
s1 = str(src)
print('src1 ', s1)
dlnp1, X, a = chug.optimizeCatalogFluxes(srcs=[src])
s2 = str(src)
dlnp2, X, a = chug.optimizeCatalogAtFixedComplexityStep(srcs=[src],
sky=False)
s3 = str(src)
#post = src.getModelPatch(timg)
print('src1 ', s1)
print('src2 ', s2)
print('src3 ', s3)
print('dlnp', dlnp1, dlnp2)
if chug.getModelPatch(timg, src) is None:
print('After optimizing, no overlap!')
print('Removing source', src)
chug.removeSource(src)
i -= 1
i += 1
# plt.clf()
# plt.subplot(2,2,1)
# img = timg.getImage()
# (x0,x1,y0,y1) = pre.getExtent()
# plt.imshow(img, **ima)
# ax = plt.axis()
# plt.plot([x0,x0,x1,x1,x0], [y0,y1,y1,y0,y0], 'k-', lw=2)
# plt.axis(ax)
# plt.subplot(2,2,3)
# plt.imshow(pre.getImage(), **ima)
# plt.subplot(2,2,4)
# plt.imshow(post.getImage(), **ima)
# plt.savefig('prepost-s%i-s%03i.png' % (step, i))
#
# mod = chug.getModelImages()[0]
# plt.clf()
# plt.imshow(mod, **ima)
# plt.colorbar()
# plt.savefig('mod-s%i-s%03i.png' % (step, i))
# chi = chug.getChiImage(0)
# plt.clf()
# plt.imshow(chi, **chia)
# plt.colorbar()
# plt.savefig('chi-s%i-s%03i.png' % (step, i))
#dlnp,x,a = chug.optimizeCatalogFluxes()
#print 'fluxes: dlnp', dlnp
#dlnp,x,a = chug.optimizeCatalogAtFixedComplexityStep()
#print 'opt: dlnp', dlnp
mod = chug.getModelImages()[0]
plt.clf()
plt.imshow(mod, **ima)
plt.colorbar()
plt.savefig('mod-%i.png' % (step + 1))
chi = chug.getChiImage(0)
plt.clf()
plt.imshow(chi, **chia)
plt.colorbar()
plt.savefig('chi-%i.png' % (step + 1))
return
for step in range(5):
chug.optimizeCatalogFluxes()
mod = chug.getModelImages()[0]
plt.clf()
plt.imshow(mod, **ima)
plt.colorbar()
plt.savefig('mod-s%i.png' % step)
chi = chug.getChiImage(0)
plt.clf()
plt.imshow(chi, **chia)
plt.colorbar()
plt.savefig('chi-s%i.png' % step)
def get_tractor_image_dr8(run, camcol, field, bandname, sdss=None,
roi=None, psf='kl-gm', roiradecsize=None,
roiradecbox=None,
savepsfimg=None, curl=False,
nanomaggies=False,
zrange=[-3,10],
invvarIgnoresSourceFlux=False,
invvarAtCenter=False,
invvarAtCenterImage=False,
imargs={}):
# retry_retrieve=True,
'''
Creates a tractor.Image given an SDSS field identifier.
If not None, roi = (x0, x1, y0, y1) defines a region-of-interest
in the image, in zero-indexed pixel coordinates. x1,y1 are
NON-inclusive; roi=(0,100,0,100) will yield a 100 x 100 image.
psf can be:
"dg" for double-Gaussian
"kl-gm" for SDSS KL-decomposition approximated as a Gaussian mixture
"bright-*", "*" one of the above PSFs, with special handling at
the bright end.
"roiradecsize" = (ra, dec, half-size in pixels) indicates that you
want to grab a ROI around the given RA,Dec.
"roiradecbox" = (ra0, ra1, dec0, dec1) indicates that you
want to grab a ROI containing the given RA,Dec ranges.
"invvarAtCentr" -- get a scalar constant inverse-variance
"invvarAtCenterImage" -- get a scalar constant inverse-variance
but still make an image out of it.
Returns: (tractor.Image, dict)
dict contains useful details like:
'sky'
'skysig'
'''
origpsf = psf
if psf.startswith('bright-'):
psf = psf[7:]
brightpsf = True
print 'Setting bright PSF handling'
else:
brightpsf = False
valid_psf = ['dg', 'kl-gm', 'kl-pix']
if psf not in valid_psf:
raise RuntimeError('PSF must be in ' + str(valid_psf))
if sdss is None:
sdss = DR8(curl=curl)
bandnum = band_index(bandname)
for ft in ['psField', 'fpM']:
fn = sdss.retrieve(ft, run, camcol, field, bandname)
fn = sdss.retrieve('frame', run, camcol, field, bandname)
# http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/frames/RERUN/RUN/CAMCOL/frame.html
frame = sdss.readFrame(run, camcol, field, bandname, filename=fn)
#image = frame.getImage().astype(np.float32)
#(H,W) = image.shape
H,W = frame.getImageShape()
info = dict()
hdr = frame.getHeader()
tai = hdr.get('TAI')
stripe = hdr.get('STRIPE')
strip = hdr.get('STRIP')
obj = hdr.get('OBJECT')
info.update(tai=tai, stripe=stripe, strip=strip, object=obj)
astrans = frame.getAsTrans()
wcs = SdssWcs(astrans)
#print 'Created SDSS Wcs:', wcs
#print '(x,y) = 1,1 -> RA,Dec', wcs.pixelToPosition(1,1)
if roiradecsize is not None:
ra,dec,S = roiradecsize
fxc,fyc = wcs.positionToPixel(RaDecPos(ra,dec))
print 'ROI center RA,Dec (%.3f, %.3f) -> x,y (%.2f, %.2f)' % (ra, dec, fxc, fyc)
xc,yc = [int(np.round(p)) for p in fxc,fyc]
roi = [np.clip(xc-S, 0, W),
np.clip(xc+S, 0, W),
np.clip(yc-S, 0, H),
np.clip(yc+S, 0, H)]
roi = [int(x) for x in roi]
if roi[0]==roi[1] or roi[2]==roi[3]:
print "ZERO ROI?", roi
print 'S = ', S, 'xc,yc = ', xc,yc
#assert(False)
return None,None
#print 'roi', roi
#roi = [max(0, xc-S), min(W, xc+S), max(0, yc-S), min(H, yc+S)]
info.update(roi=roi)
if roiradecbox is not None:
ra0,ra1,dec0,dec1 = roiradecbox
xy = []
for r,d in [(ra0,dec0),(ra1,dec0),(ra0,dec1),(ra1,dec1)]:
xy.append(wcs.positionToPixel(RaDecPos(r,d)))
xy = np.array(xy)
xy = np.round(xy).astype(int)
x0 = xy[:,0].min()
x1 = xy[:,0].max()
y0 = xy[:,1].min()
y1 = xy[:,1].max()
#print 'ROI box RA (%.3f,%.3f), Dec (%.3f,%.3f) -> xy x (%i,%i), y (%i,%i)' % (ra0,ra1, dec0,dec1, x0,x1, y0,y1)
roi = [np.clip(x0, 0, W),
np.clip(x1+1, 0, W),
np.clip(y0, 0, H),
np.clip(y1+1, 0, H)]
#print 'ROI xy box clipped x [%i,%i), y [%i,%i)' % tuple(roi)
if roi[0] == roi[1] or roi[2] == roi[3]:
#print 'Empty roi'
return None,None
info.update(roi=roi)
if roi is not None:
x0,x1,y0,y1 = roi
else:
x0 = y0 = 0
# Mysterious half-pixel shift. asTrans pixel coordinates?
wcs.setX0Y0(x0 + 0.5, y0 + 0.5)
#print 'Band name:', bandname
if nanomaggies:
photocal = LinearPhotoCal(1., band=bandname)
else:
photocal = SdssNanomaggiesPhotoCal(bandname)
sky = 0.
skyobj = ConstantSky(sky)
calibvec = frame.getCalibVec()
invvarAtCenter = invvarAtCenter or invvarAtCenterImage
psfield = sdss.readPsField(run, camcol, field)
iva = dict(ignoreSourceFlux=invvarIgnoresSourceFlux)
if invvarAtCenter:
if roi:
iva.update(constantSkyAt=((x0+x1)/2., (y0+y1)/2.))
else:
iva.update(constantSkyAt=(W/2., H/2.))
invvar = frame.getInvvar(psfield, bandnum, **iva)
invvar = invvar.astype(np.float32)
if not invvarAtCenter:
assert(invvar.shape == (H,W))
# Could get this from photoField instead
# http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/RERUN/RUN/photoField.html
gain = psfield.getGain(bandnum)
darkvar = psfield.getDarkVariance(bandnum)
meansky = np.mean(frame.sky)
meancalib = np.mean(calibvec)
skysig = sqrt((meansky / gain) + darkvar) * meancalib
# Added by @bpartridge to support photon counts
# Calculate the Poisson-distributed number of electrons detected by the instrument
dn = frame.getImage() / frame.getCalibVec() + frame.getSky()
nelec = dn * gain
info.update(sky=sky, skysig=skysig)
zr = np.array(zrange)*skysig + sky
info.update(zr=zr)
# http://data.sdss3.org/datamodel/files/PHOTO_REDUX/RERUN/RUN/objcs/CAMCOL/fpM.html
fpM = sdss.readFpM(run, camcol, field, bandname)
if roi is None:
image = frame.getImage()
else:
roislice = (slice(y0,y1), slice(x0,x1))
image = frame.getImageSlice(roislice).astype(np.float32)
# Added by @bpartridge to support photon counts
dn = dn[roislice].copy()
nelec = nelec[roislice].copy()
if invvarAtCenterImage:
invvar = invvar + np.zeros(image.shape, np.float32)
elif invvarAtCenter:
pass
else:
invvar = invvar[roislice].copy()
H,W = image.shape
# Added by @bpartridge to support photon counts
info.update(dn=dn, nelec=nelec, nmgy=hdr.get('NMGY'))
if (not invvarAtCenter) or invvarAtCenterImage:
for plane in [ 'INTERP', 'SATUR', 'CR', 'GHOST' ]:
fpM.setMaskedPixels(plane, invvar, 0, roi=roi)
if psf == 'kl-pix':
# Pixelized KL-PSF
klpsf = psfield.getPsfAtPoints(bandnum, x0+W/2, y0+H/2)
# Trim symmetric zeros
sh,sw = klpsf.shape
while True:
if (np.all(klpsf[0,:] == 0.) and
np.all(klpsf[:,0] == 0.) and
np.all(klpsf[-1,:] == 0.) and
np.all(klpsf[:,-1] == 0.)):
klpsf = klpsf[1:-1, 1:-1]
else:
break
mypsf = PixelizedPSF(klpsf)
elif psf == 'kl-gm':
from tractor.emfit import em_fit_2d
from tractor.fitpsf import em_init_params
# Create Gaussian mixture model PSF approximation.
klpsf = psfield.getPsfAtPoints(bandnum, x0+W/2, y0+H/2)
S = klpsf.shape[0]
# number of Gaussian components
K = 3
w,mu,sig = em_init_params(K, None, None, None)
II = klpsf.copy()
II /= II.sum()
# HIDEOUS HACK
II = np.maximum(II, 0)
#print 'Multi-Gaussian PSF fit...'
xm,ym = -(S/2), -(S/2)
if savepsfimg is not None:
plt.clf()
plt.imshow(II, interpolation='nearest', origin='lower')
plt.title('PSF image to fit with EM')
plt.savefig(savepsfimg)
res = em_fit_2d(II, xm, ym, w, mu, sig)
#print 'em_fit_2d result:', res
if res == 0:
# print 'w,mu,sig', w,mu,sig
mypsf = GaussianMixturePSF(w, mu, sig)
mypsf.computeRadius()
else:
# Failed! Return 'dg' model instead?
print 'PSF model fit', psf, 'failed! Returning DG model instead'
psf = 'dg'
if psf == 'dg':
dgpsf = psfield.getDoubleGaussian(bandnum)
print 'Creating double-Gaussian PSF approximation'
(a,s1, b,s2) = dgpsf
mypsf = NCircularGaussianPSF([s1, s2], [a, b])
if brightpsf:
print 'Wrapping PSF in SdssBrightPSF'
(a1,s1, a2,s2, a3,sigmap,beta) = psfield.getPowerLaw(bandnum)
mypsf = SdssBrightPSF(mypsf, a1,s1,a2,s2,a3,sigmap,beta)
print 'PSF:', mypsf
timg = Image(data=image, invvar=invvar, psf=mypsf, wcs=wcs,
sky=skyobj, photocal=photocal,
name=('SDSS (r/c/f/b=%i/%i/%i/%s)' %
(run, camcol, field, bandname)),
time=TAITime(tai),
**imargs)
timg.zr = zr
all_data = dict()
all_data['img'] = timg
all_data['counts'] = dn
all_data['info'] = info
filename = 'real_data/data_%d_%d_%d_%s.pkl' % (run, camcol, field, bandname)
output = open(filename, 'wb')
pickle.dump(all_data, output)
return timg,info
def main():
ra = 126.925
dec = 21.4833
itune1 = 5
itune2 = 5
ntune = 2
run = [4517, 4576, 4576]
field = [103, 99, 100]
camcol = [2, 6, 6]
bands = ['r']
bandname = 'r'
flipBands = ['r']
rerun = 0
TI = []
sources = []
table = pyfits.open("J082742.02+212844.7-r.fits")
table.info()
header = table[0].header
data = table[0].data
invvar = table[1].data
skyobj = ba.ConstantSky(header['skyval'])
psffn = 'J082742.02+212844.7-r-bpsf.fits.gz'
psfimg = pyfits.open(psffn)[0].data
print('PSF image shape', psfimg.shape)
# number of Gaussian components
PS = psfimg.shape[0]
K = 3
w, mu, sig = em_init_params(K, None, None, None)
II = psfimg.copy()
II /= II.sum()
# HACK
II = np.maximum(II, 0)
print('Multi-Gaussian PSF fit...')
xm, ym = -(PS / 2), -(PS / 2)
em_fit_2d(II, xm, ym, w, mu, sig)
print('w,mu,sig', w, mu, sig)
psf = GaussianMixturePSF(w, mu, sig)
sources = []
# for run,camcol,field in zip(run,camcol,field):
# sources.append(st.get_tractor_sources(run,camcol,field,bandname,bands=bands))
wcs = Tan("J082742.02+212844.7-r.fits", 0)
wcs = FitsWcs(wcs)
wcs.setX0Y0(1., 1.)
photocal = ba.NasaSloanPhotoCal(bandname) #Also probably not right
TI.append(
en.Image(data=data,
invvar=invvar,
sky=skyobj,
psf=psf,
wcs=wcs,
photocal=photocal,
name="NASA-Sloan Test"))
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
tims = [TI[0]]
tractor = st.SDSSTractor(tims)
for source in sources:
tractor.addSources(source)
zr = np.array([-5., +5.]) # * info['skysig']
print(bands)
prefix = 'ngc2595'
# saveAll('initial-'+prefix, tractor,zr,flipBands,debug=True)
bright = None
lowbright = 1000
sources = []
for timg, sources in zip(tims, sources):
wcs = timg.getWcs()
xtr, ytr = wcs.positionToPixel(RaDecPos(ra, dec))
print(xtr, ytr)
xt = xtr
yt = ytr
r = 250.
for src in sources:
xs, ys = wcs.positionToPixel(src.getPosition(), src)
if (xs - xt)**2 + (ys - yt)**2 <= r**2:
print("Removed:", src)
print(xs, ys)
tractor.removeSource(src)
# saveAll('removed-'+prefix, tractor,zr,flipBands,debug=True)
newShape = sg.GalaxyShape(30., 1., 0.)
newBright = ba.Mags(r=15.0, g=15.0, u=15.0, z=15.0, i=15.0)
EG = st.ExpGalaxy(RaDecPos(ra, dec), newBright, newShape)
print(EG)
tractor.addSource(EG)
saveAll('added-' + prefix, tractor, zr, flipBands, debug=True)
plotInvvar('added-' + prefix, tractor)
for i in range(itune1):
if (i % 5 == 0):
tractor.optimizeCatalogLoop(nsteps=1, srcs=[EG], sky=True)
else:
tractor.optimizeCatalogLoop(nsteps=1, srcs=[EG], sky=False)
tractor.changeInvvar(9.)
tractor.clearCache()
saveAll('itune1-%d-' % (i + 1) + prefix,
tractor,
zr,
flipBands,
debug=True)
plotInvvar('itune1-%d-' % (i + 1) + prefix, tractor)
CGPos = EG.getPosition()
CGShape = EG.getShape()
EGBright = EG.getBrightness()
print(EGBright)
CGg = EGBright[0] * 1.25
CGi = EGBright[1] * 1.25
CGr = EGBright[2] * 1.25
CGu = EGBright[3] * 1.25
CGz = EGBright[4] * 1.25
CGBright = ba.Mags(r=CGr, g=CGg, u=CGu, z=CGz, i=CGi)
print(EGBright)
print(CGBright)
CG = st.CompositeGalaxy(CGPos, CGBright, CGShape, CGBright, CGShape)
tractor.removeSource(EG)
tractor.addSource(CG)
for i in range(itune2):
if (i % 5 == 0):
tractor.optimizeCatalogLoop(nsteps=1, srcs=[CG], sky=True)
else:
tractor.optimizeCatalogLoop(nsteps=1, srcs=[CG], sky=False)
tractor.changeInvvar(9.)
tractor.clearCache()
saveAll('itune2-%d-' % (i + 1) + prefix,
tractor,
zr,
flipBands,
debug=True)
plotInvvar('itune2-%d-' % (i + 1) + prefix, tractor)
for i in range(ntune):
tractor.optimizeCatalogLoop(nsteps=1, sky=True)
saveAll('ntune-%d-' % (i + 1) + prefix,
tractor,
zr,
flipBands,
debug=True)
tractor.clearCache()
makeflipbook(prefix, len(tractor.getImages()), itune1, itune2, ntune)
def main():
ra = 126.925
dec = 21.4833
itune1 = 5
itune2 = 5
ntune = 2
run = [4517,4576,4576]
field = [103,99,100]
camcol = [2,6,6]
bands=['r']
bandname = 'r'
flipBands = ['r']
rerun = 0
TI = []
sources = []
table = pyfits.open("J082742.02+212844.7-r.fits")
table.info()
header = table[0].header
data = table[0].data
invvar=table[1].data
skyobj = ba.ConstantSky(header['skyval'])
psffn = 'J082742.02+212844.7-r-bpsf.fits.gz'
psfimg = pyfits.open(psffn)[0].data
print 'PSF image shape', psfimg.shape
# number of Gaussian components
PS = psfimg.shape[0]
K = 3
w,mu,sig = em_init_params(K, None, None, None)
II = psfimg.copy()
II /= II.sum()
# HACK
II = np.maximum(II, 0)
print 'Multi-Gaussian PSF fit...'
xm,ym = -(PS/2), -(PS/2)
em_fit_2d(II, xm, ym, w, mu, sig)
print 'w,mu,sig', w,mu,sig
psf = GaussianMixturePSF(w, mu, sig)
sources = []
# for run,camcol,field in zip(run,camcol,field):
# sources.append(st.get_tractor_sources(run,camcol,field,bandname,bands=bands))
wcs = Tan("J082742.02+212844.7-r.fits",0)
wcs = FitsWcs(wcs)
wcs.setX0Y0(1.,1.)
photocal = ba.NasaSloanPhotoCal(bandname) #Also probably not right
TI.append(en.Image(data=data,invvar=invvar,sky=skyobj,psf=psf,wcs=wcs,photocal=photocal,name = "NASA-Sloan Test"))
lvl = logging.DEBUG
logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)
tims = [TI[0]]
tractor = st.SDSSTractor(tims)
for source in sources:
tractor.addSources(source)
zr = np.array([-5.,+5.])# * info['skysig']
print bands
prefix = 'ngc2595'
# saveAll('initial-'+prefix, tractor,zr,flipBands,debug=True)
bright = None
lowbright = 1000
sources=[]
for timg,sources in zip(tims,sources):
wcs = timg.getWcs()
xtr,ytr = wcs.positionToPixel(RaDecPos(ra,dec))
print xtr,ytr
xt = xtr
yt = ytr
r = 250.
for src in sources:
xs,ys = wcs.positionToPixel(src.getPosition(),src)
if (xs-xt)**2+(ys-yt)**2 <= r**2:
print "Removed:", src
print xs,ys
tractor.removeSource(src)
# saveAll('removed-'+prefix, tractor,zr,flipBands,debug=True)
newShape = sg.GalaxyShape(30.,1.,0.)
newBright = ba.Mags(r=15.0,g=15.0,u=15.0,z=15.0,i=15.0)
EG = st.ExpGalaxy(RaDecPos(ra,dec),newBright,newShape)
print EG
tractor.addSource(EG)
saveAll('added-'+prefix,tractor,zr,flipBands,debug=True)
plotInvvar('added-'+prefix,tractor)
for i in range(itune1):
if (i % 5 == 0):
tractor.optimizeCatalogLoop(nsteps=1,srcs=[EG],sky=True)
else:
tractor.optimizeCatalogLoop(nsteps=1,srcs=[EG],sky=False)
tractor.changeInvvar(9.)
tractor.clearCache()
saveAll('itune1-%d-' % (i+1)+prefix,tractor,zr,flipBands,debug=True)
plotInvvar('itune1-%d-' % (i+1)+prefix,tractor)
CGPos = EG.getPosition()
CGShape = EG.getShape()
EGBright = EG.getBrightness()
print EGBright
CGg = EGBright[0]*1.25
CGi = EGBright[1]*1.25
CGr = EGBright[2]*1.25
CGu = EGBright[3]*1.25
CGz = EGBright[4]*1.25
CGBright = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi)
print EGBright
print CGBright
CG = st.CompositeGalaxy(CGPos,CGBright,CGShape,CGBright,CGShape)
tractor.removeSource(EG)
tractor.addSource(CG)
for i in range(itune2):
if (i % 5 == 0):
tractor.optimizeCatalogLoop(nsteps=1,srcs=[CG],sky=True)
else:
tractor.optimizeCatalogLoop(nsteps=1,srcs=[CG],sky=False)
tractor.changeInvvar(9.)
tractor.clearCache()
saveAll('itune2-%d-' % (i+1)+prefix,tractor,zr,flipBands,debug=True)
plotInvvar('itune2-%d-' % (i+1)+prefix,tractor)
for i in range(ntune):
tractor.optimizeCatalogLoop(nsteps=1,sky=True)
saveAll('ntune-%d-' % (i+1)+prefix,tractor,zr,flipBands,debug=True)
tractor.clearCache()
makeflipbook(prefix,len(tractor.getImages()),itune1,itune2,ntune)
def _get_tractor_image_dr8(run,
camcol,
field,
bandname,
sdss=None,
roi=None,
psf='kl-gm',
roiradecsize=None,
roiradecbox=None,
savepsfimg=None,
curl=False,
nanomaggies=False,
zrange=[-3, 10],
invvarIgnoresSourceFlux=False,
invvarAtCenter=False,
invvarAtCenterImage=False,
retrieve=True,
imargs={}):
from astrometry.sdss import band_index
origpsf = psf
if psf.startswith('bright-'):
psf = psf[7:]
brightpsf = True
print('Setting bright PSF handling')
else:
brightpsf = False
valid_psf = ['dg', 'kl-gm', 'kl-pix']
if psf not in valid_psf:
raise RuntimeError('PSF must be in ' + str(valid_psf))
if sdss is None:
from astrometry.sdss import DR8
sdss = DR8(curl=curl)
bandnum = band_index(bandname)
if retrieve:
for ft in ['psField', 'fpM']:
fn = sdss.retrieve(ft, run, camcol, field, bandname)
fn = sdss.retrieve('frame', run, camcol, field, bandname)
else:
fn = sdss.getPath('frame', run, camcol, field, bandname)
# http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/frames/RERUN/RUN/CAMCOL/frame.html
frame = sdss.readFrame(run, camcol, field, bandname, filename=fn)
H, W = frame.getImageShape()
info = dict()
hdr = frame.getHeader()
tai = hdr.get('TAI')
stripe = hdr.get('STRIPE')
strip = hdr.get('STRIP')
obj = hdr.get('OBJECT')
info.update(tai=tai, stripe=stripe, strip=strip, object=obj, hdr=hdr)
astrans = frame.getAsTrans()
wcs = SdssWcs(astrans)
#print('Created SDSS Wcs:', wcs)
#print('(x,y) = 1,1 -> RA,Dec', wcs.pixelToPosition(1,1))
X = interpret_roi(wcs, (H, W),
roi=roi,
roiradecsize=roiradecsize,
roiradecbox=roiradecbox)
if X is None:
return None, None
roi, hasroi = X
info.update(roi=roi)
x0, x1, y0, y1 = roi
# Half-pixel shift for asTrans pixel coordinates.
wcs.setX0Y0(x0 + 0.5, y0 + 0.5)
if nanomaggies:
photocal = LinearPhotoCal(1., band=bandname)
else:
photocal = MagsPhotoCal(bandname, 22.5)
sky = 0.
skyobj = ConstantSky(sky)
calibvec = frame.getCalibVec()
invvarAtCenter = invvarAtCenter or invvarAtCenterImage
psfield = sdss.readPsField(run, camcol, field)
iva = dict(ignoreSourceFlux=invvarIgnoresSourceFlux)
if invvarAtCenter:
if hasroi:
iva.update(constantSkyAt=(int((x0 + x1) / 2.),
int((y0 + y1) / 2.)))
else:
iva.update(constantSkyAt=(int(W / 2.), int(H / 2.)))
invvar = frame.getInvvar(psfield, bandnum, **iva)
invvar = invvar.astype(np.float32)
if not invvarAtCenter:
assert (invvar.shape == (H, W))
# Could get this from photoField instead
# http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/RERUN/RUN/photoField.html
gain = psfield.getGain(bandnum)
darkvar = psfield.getDarkVariance(bandnum)
meansky = np.mean(frame.sky)
meancalib = np.mean(calibvec)
skysig = sqrt((meansky / gain) + darkvar) * meancalib
info.update(sky=sky, skysig=skysig)
zr = np.array(zrange) * skysig + sky
info.update(zr=zr)
# http://data.sdss3.org/datamodel/files/PHOTO_REDUX/RERUN/RUN/objcs/CAMCOL/fpM.html
fpM = sdss.readFpM(run, camcol, field, bandname)
if not hasroi:
image = frame.getImage()
else:
roislice = (slice(y0, y1), slice(x0, x1))
image = frame.getImageSlice(roislice).astype(np.float32)
if invvarAtCenterImage:
invvar = invvar + np.zeros(image.shape, np.float32)
elif invvarAtCenter:
pass
else:
invvar = invvar[roislice].copy()
H, W = image.shape
if (not invvarAtCenter) or invvarAtCenterImage:
for plane in ['INTERP', 'SATUR', 'CR', 'GHOST']:
fpM.setMaskedPixels(plane, invvar, 0, roi=roi)
dgpsf = psfield.getDoubleGaussian(bandnum, normalize=True)
info.update(dgpsf=dgpsf)
if psf == 'kl-pix':
# Pixelized KL-PSF
klpsf = psfield.getPsfAtPoints(bandnum, x0 + W / 2, y0 + H / 2)
# Trim symmetric zeros
sh, sw = klpsf.shape
while True:
if (np.all(klpsf[0, :] == 0.) and np.all(klpsf[:, 0] == 0.)
and np.all(klpsf[-1, :] == 0.)
and np.all(klpsf[:, -1] == 0.)):
klpsf = klpsf[1:-1, 1:-1]
else:
break
mypsf = PixelizedPSF(klpsf)
elif psf == 'kl-gm':
from tractor.emfit import em_fit_2d
from tractor.fitpsf import em_init_params
# Create Gaussian mixture model PSF approximation.
klpsf = psfield.getPsfAtPoints(bandnum, x0 + W / 2, y0 + H / 2)
S = klpsf.shape[0]
# number of Gaussian components
K = 3
w, mu, sig = em_init_params(K, None, None, None)
II = klpsf.copy()
II /= II.sum()
# HIDEOUS HACK
II = np.maximum(II, 0)
#print('Multi-Gaussian PSF fit...')
xm, ym = -(S // 2), -(S // 2)
if savepsfimg is not None:
plt.clf()
plt.imshow(II, interpolation='nearest', origin='lower')
plt.title('PSF image to fit with EM')
plt.savefig(savepsfimg)
res = em_fit_2d(II, xm, ym, w, mu, sig)
#print('em_fit_2d result:', res)
if res == 0:
# print('w,mu,sig', w,mu,sig)
mypsf = GaussianMixturePSF(w, mu, sig)
mypsf.computeRadius()
else:
# Failed! Return 'dg' model instead?
print('PSF model fit', psf, 'failed! Returning DG model instead')
psf = 'dg'
if psf == 'dg':
print('Creating double-Gaussian PSF approximation')
(a, s1, b, s2) = dgpsf
mypsf = NCircularGaussianPSF([s1, s2], [a, b])
if brightpsf:
print('Wrapping PSF in SdssBrightPSF')
(a1, s1, a2, s2, a3, sigmap, beta) = psfield.getPowerLaw(bandnum)
mypsf = SdssBrightPSF(mypsf, a1, s1, a2, s2, a3, sigmap, beta)
print('PSF:', mypsf)
timg = Image(data=image,
invvar=invvar,
psf=mypsf,
wcs=wcs,
sky=skyobj,
photocal=photocal,
name=('SDSS (r/c/f/b=%i/%i/%i/%s)' %
(run, camcol, field, bandname)),
time=TAITime(tai),
**imargs)
timg.zr = zr
return timg, info
def get_cfht_img(ra, dec, extent):
# Create CFHT tractor.Image
cffn = 'cr.fits'
psffn = 'psfimg.fits'
wcs = FitsWcs(Tan(cffn, 0))
print 'CFHT WCS', wcs
x,y = wcs.positionToPixel(RaDecPos(ra,dec))
print 'x,y', x,y
cfx,cfy = x,y
cd = wcs.cdAtPixel(x,y)
pixscale = np.sqrt(np.abs(np.linalg.det(cd)))
print 'pixscale', pixscale
S = int(extent / pixscale)
print 'S', S
#cfx,cfy = 734,4352
#S = 200
cfroi = [cfx-S, cfx+S, cfy-S, cfy+S]
x0,x1,y0,y1 = cfroi
wcs.setX0Y0(x0, y0)
# From fit
#wcs.setX0Y0(535.14208988131043, 4153.665639423165)
I = pyfits.open(cffn)[1].data
print 'Img data', I.shape
roislice = (slice(y0,y1), slice(x0,x1))
image = I[roislice]
sky = np.median(image)
print 'Sky', sky
# save for later...
cfsky = sky
skyobj = ConstantSky(sky)
# Third plane in image: variance map.
I = pyfits.open(cffn)[3].data
var = I[roislice]
cfstd = np.sqrt(np.median(var))
## FIXME -- add source photon noise, read noise
# actually the read noise will have already been measured by LSST
phdr = pyfits.open(cffn)[0].header
# e/ADU
gain = phdr.get('GAIN')
# Poisson statistics are on electrons; var = mean
el = np.maximum(0, (image - sky) * gain)
# var in ADU...
srcvar = el / gain**2
invvar = 1./(var + srcvar)
#darkcur = phdr.get('DARKCUR')
#readnoise = phdr.get('RDNOISE')
I = pyfits.open(cffn)[2].data
mask = I[roislice]
invvar[mask > 0] = 0.
del I
del var
psfimg = pyfits.open(psffn)[0].data
print 'PSF image shape', psfimg.shape
from tractor.emfit import em_fit_2d
from tractor.fitpsf import em_init_params
# number of Gaussian components
S = psfimg.shape[0]
K = 3
w,mu,sig = em_init_params(K, None, None, None)
II = psfimg.copy()
II /= II.sum()
# HACK
II = np.maximum(II, 0)
print 'Multi-Gaussian PSF fit...'
xm,ym = -(S/2), -(S/2)
em_fit_2d(II, xm, ym, w, mu, sig)
print 'w,mu,sig', w,mu,sig
psf = GaussianMixturePSF(w, mu, sig)
photocal = cf.CfhtPhotoCal(hdr=phdr,
bandname='r')
cftimg = Image(data=image, invvar=invvar, psf=psf, wcs=wcs,
sky=skyobj, photocal=photocal,
name='CFHT')
return cftimg, cfsky, cfstd
def get_cfht_coadd_image(RA, DEC, S, bandname=None, filtermap=None,
doplots=False, psfK=3, nanomaggies=False):
if filtermap is None:
filtermap = {'i.MP9701': 'i'}
fn = 'cs82data/W4p1m1_i.V2.7A.swarp.cut.fits'
wcs = Tan(fn, 0, 1)
P = pyfits.open(fn)
image = P[0].data
phdr = P[0].header
print 'Image', image.shape
(H,W) = image.shape
OH,OW = H,W
#x,y = np.array([1,W,W,1,1]), np.array([1,1,H,H,1])
#rco,dco = wcs.pixelxy2radec(x, y)
# The coadd image has my ROI roughly in the middle.
# Pixel 1,1 is the high-RA, low-Dec corner.
x,y = wcs.radec2pixelxy(RA, DEC)
x -= 1
y -= 1
print 'Center pix:', x,y
xc,yc = int(x), int(y)
image = image[yc-S: yc+S, xc-S: xc+S]
image = image.copy()
print 'Subimage:', image.shape
twcs = FitsWcs(wcs)
twcs.setX0Y0(xc-S, yc-S)
xs,ys = twcs.positionToPixel(RaDecPos(RA, DEC))
print 'Subimage center pix:', xs,ys
rd = twcs.pixelToPosition(xs, ys)
print 'RA,DEC vs RaDec', RA,DEC, rd
if bandname is None:
# try looking up in filtermap.
filt = phdr['FILTER']
if filt in filtermap:
print 'Mapping filter', filt, 'to', filtermap[filt]
bandname = filtermap[filt]
else:
print 'No mapping found for filter', filt
bandname = filt
zp = float(phdr['MAGZP'])
print 'Zeropoint', zp
if nanomaggies:
photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp), band=bandname)
else:
photocal = MagsPhotoCal(bandname, zp)
print photocal
fn = 'cs82data/W4p1m1_i.V2.7A.swarp.cut.weight.fits'
P = pyfits.open(fn)
weight = P[0].data
weight = weight[yc-S:yc+S, xc-S:xc+S].copy()
print 'Weight', weight.shape
print 'Median', np.median(weight.ravel())
invvar = weight
fn = 'cs82data/W4p1m1_i.V2.7A.swarp.cut.flag.fits'
P = pyfits.open(fn)
flags = P[0].data
flags = flags[yc-S:yc+S, xc-S:xc+S].copy()
print 'Flags', flags.shape
del P
invvar[flags == 1] = 0.
fn = 'cs82data/snap_W4p1m1_i.V2.7A.swarp.cut.fits'
psfim = pyfits.open(fn)[0].data
H,W = psfim.shape
N = 9
assert(((H % N) == 0) and ((W % N) == 0))
# Select which of the NxN PSF images applies to our cutout.
ix = int(N * float(xc) / OW)
iy = int(N * float(yc) / OH)
print 'PSF image number', ix,iy
PW,PH = W/N, H/N
print 'PSF image shape', PW,PH
psfim = psfim[iy*PH: (iy+1)*PH, ix*PW: (ix+1)*PW]
print 'my PSF image shape', PW,PH
psfim = np.maximum(psfim, 0)
psfim /= np.sum(psfim)
K = psfK
w,mu,sig = em_init_params(K, None, None, None)
xm,ym = -(PW/2), -(PH/2)
em_fit_2d(psfim, xm, ym, w, mu, sig)
tpsf = GaussianMixturePSF(w, mu, sig)
tsky = ConstantSky(0.)
obj = phdr['OBJECT'].strip()
tim = Image(data=image, invvar=invvar, psf=tpsf, wcs=twcs, photocal=photocal,
sky=tsky, name='CFHT coadd %s %s' % (obj, bandname))
# set "zr" for plots
sig = 1./np.median(tim.inverr)
tim.zr = np.array([-1., +20.]) * sig
if not doplots:
return tim
psfimpatch = Patch(-(PW/2), -(PH/2), psfim)
# number of Gaussian components
for K in range(1, 4):
w,mu,sig = em_init_params(K, None, None, None)
xm,ym = -(PW/2), -(PH/2)
em_fit_2d(psfim, xm, ym, w, mu, sig)
#print 'w,mu,sig', w,mu,sig
psf = GaussianMixturePSF(w, mu, sig)
patch = psf.getPointSourcePatch(0, 0)
plt.clf()
plt.subplot(1,2,1)
plt.imshow(patch.getImage(), interpolation='nearest', origin='lower')
plt.colorbar()
plt.subplot(1,2,2)
plt.imshow((patch - psfimpatch).getImage(), interpolation='nearest', origin='lower')
plt.colorbar()
plt.savefig('copsf-%i.png' % K)
plt.clf()
plt.imshow(psfim, interpolation='nearest', origin='lower')
plt.colorbar()
plt.savefig('copsf.png')
print 'image min', image.min()
plt.clf()
plt.imshow(image, interpolation='nearest', origin='lower',
vmin=0, vmax=10.)
plt.colorbar()
plt.savefig('coim.png')
plt.clf()
plt.imshow(image, interpolation='nearest', origin='lower',
vmin=0, vmax=3.)
plt.colorbar()
plt.savefig('coim2.png')
plt.clf()
plt.imshow(image, interpolation='nearest', origin='lower',
vmin=0, vmax=1.)
plt.colorbar()
plt.savefig('coim3.png')
plt.clf()
plt.imshow(image, interpolation='nearest', origin='lower',
vmin=0, vmax=0.3)
plt.colorbar()
plt.savefig('coim4.png')
plt.clf()
plt.imshow(image * np.sqrt(invvar), interpolation='nearest', origin='lower',
vmin=-3, vmax=10.)
plt.colorbar()
plt.savefig('cochi.png')
plt.clf()
plt.imshow(weight, interpolation='nearest', origin='lower')
plt.colorbar()
plt.savefig('cowt.png')
plt.clf()
plt.imshow(invvar, interpolation='nearest', origin='lower')
plt.colorbar()
plt.savefig('coiv.png')
plt.clf()
plt.imshow(flags, interpolation='nearest', origin='lower')
plt.colorbar()
plt.savefig('cofl.png')
return tim
def get_cfht_image(fn, psffn, pixscale, RA, DEC, sz, bandname=None,
filtermap=None, rotate=True):
if filtermap is None:
filtermap = {'i.MP9701': 'i'}
wcs = Tan(fn, 0)
x,y = wcs.radec2pixelxy(RA,DEC)
x -= 1
y -= 1
print 'x,y', x,y
S = int(sz / pixscale) / 2
print '(half) S', S
cfx,cfy = int(np.round(x)),int(np.round(y))
P = pyfits.open(fn)
I = P[1].data
print 'Img data', I.shape
H,W = I.shape
cfroi = [np.clip(cfx-S, 0, W),
np.clip(cfx+S, 0, W),
np.clip(cfy-S, 0, H),
np.clip(cfy+S, 0, H)]
x0,x1,y0,y1 = cfroi
roislice = (slice(y0,y1), slice(x0,x1))
image = I[roislice]
sky = np.median(image)
print 'Sky', sky
# save for later...
cfsky = sky
skyobj = ConstantSky(sky)
# Third plane in image: variance map.
I = P[3].data
var = I[roislice]
cfstd = np.sqrt(np.median(var))
# Add source noise...
phdr = P[0].header
# e/ADU
gain = phdr.get('GAIN')
# Poisson statistics are on electrons; var = mean
el = np.maximum(0, (image - sky) * gain)
# var in ADU...
srcvar = el / gain**2
invvar = 1./(var + srcvar)
# Apply mask
# MP_BAD = 0
# MP_SAT = 1
# MP_INTRP= 2
# MP_CR = 3
# MP_EDGE = 4
# HIERARCH MP_DETECTED = 5
# HIERARCH MP_DETECTED_NEGATIVE = 6
I = P[2].data.astype(np.uint16)
#print 'I:', I
#print I.dtype
mask = I[roislice]
#print 'Mask:', mask
hdr = P[2].header
badbits = [hdr.get('MP_%s' % nm) for nm in ['BAD', 'SAT', 'INTRP', 'CR']]
print 'Bad bits:', badbits
badmask = sum([1 << bit for bit in badbits])
#print 'Bad mask:', badmask
#print 'Mask dtype', mask.dtype
invvar[(mask & int(badmask)) > 0] = 0.
del I
del var
psfimg = pyfits.open(psffn)[0].data
print 'PSF image shape', psfimg.shape
# number of Gaussian components
K = 3
PS = psfimg.shape[0]
w,mu,sig = em_init_params(K, None, None, None)
II = psfimg.copy()
II /= II.sum()
# HACK
II = np.maximum(II, 0)
print 'Multi-Gaussian PSF fit...'
xm,ym = -(PS/2), -(PS/2)
em_fit_2d(II, xm, ym, w, mu, sig)
print 'w,mu,sig', w,mu,sig
psf = GaussianMixturePSF(w, mu, sig)
if bandname is None:
# try looking up in filtermap.
filt = phdr['FILTER']
if filt in filtermap:
print 'Mapping filter', filt, 'to', filtermap[filt]
bandname = filtermap[filt]
else:
print 'No mapping found for filter', filt
bandname = flit
photocal = cf.CfhtPhotoCal(hdr=phdr, bandname=bandname)
filename = phdr['FILENAME'].strip()
(H,W) = image.shape
print 'Image shape', W, H
print 'x0,y0', x0,y0
print 'Original WCS:', wcs
rdcorners = [wcs.pixelxy2radec(x+x0,y+y0) for x,y in [(1,1),(W,1),(W,H),(1,H)]]
print 'Original RA,Dec corners:', rdcorners
wcs = crop_wcs(wcs, x0, y0, W, H)
print 'Cropped WCS:', wcs
rdcorners = [wcs.pixelxy2radec(x,y) for x,y in [(1,1),(W,1),(W,H),(1,H)]]
print 'cropped RA,Dec corners:', rdcorners
if rotate:
wcs = rot90_wcs(wcs, W, H)
print 'Rotated WCS:', wcs
rdcorners = [wcs.pixelxy2radec(x,y) for x,y in [(1,1),(H,1),(H,W),(1,W)]]
print 'rotated RA,Dec corners:', rdcorners
print 'rotating images...'
image = np.rot90(image, k=1)
invvar = np.rot90(invvar, k=1)
wcs = FitsWcs(wcs)
cftimg = Image(data=image, invvar=invvar, psf=psf, wcs=wcs,
sky=skyobj, photocal=photocal, name='CFHT %s' % filename)
return cftimg, cfsky, cfstd
def main():
# In LSST meas-deblend (on lsst6):
# python examples/suprimePlot.py --data ~dstn/lsst/ACT-data -v 126969 -c 5 --data-range -100 300 --roi 0 500 0 500 --psf psf.fits --image img.fits --sources srcs.fits
from optparse import OptionParser
import sys
parser = OptionParser(usage=('%prog <img> <psf> <srcs>'))
parser.add_option('-v', '--verbose', dest='verbose', action='count', default=0,
help='Make more verbose')
opt,args = parser.parse_args()
if len(args) != 3:
parser.print_help()
sys.exit(-1)
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
imgfn, psffn, srcfn = args
pimg = pyfits.open(imgfn)
if len(pimg) != 4:
print 'Image must have 3 extensions'
sys.exit(-1)
img = pimg[1].data
mask = pimg[2].data
maskhdr = pimg[2].header
var = pimg[3].data
del pimg
print 'var', var.shape
#print var
print 'mask', mask.shape
#print mask
print 'img', img.shape
#print img
mask = mask.astype(np.int16)
for bit in range(16):
on = ((mask & (1<<bit)) != 0)
print 'Bit', bit, 'has', np.sum(on), 'pixels set'
'''
MP_BAD = 0
Bit 0 has 2500 pixels set
MP_SAT = 1
Bit 1 has 5771 pixels set
MP_INTRP= 2
Bit 2 has 11269 pixels set
MP_CR = 3
Bit 3 has 136 pixels set
MP_EDGE = 4
Bit 4 has 11856 pixels set
HIERARCH MP_DETECTED = 5
Bit 5 has 37032 pixels set
'''
print 'Mask header:', maskhdr
maskplanes = {}
print 'Mask planes:'
for card in maskhdr.ascardlist():
if not card.key.startswith('MP_'):
continue
print card.value, card.key
maskplanes[card.key[3:]] = card.value
print 'Variance range:', var.min(), var.max()
print 'Image median:', np.median(img.ravel())
invvar = 1./var
invvar[var == 0] = 0.
invvar[var < 0] = 0.
sig = np.sqrt(np.median(var))
H,W = img.shape
for k,v in maskplanes.items():
plt.clf()
I = ((mask & (1 << v)) != 0)
rgb = np.zeros((H,W,3))
clipimg = np.clip((img - (-3.*sig)) / (13.*sig), 0, 1)
cimg = clipimg.copy()
cimg[I] = 1
rgb[:,:,0] = cimg
cimg = clipimg.copy()
cimg[I] = 0
rgb[:,:,1] = cimg
rgb[:,:,2] = cimg
plt.imshow(rgb, interpolation='nearest', origin='lower')
plt.title(k)
plt.savefig('mask-%s.png' % k.lower())
badmask = sum([(1 << maskplanes[k]) for k in ['BAD', 'SAT', 'INTRP', 'CR']])
# HACK -- left EDGE sucks
badmask += (1 << maskplanes['EDGE'])
#badmask = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3)
#badmask |= (1 << 4)
print 'Masking out: 0x%x' % badmask
invvar[(mask & badmask) != 0] = 0.
assert(all(np.isfinite(img.ravel())))
assert(all(np.isfinite(invvar.ravel())))
psf = pyfits.open(psffn)[0].data
print 'psf', psf.shape
psf /= psf.sum()
from tractor.emfit import em_fit_2d
from tractor.fitpsf import em_init_params
# Create Gaussian mixture model PSF approximation.
S = psf.shape[0]
# number of Gaussian components
K = 3
w,mu,sig = em_init_params(K, None, None, None)
II = psf.copy()
II /= II.sum()
# HIDEOUS HACK
II = np.maximum(II, 0)
print 'Multi-Gaussian PSF fit...'
xm,ym = -(S/2), -(S/2)
em_fit_2d(II, xm, ym, w, mu, sig)
print 'w,mu,sig', w,mu,sig
mypsf = tractor.GaussianMixturePSF(w, mu, sig)
P = mypsf.getPointSourcePatch(S/2, S/2)
mn,mx = psf.min(), psf.max()
ima = dict(interpolation='nearest', origin='lower',
vmin=mn, vmax=mx)
plt.clf()
plt.subplot(1,2,1)
plt.imshow(psf, **ima)
plt.subplot(1,2,2)
pimg = np.zeros_like(psf)
P.addTo(pimg)
plt.imshow(pimg, **ima)
plt.savefig('psf.png')
sig = np.sqrt(np.median(var))
plt.clf()
plt.hist(img.ravel(), 100, range=(-3.*sig, 3.*sig))
plt.savefig('imghist.png')
srcs = fits_table(srcfn)
print 'Initial:', len(srcs), 'sources'
# Trim sources with x=0 or y=0
srcs = srcs[(srcs.x != 0) * (srcs.y != 0)]
print 'Trim on x,y:', len(srcs), 'sources left'
# Zero out nans & infs
for c in ['theta', 'a', 'b']:
I = np.logical_not(np.isfinite(srcs.get(c)))
srcs.get(c)[I] = 0.
# Set sources with flux=NaN to something more sensible...
I = np.logical_not(np.isfinite(srcs.flux))
srcs.flux[I] = 1.
# Sort sources by flux.
srcs = srcs[np.argsort(-srcs.flux)]
# Trim sources that are way outside the image.
margin = 8. * np.maximum(srcs.a, srcs.b)
H,W = img.shape
srcs = srcs[(srcs.x > -margin) * (srcs.y > -margin) *
(srcs.x < (W+margin) * (srcs.y < (H+margin)))]
print 'Trim out-of-bounds:', len(srcs), 'sources left'
wcs = tractor.FitsWcs(Sip(imgfn, 1))
#wcs = tractor.NullWCS()
timg = tractor.Image(data=img, invvar=invvar, psf=mypsf, wcs=wcs,
sky=tractor.ConstantSky(0.),
photocal=tractor.NullPhotoCal(),
name='image')
inverr = timg.getInvError()
assert(all(np.isfinite(inverr.ravel())))
tsrcs = []
for s in srcs:
#pos = tractor.PixPos(s.x, s.y)
pos = tractor.RaDecPos(s.ra, s.dec)
if s.a > 0 and s.b > 0:
eflux = tractor.Flux(s.flux / 2.)
dflux = tractor.Flux(s.flux / 2.)
re,ab,phi = s.a, s.b/s.a, 90.-s.theta
eshape = gal.GalaxyShape(re,ab,phi)
dshape = gal.GalaxyShape(re,ab,phi)
print 'Fluxes', eflux, dflux
tsrc = gal.CompositeGalaxy(pos, eflux, eshape, dflux, dshape)
else:
flux = tractor.Flux(s.flux)
print 'Flux', flux
tsrc = tractor.PointSource(pos, flux)
tsrcs.append(tsrc)
chug = tractor.Tractor([timg])
for src in tsrcs:
if chug.getModelPatch(timg, src) is None:
print 'Dropping non-overlapping source:', src
continue
chug.addSource(src)
print 'Kept a total of', len(chug.catalog), 'sources'
ima = dict(interpolation='nearest', origin='lower',
vmin=-3.*sig, vmax=10.*sig)
chia = dict(interpolation='nearest', origin='lower',
vmin=-5., vmax=5.)
plt.clf()
plt.imshow(img, **ima)
plt.colorbar()
plt.savefig('img.png')
plt.clf()
plt.imshow(invvar, interpolation='nearest', origin='lower')
plt.colorbar()
plt.savefig('invvar.png')
mod = chug.getModelImages()[0]
plt.clf()
plt.imshow(mod, **ima)
plt.colorbar()
plt.savefig('mod-0.png')
chi = chug.getChiImage(0)
plt.clf()
plt.imshow(chi, **chia)
plt.colorbar()
plt.savefig('chi-0.png')
for step in range(5):
cat = chug.getCatalog()
for src in cat:
if chug.getModelPatch(timg, src) is None:
print 'Dropping non-overlapping source:', src
chug.removeSource(src)
print 'Kept a total of', len(chug.catalog), 'sources'
#cat = chug.getCatalog()
#for i,src in enumerate([]):
#for i,src in enumerate(chug.getCatalog()):
#for i in range(len(cat)):
i = 0
while i < len(cat):
src = cat[i]
#print 'Step', i
#for j,s in enumerate(cat):
# x,y = timg.getWcs().positionToPixel(s, s.getPosition())
# print ' ',
# if j == i:
# print '*',
# print '(%6.1f, %6.1f)'%(x,y), s
print 'Optimizing source', i, 'of', len(cat)
x,y = timg.getWcs().positionToPixel(src.getPosition(), src)
print '(%6.1f, %6.1f)'%(x,y), src
# pre = src.getModelPatch(timg)
s1 = str(src)
print 'src1 ', s1
dlnp1,X,a = chug.optimizeCatalogFluxes(srcs=[src])
s2 = str(src)
dlnp2,X,a = chug.optimizeCatalogAtFixedComplexityStep(srcs=[src], sky=False)
s3 = str(src)
#post = src.getModelPatch(timg)
print 'src1 ', s1
print 'src2 ', s2
print 'src3 ', s3
print 'dlnp', dlnp1, dlnp2
if chug.getModelPatch(timg, src) is None:
print 'After optimizing, no overlap!'
print 'Removing source', src
chug.removeSource(src)
i -= 1
i += 1
# plt.clf()
# plt.subplot(2,2,1)
# img = timg.getImage()
# (x0,x1,y0,y1) = pre.getExtent()
# plt.imshow(img, **ima)
# ax = plt.axis()
# plt.plot([x0,x0,x1,x1,x0], [y0,y1,y1,y0,y0], 'k-', lw=2)
# plt.axis(ax)
# plt.subplot(2,2,3)
# plt.imshow(pre.getImage(), **ima)
# plt.subplot(2,2,4)
# plt.imshow(post.getImage(), **ima)
# plt.savefig('prepost-s%i-s%03i.png' % (step, i))
#
# mod = chug.getModelImages()[0]
# plt.clf()
# plt.imshow(mod, **ima)
# plt.colorbar()
# plt.savefig('mod-s%i-s%03i.png' % (step, i))
# chi = chug.getChiImage(0)
# plt.clf()
# plt.imshow(chi, **chia)
# plt.colorbar()
# plt.savefig('chi-s%i-s%03i.png' % (step, i))
#dlnp,x,a = chug.optimizeCatalogFluxes()
#print 'fluxes: dlnp', dlnp
#dlnp,x,a = chug.optimizeCatalogAtFixedComplexityStep()
#print 'opt: dlnp', dlnp
mod = chug.getModelImages()[0]
plt.clf()
plt.imshow(mod, **ima)
plt.colorbar()
plt.savefig('mod-%i.png' % (step+1))
chi = chug.getChiImage(0)
plt.clf()
plt.imshow(chi, **chia)
plt.colorbar()
plt.savefig('chi-%i.png' % (step+1))
return
for step in range(5):
chug.optimizeCatalogFluxes()
mod = chug.getModelImages()[0]
plt.clf()
plt.imshow(mod, **ima)
plt.colorbar()
plt.savefig('mod-s%i.png' % step)
chi = chug.getChiImage(0)
plt.clf()
plt.imshow(chi, **chia)
plt.colorbar()
plt.savefig('chi-s%i.png' % step)
def fromStamp(stamp,
N=3,
P0=None,
xy0=None,
alpha=0.,
emsteps=1000,
v2=False,
approx=1e-30,
v3=False):
'''
optional P0 = (w,mu,var): initial parameter guess.
w has shape (N,)
mu has shape (N,2)
var (variance) has shape (N,2,2)
optional xy0 = int x0,y0 origin of stamp.
'''
from tractor.emfit import em_fit_2d_reg
from tractor.fitpsf import em_init_params
if P0 is not None:
w, mu, var = P0
else:
w, mu, var = em_init_params(N, None, None, None)
stamp = stamp.copy()
if xy0 is None:
xm, ym = -(stamp.shape[1] // 2), -(stamp.shape[0] // 2)
else:
xm, ym = xy0
if v3:
tpsf = GaussianMixturePSF(w, mu, var)
tim = Image(data=stamp, invvar=1e6 * np.ones_like(stamp), psf=tpsf)
h, w = tim.shape
src = PointSource(PixPos(w // 2, h // 2), Flux(1.))
tr = Tractor([tim], [src])
tr.freezeParam('catalog')
tim.freezeAllBut('psf')
tim.modelMinval = approx
for step in range(20):
dlnp, X, alpha = tr.optimize(shared_params=False)
print('dlnp', dlnp)
if dlnp < 1e-6:
break
return tpsf
elif v2:
from tractor.emfit import em_fit_2d_reg2
print('stamp sum:', np.sum(stamp))
#stamp *= 1000.
ok, skyamp = em_fit_2d_reg2(stamp, xm, ym, w, mu, var, alpha,
emsteps, approx)
# print 'sky amp:', skyamp
# print 'w sum:', sum(w)
tpsf = GaussianMixturePSF(w, mu, var)
return tpsf, skyamp
else:
stamp /= stamp.sum()
stamp = np.maximum(stamp, 0)
em_fit_2d_reg(stamp, xm, ym, w, mu, var, alpha, emsteps)
tpsf = GaussianMixturePSF(w, mu, var)
return tpsf
fn = os.path.join(os.path.dirname(__file__),
'c4d_140818_002108_ooi_z_v1.ext27.psf')
psf = PsfEx(fn, 2048, 4096)
nrounds = 1
psfimg = psf.instantiateAt(100,100)
print 'psfimg sum', psfimg.sum()
print ' ', psfimg.min(), psfimg.max()
plt.clf()
dimshow(np.log10(psfimg + 1e-3))
ps.savefig()
from tractor.fitpsf import em_init_params
N = 3
w,mu,var = em_init_params(N, None, None, None)
print 'w,mu,var', w.shape,mu.shape,var.shape
ph,pw = psfimg.shape
parm = np.hstack([[w[i], mu[i,1] + ph/2, mu[i,0] + pw/2,
var[i,1,1],var[i,0,1],var[i,0,0]]
for i in range(len(w))])
print 'parm', parm
t0 = Time()
for i in range(nrounds):
imx,sky = prep_image(psfimg)
ob = Observation(imx)
mix = GMixEM(ob)
start = GMix(pars=parm)
mix.run_em(start, sky, maxiter=10000)
def get_cfht_img(ra, dec, extent):
# Create CFHT tractor.Image
cffn = 'cr.fits'
psffn = 'psfimg.fits'
wcs = FitsWcs(Tan(cffn, 0))
print('CFHT WCS', wcs)
x, y = wcs.positionToPixel(RaDecPos(ra, dec))
print('x,y', x, y)
cfx, cfy = x, y
cd = wcs.cdAtPixel(x, y)
pixscale = np.sqrt(np.abs(np.linalg.det(cd)))
print('pixscale', pixscale)
S = int(extent / pixscale)
print('S', S)
#cfx,cfy = 734,4352
#S = 200
cfroi = [cfx - S, cfx + S, cfy - S, cfy + S]
x0, x1, y0, y1 = cfroi
wcs.setX0Y0(x0, y0)
# From fit
#wcs.setX0Y0(535.14208988131043, 4153.665639423165)
I = pyfits.open(cffn)[1].data
print('Img data', I.shape)
roislice = (slice(y0, y1), slice(x0, x1))
image = I[roislice]
sky = np.median(image)
print('Sky', sky)
# save for later...
cfsky = sky
skyobj = ConstantSky(sky)
# Third plane in image: variance map.
I = pyfits.open(cffn)[3].data
var = I[roislice]
cfstd = np.sqrt(np.median(var))
## FIXME -- add source photon noise, read noise
# actually the read noise will have already been measured by LSST
phdr = pyfits.open(cffn)[0].header
# e/ADU
gain = phdr.get('GAIN')
# Poisson statistics are on electrons; var = mean
el = np.maximum(0, (image - sky) * gain)
# var in ADU...
srcvar = el / gain**2
invvar = 1. / (var + srcvar)
#darkcur = phdr.get('DARKCUR')
#readnoise = phdr.get('RDNOISE')
I = pyfits.open(cffn)[2].data
mask = I[roislice]
invvar[mask > 0] = 0.
del I
del var
psfimg = pyfits.open(psffn)[0].data
print('PSF image shape', psfimg.shape)
from tractor.emfit import em_fit_2d
from tractor.fitpsf import em_init_params
# number of Gaussian components
S = psfimg.shape[0]
K = 3
w, mu, sig = em_init_params(K, None, None, None)
II = psfimg.copy()
II /= II.sum()
# HACK
II = np.maximum(II, 0)
print('Multi-Gaussian PSF fit...')
xm, ym = -(S / 2), -(S / 2)
em_fit_2d(II, xm, ym, w, mu, sig)
print('w,mu,sig', w, mu, sig)
psf = GaussianMixturePSF(w, mu, sig)
photocal = cf.CfhtPhotoCal(hdr=phdr, bandname='r')
cftimg = Image(data=image,
invvar=invvar,
psf=psf,
wcs=wcs,
sky=skyobj,
photocal=photocal,
name='CFHT')
return cftimg, cfsky, cfstd
fn = os.path.join(os.path.dirname(__file__),
'c4d_140818_002108_ooi_z_v1.ext27.psf')
psf = PsfEx(fn, 2048, 4096)
nrounds = 1
psfimg = psf.instantiateAt(100, 100)
print('psfimg sum', psfimg.sum())
print(' ', psfimg.min(), psfimg.max())
plt.clf()
dimshow(np.log10(psfimg + 1e-3))
ps.savefig()
from tractor.fitpsf import em_init_params
N = 3
w, mu, var = em_init_params(N, None, None, None)
print('w,mu,var', w.shape, mu.shape, var.shape)
ph, pw = psfimg.shape
parm = np.hstack([[
w[i], mu[i, 1] + ph / 2, mu[i, 0] + pw / 2, var[i, 1, 1], var[i, 0, 1],
var[i, 0, 0]
] for i in range(len(w))])
print('parm', parm)
t0 = Time()
for i in range(nrounds):
imx, sky = prep_image(psfimg)
ob = Observation(imx)
mix = GMixEM(ob)
start = GMix(pars=parm)
def get_tractor_image_dr8(run,
camcol,
field,
bandname,
sdss=None,
roi=None,
psf='kl-gm',
roiradecsize=None,
roiradecbox=None,
savepsfimg=None,
curl=False,
nanomaggies=False,
zrange=[-3, 10],
invvarIgnoresSourceFlux=False,
invvarAtCenter=False,
invvarAtCenterImage=False,
imargs={}):
# retry_retrieve=True,
'''
Creates a tractor.Image given an SDSS field identifier.
If not None, roi = (x0, x1, y0, y1) defines a region-of-interest
in the image, in zero-indexed pixel coordinates. x1,y1 are
NON-inclusive; roi=(0,100,0,100) will yield a 100 x 100 image.
psf can be:
"dg" for double-Gaussian
"kl-gm" for SDSS KL-decomposition approximated as a Gaussian mixture
"bright-*", "*" one of the above PSFs, with special handling at
the bright end.
"roiradecsize" = (ra, dec, half-size in pixels) indicates that you
want to grab a ROI around the given RA,Dec.
"roiradecbox" = (ra0, ra1, dec0, dec1) indicates that you
want to grab a ROI containing the given RA,Dec ranges.
"invvarAtCentr" -- get a scalar constant inverse-variance
"invvarAtCenterImage" -- get a scalar constant inverse-variance
but still make an image out of it.
Returns: (tractor.Image, dict)
dict contains useful details like:
'sky'
'skysig'
'''
origpsf = psf
if psf.startswith('bright-'):
psf = psf[7:]
brightpsf = True
print 'Setting bright PSF handling'
else:
brightpsf = False
valid_psf = ['dg', 'kl-gm', 'kl-pix']
if psf not in valid_psf:
raise RuntimeError('PSF must be in ' + str(valid_psf))
if sdss is None:
sdss = DR8(curl=curl)
bandnum = band_index(bandname)
for ft in ['psField', 'fpM']:
fn = sdss.retrieve(ft, run, camcol, field, bandname)
fn = sdss.retrieve('frame', run, camcol, field, bandname)
# http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/frames/RERUN/RUN/CAMCOL/frame.html
frame = sdss.readFrame(run, camcol, field, bandname, filename=fn)
#image = frame.getImage().astype(np.float32)
#(H,W) = image.shape
H, W = frame.getImageShape()
info = dict()
hdr = frame.getHeader()
tai = hdr.get('TAI')
stripe = hdr.get('STRIPE')
strip = hdr.get('STRIP')
obj = hdr.get('OBJECT')
info.update(tai=tai, stripe=stripe, strip=strip, object=obj)
astrans = frame.getAsTrans()
wcs = SdssWcs(astrans)
#print 'Created SDSS Wcs:', wcs
#print '(x,y) = 1,1 -> RA,Dec', wcs.pixelToPosition(1,1)
if roiradecsize is not None:
ra, dec, S = roiradecsize
fxc, fyc = wcs.positionToPixel(RaDecPos(ra, dec))
print 'ROI center RA,Dec (%.3f, %.3f) -> x,y (%.2f, %.2f)' % (ra, dec,
fxc, fyc)
xc, yc = [int(np.round(p)) for p in fxc, fyc]
roi = [
np.clip(xc - S, 0, W),
np.clip(xc + S, 0, W),
np.clip(yc - S, 0, H),
np.clip(yc + S, 0, H)
]
roi = [int(x) for x in roi]
if roi[0] == roi[1] or roi[2] == roi[3]:
print "ZERO ROI?", roi
print 'S = ', S, 'xc,yc = ', xc, yc
#assert(False)
return None, None
#print 'roi', roi
#roi = [max(0, xc-S), min(W, xc+S), max(0, yc-S), min(H, yc+S)]
info.update(roi=roi)
if roiradecbox is not None:
ra0, ra1, dec0, dec1 = roiradecbox
xy = []
for r, d in [(ra0, dec0), (ra1, dec0), (ra0, dec1), (ra1, dec1)]:
xy.append(wcs.positionToPixel(RaDecPos(r, d)))
xy = np.array(xy)
xy = np.round(xy).astype(int)
x0 = xy[:, 0].min()
x1 = xy[:, 0].max()
y0 = xy[:, 1].min()
y1 = xy[:, 1].max()
#print 'ROI box RA (%.3f,%.3f), Dec (%.3f,%.3f) -> xy x (%i,%i), y (%i,%i)' % (ra0,ra1, dec0,dec1, x0,x1, y0,y1)
roi = [
np.clip(x0, 0, W),
np.clip(x1 + 1, 0, W),
np.clip(y0, 0, H),
np.clip(y1 + 1, 0, H)
]
#print 'ROI xy box clipped x [%i,%i), y [%i,%i)' % tuple(roi)
if roi[0] == roi[1] or roi[2] == roi[3]:
#print 'Empty roi'
return None, None
info.update(roi=roi)
if roi is not None:
x0, x1, y0, y1 = roi
else:
x0 = y0 = 0
# Mysterious half-pixel shift. asTrans pixel coordinates?
wcs.setX0Y0(x0 + 0.5, y0 + 0.5)
#print 'Band name:', bandname
if nanomaggies:
photocal = LinearPhotoCal(1., band=bandname)
else:
photocal = SdssNanomaggiesPhotoCal(bandname)
sky = 0.
skyobj = ConstantSky(sky)
calibvec = frame.getCalibVec()
invvarAtCenter = invvarAtCenter or invvarAtCenterImage
psfield = sdss.readPsField(run, camcol, field)
iva = dict(ignoreSourceFlux=invvarIgnoresSourceFlux)
if invvarAtCenter:
if roi:
iva.update(constantSkyAt=((x0 + x1) / 2., (y0 + y1) / 2.))
else:
iva.update(constantSkyAt=(W / 2., H / 2.))
invvar = frame.getInvvar(psfield, bandnum, **iva)
invvar = invvar.astype(np.float32)
if not invvarAtCenter:
assert (invvar.shape == (H, W))
# Could get this from photoField instead
# http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/RERUN/RUN/photoField.html
gain = psfield.getGain(bandnum)
darkvar = psfield.getDarkVariance(bandnum)
meansky = np.mean(frame.sky)
meancalib = np.mean(calibvec)
skysig = sqrt((meansky / gain) + darkvar) * meancalib
# Added by @bpartridge to support photon counts
# Calculate the Poisson-distributed number of electrons detected by the instrument
dn = frame.getImage() / frame.getCalibVec() + frame.getSky()
nelec = dn * gain
info.update(sky=sky, skysig=skysig)
zr = np.array(zrange) * skysig + sky
info.update(zr=zr)
# http://data.sdss3.org/datamodel/files/PHOTO_REDUX/RERUN/RUN/objcs/CAMCOL/fpM.html
fpM = sdss.readFpM(run, camcol, field, bandname)
if roi is None:
image = frame.getImage()
else:
roislice = (slice(y0, y1), slice(x0, x1))
image = frame.getImageSlice(roislice).astype(np.float32)
# Added by @bpartridge to support photon counts
dn = dn[roislice].copy()
nelec = nelec[roislice].copy()
if invvarAtCenterImage:
invvar = invvar + np.zeros(image.shape, np.float32)
elif invvarAtCenter:
pass
else:
invvar = invvar[roislice].copy()
H, W = image.shape
# Added by @bpartridge to support photon counts
info.update(dn=dn, nelec=nelec, nmgy=hdr.get('NMGY'))
if (not invvarAtCenter) or invvarAtCenterImage:
for plane in ['INTERP', 'SATUR', 'CR', 'GHOST']:
fpM.setMaskedPixels(plane, invvar, 0, roi=roi)
if psf == 'kl-pix':
# Pixelized KL-PSF
klpsf = psfield.getPsfAtPoints(bandnum, x0 + W / 2, y0 + H / 2)
# Trim symmetric zeros
sh, sw = klpsf.shape
while True:
if (np.all(klpsf[0, :] == 0.) and np.all(klpsf[:, 0] == 0.)
and np.all(klpsf[-1, :] == 0.)
and np.all(klpsf[:, -1] == 0.)):
klpsf = klpsf[1:-1, 1:-1]
else:
break
mypsf = PixelizedPSF(klpsf)
elif psf == 'kl-gm':
from tractor.emfit import em_fit_2d
from tractor.fitpsf import em_init_params
# Create Gaussian mixture model PSF approximation.
klpsf = psfield.getPsfAtPoints(bandnum, x0 + W / 2, y0 + H / 2)
S = klpsf.shape[0]
# number of Gaussian components
K = 3
w, mu, sig = em_init_params(K, None, None, None)
II = klpsf.copy()
II /= II.sum()
# HIDEOUS HACK
II = np.maximum(II, 0)
#print 'Multi-Gaussian PSF fit...'
xm, ym = -(S / 2), -(S / 2)
if savepsfimg is not None:
plt.clf()
plt.imshow(II, interpolation='nearest', origin='lower')
plt.title('PSF image to fit with EM')
plt.savefig(savepsfimg)
res = em_fit_2d(II, xm, ym, w, mu, sig)
#print 'em_fit_2d result:', res
if res == 0:
# print 'w,mu,sig', w,mu,sig
mypsf = GaussianMixturePSF(w, mu, sig)
mypsf.computeRadius()
else:
# Failed! Return 'dg' model instead?
print 'PSF model fit', psf, 'failed! Returning DG model instead'
psf = 'dg'
if psf == 'dg':
dgpsf = psfield.getDoubleGaussian(bandnum)
print 'Creating double-Gaussian PSF approximation'
(a, s1, b, s2) = dgpsf
mypsf = NCircularGaussianPSF([s1, s2], [a, b])
if brightpsf:
print 'Wrapping PSF in SdssBrightPSF'
(a1, s1, a2, s2, a3, sigmap, beta) = psfield.getPowerLaw(bandnum)
mypsf = SdssBrightPSF(mypsf, a1, s1, a2, s2, a3, sigmap, beta)
print 'PSF:', mypsf
timg = Image(data=image,
invvar=invvar,
psf=mypsf,
wcs=wcs,
sky=skyobj,
photocal=photocal,
name=('SDSS (r/c/f/b=%i/%i/%i/%s)' %
(run, camcol, field, bandname)),
time=TAITime(tai),
**imargs)
timg.zr = zr
all_data = dict()
all_data['img'] = timg
all_data['counts'] = dn
all_data['info'] = info
filename = 'real_data/data_%d_%d_%d_%s.pkl' % (run, camcol, field,
bandname)
output = open(filename, 'wb')
pickle.dump(all_data, output)
return timg, info
def forced2():
from bigboss_test import radecroi
ps = PlotSequence('forced')
basedir = os.environ.get('BIGBOSS_DATA', '/project/projectdirs/bigboss')
wisedatadir = os.path.join(basedir, 'data', 'wise')
l1bdir = os.path.join(wisedatadir, 'level1b')
wisecat = fits_table(os.path.join(
wisedatadir, 'catalogs', 'wisecat2.fits'))
# CAS PhotoObjAll.resolveStatus bits
sprim = 0x100
#sbad = 0x800
sedge = 0x1000
sbest = 0x200
(ra0, ra1, dec0, dec1) = radecroi
ra = (ra0 + ra1) / 2.
dec = (dec0 + dec1) / 2.
cas = fits_table('sdss-cas-testarea-3.fits')
print('Read', len(cas), 'CAS sources')
cas.cut((cas.resolvestatus & sedge) == 0)
print('Cut to ', len(cas), 'without SURVEY_EDGE set')
# Drop "sbest" sources that have an "sprim" nearby.
Ibest = (cas.resolvestatus & (sprim | sbest)) == sbest
Iprim = (cas.resolvestatus & (sprim | sbest)) == sprim
I, J, d = match_radec(
cas.ra[Ibest], cas.dec[Ibest], cas.ra[Iprim], cas.dec[Iprim], 2. / 3600.)
Ibest[np.flatnonzero(Ibest)[I]] = False
#Ikeep = np.ones(len(Ibest), bool)
#Ikeep[I] = False
cas.cut(np.logical_or(Ibest, Iprim))
print('Cut to', len(cas), 'PRIMARY + BEST-not-near-PRIMARY')
I, J, d = match_radec(cas.ra, cas.dec, cas.ra,
cas.dec, 2. / 3600., notself=True)
plt.clf()
loghist((cas.ra[I] - cas.ra[J]) * 3600.,
(cas.dec[I] - cas.dec[J]) * 3600., 200)
plt.title('CAS self-matches')
ps.savefig()
psf = pyfits.open('wise-psf-w1-500-500.fits')[0].data
S = psf.shape[0]
# number of Gaussian components
K = 3
w, mu, sig = em_init_params(K, None, None, None)
II = psf.copy()
II = np.maximum(II, 0)
II /= II.sum()
xm, ym = -(S / 2), -(S / 2)
res = em_fit_2d(II, xm, ym, w, mu, sig)
if res != 0:
raise RuntimeError('Failed to fit PSF')
print('W1 PSF:')
print(' w', w)
print(' mu', mu)
print(' sigma', sig)
w1psf = GaussianMixturePSF(w, mu, sig)
w1psf.computeRadius()
print('PSF radius:', w1psf.getRadius(), 'pixels')
T = fits_table('wise-roi.fits')
for i in range(len(T)):
basefn = os.path.join(l1bdir, '%s%i-w1' %
(T.scan_id[i], T.frame_num[i]))
fn = basefn + '-int-1b.fits'
print('Looking for', fn)
if not os.path.exists(fn):
continue
print(' -> Found it!')
tim = read_wise_image(basefn, nanomaggies=True)
tim.psf = w1psf
wcs = tim.wcs.wcs
r0, r1, d0, d1 = wcs.radec_bounds()
print('RA,Dec bounds:', r0, r1, d0, d1)
w, h = wcs.imagew, wcs.imageh
rd = np.array([wcs.pixelxy2radec(x, y) for x, y in
[(1, 1), (w, 1), (w, h), (1, h), (1, 1)]])
I = np.flatnonzero((cas.ra > r0) * (cas.ra < r1) *
(cas.dec > d0) * (cas.dec < d1))
J = point_in_poly(cas.ra[I], cas.dec[I], rd)
I = I[J]
cashere = cas[I]
# 10-20k sources...
wbands = ['w1']
sdssband = 'i'
tsrcs = get_tractor_sources_cas_dr9(cashere, nanomaggies=True,
bandname=sdssband, bands=[
sdssband],
extrabands=wbands)
#keepsrcs = []
for src in tsrcs:
for br in src.getBrightnesses():
f = br.getBand(sdssband)
# if f < 0:
# continue
for wb in wbands:
br.setBand(wb, f)
# keepsrcs.append(src)
#tsrcs = keepsrcs
print('Created', len(tsrcs), 'tractor sources in this image')
I = np.flatnonzero((wisecat.ra > r0) * (wisecat.ra < r1) *
(wisecat.dec > d0) * (wisecat.dec < d1))
J = point_in_poly(wisecat.ra[I], wisecat.dec[I], rd)
I = I[J]
print('Found', len(I), 'WISE catalog sources in this image')
wc = wisecat[I]
tra = np.array([src.getPosition().ra for src in tsrcs])
tdec = np.array([src.getPosition().dec for src in tsrcs])
R = 4.
I, J, d = match_radec(wc.ra, wc.dec, tra, tdec,
R / 3600., nearest=True)
# cashere.ra, cashere.dec,
print('Found', len(I), 'SDSS-WISE matches within', R, 'arcsec')
for i, j in zip(I, J):
w1 = wc.w1mpro[i]
w1 = NanoMaggies.magToNanomaggies(w1)
bb = tsrcs[j].getBrightnesses()
for b in bb:
b.setBand('w1', w1 / float(len(bb)))
keepsrcs = []
for src in tsrcs:
# for b in src.getBrightness():
b = src.getBrightness()
if b.getBand(sdssband) > 0 or b.getBand(wbands[0]) > 0:
keepsrcs.append(src)
tsrcs = keepsrcs
print('Keeping', len(tsrcs), 'tractor sources from SDSS')
unmatched = np.ones(len(wc), bool)
unmatched[I] = False
wun = wc[unmatched]
print(len(wun), 'unmatched WISE sources')
for i in range(len(wun)):
pos = RaDecPos(wun.ra[i], wun.dec[i])
nm = NanoMaggies.magToNanomaggies(wun.w1mpro[i])
br = NanoMaggies(i=25., w1=nm)
tsrcs.append(PointSource(pos, br))
plt.clf()
plt.plot(rd[:, 0], rd[:, 1], 'k-')
plt.plot(cashere.ra, cashere.dec, 'r.', alpha=0.1)
plt.plot(wc.ra, wc.dec, 'bx', alpha=0.1)
setRadecAxes(r0, r1, d0, d1)
ps.savefig()
zlo, zhi = tim.zr
ima = dict(interpolation='nearest', origin='lower', vmin=zlo, vmax=zhi)
imchi = dict(interpolation='nearest', origin='lower',
vmin=-5, vmax=5)
plt.clf()
plt.imshow(tim.getImage(), **ima)
plt.hot()
plt.title(tim.name + ': data')
ps.savefig()
wsrcs = []
for i in range(len(wc)):
pos = RaDecPos(wc.ra[i], wc.dec[i])
nm = NanoMaggies.magToNanomaggies(wc.w1mpro[i])
br = NanoMaggies(i=25., w1=nm)
wsrcs.append(PointSource(pos, br))
tr = Tractor([tim], wsrcs)
tr.freezeParam('images')
for jj in range(2):
print('Rendering WISE model image...')
wmod = tr.getModelImage(0)
plt.clf()
plt.imshow(wmod, **ima)
plt.hot()
plt.title(tim.name + ': WISE sources only')
ps.savefig()
assert(np.all(np.isfinite(wmod)))
assert(np.all(np.isfinite(tim.getInvError())))
assert(np.all(np.isfinite(tim.getImage())))
wchi = tr.getChiImage(0)
plt.clf()
plt.imshow(wchi, **imchi)
plt.title(tim.name + ': chi, WISE sources only')
plt.gray()
ps.savefig()
if jj == 1:
break
tr.optimize()
tr = Tractor([tim], tsrcs)
print('Rendering model image...')
mod = tr.getModelImage(0)
plt.clf()
plt.imshow(mod, **ima)
plt.title(tim.name + ': SDSS + WISE sources')
ps.savefig()
print('tim', tim)
print('tim.photocal:', tim.photocal)
wsrcs = []
for i in range(len(wc)):
pos = RaDecPos(wc.ra[i], wc.dec[i])
nm = NanoMaggies.magToNanomaggies(wc.w1mpro[i])
br = NanoMaggies(i=25., w1=nm)
wsrcs.append(PointSource(pos, br))
tr = Tractor([tim], wsrcs)
print('Rendering WISE model image...')
wmod = tr.getModelImage(0)
plt.clf()
plt.imshow(wmod, **ima)
plt.title(tim.name + ': WISE sources only')
ps.savefig()
