def main():
import optparse
import logging
import sys
parser = optparse.OptionParser()
parser.add_option('--threads', dest='threads', default=1, type=int, help='Use this many concurrent processors')
parser.add_option('-v', '--verbose', dest='verbose', action='count', default=0,
help='Make more verbose')
parser.add_option('--grid', '-g', dest='gridn', type=int, default=5, help='Dust parameter grid size')
parser.add_option('--steps', '-s', dest='steps', type=int, default=10, help='# Optimization step')
parser.add_option('--suffix', dest='suffix', default='', help='Output file suffix')
parser.add_option('--no-100', dest='no100', action='store_true', default=False,
help='Omit PACS-100 data?')
parser.add_option('--callgrind', dest='callgrind', action='store_true', default=False, help='Turn on callgrind around tractor.optimize()')
parser.add_option('--resume', '-r', dest='resume', type=int, default=-1, help='Resume from a previous run at the given step?')
parser.add_option('--zoom', dest='zoom', type=float, default=1, help='Scale down the model to only touch the (1/zoom x 1/zoom) central region of the images')
parser.add_option('--damp', dest='damp', type=float, default=1., help='LSQR damping')
opt,args = parser.parse_args()
if opt.verbose == 0:
lvl = logging.INFO
log_init(2)
else:
lvl = logging.DEBUG
log_init(3)
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
if opt.threads > 1 and False:
global dpool
import debugpool
dpool = debugpool.DebugPool(opt.threads)
Time.add_measurement(debugpool.DebugPoolMeas(dpool))
mp = multiproc(pool=dpool)
else:
print('N threads', opt.threads)
mp = multiproc(opt.threads)#, wrap_all=True)
if opt.callgrind:
import callgrind
else:
callgrind = None
np.seterrcall(np_err_handler)
np.seterr(all='call')
#np.seterr(all='raise')
if opt.resume > -1:
pfn = 'herschel-%02i%s.pickle' % (opt.resume, opt.suffix)
print('Reading from', pfn)
tractor = unpickle_from_file(pfn)
tractor.mp = mp
ds = tractor.getCatalog()[0]
print('DustSheet:', ds)
# derivs = ds.getParamDerivatives(tim)
# dim = np.zeros(tim.shape)
# #for k,deriv in enumerate(derivs[:40]):
# for k,deriv in enumerate(derivs[::10]):
# dim[:,:] = 0
# deriv.addTo(dim)
# plt.clf()
# plt.imshow(dim, interpolation='nearest', origin='lower')
# plt.savefig('deriv-%04i.png' % k)
#tim = tractor.getImages()[0]
# for it,tim in enumerate(tractor.getImages()):
# X = ds._getTransformation(tim)
# # #print 'X', X
# keys = X.keys()
# keys.sort()
# # for k in keys[::10]:
# # for k in keys[:40]:
# for k in keys[::202]:
# I,G,nil,nil = X[k]
# rim = np.zeros_like(tim.getImage())
# rim.ravel()[I] = G
# plt.clf()
# plt.imshow(rim, interpolation='nearest', origin='lower')
# plt.colorbar()
# plt.savefig('rim-%i-%04i.png' % (it,k))
# print 'pix', k
# sys.exit(0)
makeplots(tractor, opt.resume, opt.suffix)
step0 = opt.resume + 1
else:
step0 = 0
tractor = create_tractor(opt)
tractor.mp = mp
# zero out invvar outside the model bounds.
ds = tractor.getCatalog()[0]
rd = ds.getRaDecCorners(margin=0.5)
for i,tim in enumerate(tractor.getImages()):
poly = np.array([tim.getWcs().positionToPixel(
RaDecPos(rdi[0], rdi[1])) for rdi in rd])
poly = poly[:-1,:]
print('Model bounding box in image', tim.name, 'coordinates:')
#print poly.shape
print(poly)
H,W = tim.shape
xx,yy = np.meshgrid(np.arange(W), np.arange(H))
inside = point_in_poly(xx, yy, poly)
tim.inverr[inside == 0] = 0.
plt.clf()
for i,tim in enumerate(tractor.images):
h,w = tim.shape
rd = [tim.getWcs().pixelToPosition(x,y)
for x,y in [(-0.5,-0.5),(w-0.5,-0.5),(w-0.5,h-0.5),
(-0.5,h-0.5),(-0.5,-0.5)]]
plt.plot([p.ra for p in rd], [p.dec for p in rd], '-',
label=tim.name)
rd = ds.getRaDecCorners(margin=0.5)
plt.plot(rd[:,0], rd[:,1], 'k-', label='Grid')
mh,mw = ds.shape
r,d = ds.wcs.pixelxy2radec(1 + np.arange(mw), np.ones(mw))
plt.plot(r, d, 'k.')
r,d = ds.wcs.pixelxy2radec(np.ones(mh), 1 + np.arange(mh))
plt.plot(r, d, 'k.')
r,d = ds.wcs.pixelxy2radec(1 + np.arange(mw), np.zeros(mw)+mh)
plt.plot(r, d, 'k.')
r,d = ds.wcs.pixelxy2radec(np.zeros(mh)+mw, 1 + np.arange(mh))
plt.plot(r, d, 'k.')
plt.legend()
plt.savefig('radec.png')
print('Precomputing transformations...')
ds = tractor.getCatalog()[0]
# Split the grid-spread matrix into strips...
async_results = []
for im in tractor.getImages():
args = []
H,W = ds.shape
dy = 10
y = 0
while y <= H:
args.append((ds, im, y, min(H, y+dy)))
y += dy
async_results.append(mp.map_async(_map_trans, args))
# Glue to strips back together...
XX = []
for ar in async_results:
Xblocks = ar.get()
X = Xblocks[0]
for xi in Xblocks[1:]:
X.update(xi)
XX.append(X)
for im,X in zip(tractor.getImages(), XX):
ds._normalizeTransformation(im, X)
ds._setTransformation(im, X)
print('done precomputing.')
# Plot the grid-spread functions.
for itim,tim in enumerate(tractor.images):
T = ds._getTransformation(tim)
(I,G,nz,NZI) = T[0]
plt.clf()
g = np.zeros(tim.shape, np.float32)
g.flat[I] = G
plt.imshow(g, interpolation='nearest', origin='lower', cmap='hot')
plt.colorbar()
plt.title('Grid-spread function for cell 0, image %s' % tim.name)
plt.savefig('gsf-%i.png' % itim)
makeplots(tractor, 0, opt.suffix)
pfn = 'herschel-%02i%s.pickle' % (0, opt.suffix)
pickle_to_file(tractor, pfn)
print('Wrote', pfn)
for im in tractor.getImages():
im.freezeAllBut('sky')
for i in range(step0, opt.steps):
print('Step', i)
if callgrind:
callgrind.callgrind_start_instrumentation()
tractor.optimize(damp=opt.damp, alphas=[1e-3, 1e-2, 0.1, 0.3, 1., 3., 10., 30., 100.])
if callgrind:
callgrind.callgrind_stop_instrumentation()
makeplots(tractor, 1 + i, opt.suffix)
pfn = 'herschel-%02i%s.pickle' % (1 + i, opt.suffix)
pickle_to_file(tractor, pfn)
print('Wrote', pfn)
def main():
import optparse
import logging
import sys
parser = optparse.OptionParser()
parser.add_option('--threads', dest='threads', default=1, type=int, help='Use this many concurrent processors')
parser.add_option('-v', '--verbose', dest='verbose', action='count', default=0,
help='Make more verbose')
parser.add_option('--grid', '-g', dest='gridn', type=int, default=5, help='Dust parameter grid size')
parser.add_option('--steps', '-s', dest='steps', type=int, default=10, help='# Optimization step')
parser.add_option('--suffix', dest='suffix', default='', help='Output file suffix')
parser.add_option('--no-100', dest='no100', action='store_true', default=False,
help='Omit PACS-100 data?')
parser.add_option('--callgrind', dest='callgrind', action='store_true', default=False, help='Turn on callgrind around tractor.optimize()')
parser.add_option('--resume', '-r', dest='resume', type=int, default=-1, help='Resume from a previous run at the given step?')
parser.add_option('--zoom', dest='zoom', type=float, default=1, help='Scale down the model to only touch the (1/zoom x 1/zoom) central region of the images')
parser.add_option('--damp', dest='damp', type=float, default=1., help='LSQR damping')
opt,args = parser.parse_args()
if opt.verbose == 0:
lvl = logging.INFO
log_init(2)
else:
lvl = logging.DEBUG
log_init(3)
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
if opt.threads > 1 and False:
global dpool
import debugpool
dpool = debugpool.DebugPool(opt.threads)
Time.add_measurement(debugpool.DebugPoolMeas(dpool))
mp = multiproc(pool=dpool)
else:
print 'N threads', opt.threads
mp = multiproc(opt.threads)#, wrap_all=True)
if opt.callgrind:
import callgrind
else:
callgrind = None
np.seterrcall(np_err_handler)
np.seterr(all='call')
#np.seterr(all='raise')
if opt.resume > -1:
pfn = 'herschel-%02i%s.pickle' % (opt.resume, opt.suffix)
print 'Reading from', pfn
tractor = unpickle_from_file(pfn)
tractor.mp = mp
ds = tractor.getCatalog()[0]
print 'DustSheet:', ds
# derivs = ds.getParamDerivatives(tim)
# dim = np.zeros(tim.shape)
# #for k,deriv in enumerate(derivs[:40]):
# for k,deriv in enumerate(derivs[::10]):
# dim[:,:] = 0
# deriv.addTo(dim)
# plt.clf()
# plt.imshow(dim, interpolation='nearest', origin='lower')
# plt.savefig('deriv-%04i.png' % k)
#tim = tractor.getImages()[0]
# for it,tim in enumerate(tractor.getImages()):
# X = ds._getTransformation(tim)
# # #print 'X', X
# keys = X.keys()
# keys.sort()
# # for k in keys[::10]:
# # for k in keys[:40]:
# for k in keys[::202]:
# I,G,nil,nil = X[k]
# rim = np.zeros_like(tim.getImage())
# rim.ravel()[I] = G
# plt.clf()
# plt.imshow(rim, interpolation='nearest', origin='lower')
# plt.colorbar()
# plt.savefig('rim-%i-%04i.png' % (it,k))
# print 'pix', k
# sys.exit(0)
makeplots(tractor, opt.resume, opt.suffix)
step0 = opt.resume + 1
else:
step0 = 0
tractor = create_tractor(opt)
tractor.mp = mp
# zero out invvar outside the model bounds.
ds = tractor.getCatalog()[0]
rd = ds.getRaDecCorners()
for i,tim in enumerate(tractor.getImages()):
poly = np.array([tim.getWcs().positionToPixel(RaDecPos(rdi[0], rdi[1])) for rdi in rd])
poly = poly[:-1,:]
print 'Model bounding box in image', tim.name, 'coordinates:'
print poly.shape
print poly
H,W = tim.shape
xx,yy = np.meshgrid(np.arange(W), np.arange(H))
inside = point_in_poly(xx, yy, poly)
iv = tim.getInvvar()
iv[(inside == 0)] = 0.
tim.setInvvar(iv)
print 'Precomputing transformations...'
ds = tractor.getCatalog()[0]
# Split the grid-spread matrix into strips...
async_results = []
for im in tractor.getImages():
args = []
H,W = ds.shape
dy = 10
y = 0
while y <= H:
args.append((ds, im, y, min(H, y+dy)))
y += dy
async_results.append(mp.map_async(_map_trans, args))
# Glue to strips back together...
XX = []
for ar in async_results:
Xblocks = ar.get()
X = Xblocks[0]
for xi in Xblocks[1:]:
X.update(xi)
XX.append(X)
for im,X in zip(tractor.getImages(), XX):
ds._normalizeTransformation(im, X)
ds._setTransformation(im, X)
print 'done precomputing.'
makeplots(tractor, 0, opt.suffix)
pfn = 'herschel-%02i%s.pickle' % (0, opt.suffix)
pickle_to_file(tractor, pfn)
print 'Wrote', pfn
for im in tractor.getImages():
im.freezeAllBut('sky')
for i in range(step0, opt.steps):
if callgrind:
callgrind.callgrind_start_instrumentation()
tractor.optimize(damp=opt.damp, alphas=[1e-3, 1e-2, 0.1, 0.3, 1., 3., 10., 30., 100.])
if callgrind:
callgrind.callgrind_stop_instrumentation()
makeplots(tractor, 1 + i, opt.suffix)
pfn = 'herschel-%02i%s.pickle' % (1 + i, opt.suffix)
pickle_to_file(tractor, pfn)
print 'Wrote', pfn
def main():
# make a simple psf
amp = np.array([0.9, 0.1])
mean = np.random.uniform(size=(2, 2))
var = np.array([2.0, 20.])
psf = MixtureOfGaussians(amp, mean, var)
psf.normalize()
trueimage = psf.evaluate_grid(-5, 7, -5, 7)
plt.clf()
plt.imshow(trueimage, interpolation='nearest', cmap='gray')
plt.savefig('truth.png')
# do some simple test fitting to check likelihood code
invvar = 0.01 * np.ones_like(trueimage)
flux = 1.0
y = 0.5
xlist = np.arange(-1.0, 1.0, 0.01)
lnLlist = np.array(
[lnLikelihood(trueimage, flux, x, y, psf, invvar) for x in xlist])
plt.clf()
plt.plot(xlist, lnLlist, 'k-', alpha=0.5)
plt.xlabel('model star position')
plt.ylabel('$\ln$ likelihood')
plt.savefig('lnL.png')
# try an optimization
if False:
pars = np.array([0.9, -1.0, 1.0])
notpars = (trueimage, psf, invvar)
bestpars = op.fmin(cost, pars, args=(notpars, ))
print(bestpars)
# now loop over noisiness
minx = -3.5
maxx = -1.5
picklefn = 'fluxes.pickle'
if not os.path.exists(picklefn):
logvarlist = np.random.uniform(minx, maxx, size=(3000))
fluxlist = np.zeros_like(logvarlist)
for i, logvar in enumerate(logvarlist):
noise = np.random.normal(size=trueimage.shape)
var = 10.**logvar
image = trueimage + var * noise
invvar = np.ones_like(image) / var / var
pars = np.array([1.0, 0.0, 0.0])
notpars = (image, psf, invvar)
bestpars = op.fmin(cost, pars, args=(notpars, ))
fluxlist[i] = bestpars[0]
print(i, logvar, bestpars)
an.pickle_to_file((logvarlist, fluxlist), picklefn)
(logvarlist, fluxlist) = an.unpickle_from_file(picklefn)
# make plots
plt.clf()
plt.axhline(1.1, color='k', alpha=0.25)
plt.axhline(1.0, color='k', alpha=0.25)
plt.axhline(0.9, color='k', alpha=0.25)
plt.plot(logvarlist, fluxlist, 'k.', alpha=0.5)
plt.xlabel('$\log_{10}$ pixel noise variance')
plt.ylabel('inferred flux')
plt.xlim(minx, maxx)
plt.ylim(0.8, 1.2)
plt.savefig('flux.png')
medbins = np.arange(minx, maxx + 0.0001, 0.25)
for i in range(len(medbins) - 1):
a, b = medbins[[i, i + 1]]
inside = np.sort(fluxlist[(logvarlist > a) * (logvarlist < b)])
nin = len(inside)
plt.plot([a, b], [inside[0.025 * nin], inside[0.025 * nin]],
'k-',
alpha=0.5)
plt.plot([a, b], [inside[0.16 * nin], inside[0.16 * nin]],
'k-',
alpha=0.5)
plt.plot([a, b], [inside[0.50 * nin], inside[0.50 * nin]],
'k-',
alpha=0.5)
plt.plot([a, b], [inside[0.84 * nin], inside[0.84 * nin]],
'k-',
alpha=0.5)
plt.plot([a, b], [inside[0.975 * nin], inside[0.975 * nin]],
'k-',
alpha=0.5)
plt.savefig('fluxquant.png')
def main():
# make a simple psf
amp = np.array([0.9, 0.1])
mean = np.random.uniform(size=(2, 2))
var = np.array([2.0, 20.])
psf = MixtureOfGaussians(amp, mean, var)
psf.normalize()
trueimage = psf.evaluate_grid(-5, 7, -5, 7)
plt.clf()
plt.imshow(trueimage, interpolation='nearest', cmap='gray')
plt.savefig('truth.png')
# do some simple test fitting to check likelihood code
invvar = 0.01 * np.ones_like(trueimage)
flux = 1.0
y = 0.5
xlist = np.arange(-1.0, 1.0, 0.01)
lnLlist = np.array([lnLikelihood(trueimage, flux, x, y, psf, invvar) for x in xlist])
plt.clf()
plt.plot(xlist, lnLlist, 'k-', alpha=0.5)
plt.xlabel('model star position')
plt.ylabel('$\ln$ likelihood')
plt.savefig('lnL.png')
# try an optimization
if False:
pars = np.array([0.9, -1.0, 1.0])
notpars = (trueimage, psf, invvar)
bestpars = op.fmin(cost, pars, args=(notpars, ))
print bestpars
# now loop over noisiness
minx = -3.5
maxx = -1.5
picklefn = 'fluxes.pickle'
if not os.path.exists(picklefn):
logvarlist = np.random.uniform(minx, maxx, size=(3000))
fluxlist = np.zeros_like(logvarlist)
for i, logvar in enumerate(logvarlist):
noise = np.random.normal(size=trueimage.shape)
var = 10.**logvar
image = trueimage + var * noise
invvar = np.ones_like(image) / var / var
pars = np.array([1.0, 0.0, 0.0])
notpars = (image, psf, invvar)
bestpars = op.fmin(cost, pars, args=(notpars, ))
fluxlist[i] = bestpars[0]
print i, logvar, bestpars
an.pickle_to_file((logvarlist, fluxlist), picklefn)
(logvarlist, fluxlist) = an.unpickle_from_file(picklefn)
# make plots
plt.clf()
plt.axhline(1.1, color='k', alpha=0.25)
plt.axhline(1.0, color='k', alpha=0.25)
plt.axhline(0.9, color='k', alpha=0.25)
plt.plot(logvarlist, fluxlist, 'k.', alpha=0.5)
plt.xlabel('$\log_{10}$ pixel noise variance')
plt.ylabel('inferred flux')
plt.xlim(minx, maxx)
plt.ylim(0.8, 1.2)
plt.savefig('flux.png')
medbins = np.arange(minx, maxx+0.0001, 0.25)
for i in range(len(medbins)-1):
a, b = medbins[[i, i+1]]
inside = np.sort(fluxlist[(logvarlist > a) * (logvarlist < b)])
nin = len(inside)
plt.plot([a, b], [inside[0.025* nin], inside[0.025* nin]], 'k-', alpha=0.5)
plt.plot([a, b], [inside[0.16 * nin], inside[0.16 * nin]], 'k-', alpha=0.5)
plt.plot([a, b], [inside[0.50 * nin], inside[0.50 * nin]], 'k-', alpha=0.5)
plt.plot([a, b], [inside[0.84 * nin], inside[0.84 * nin]], 'k-', alpha=0.5)
plt.plot([a, b], [inside[0.975* nin], inside[0.975* nin]], 'k-', alpha=0.5)
plt.savefig('fluxquant.png')
def main():
if os.path.exists('results.pickle'):
results = unpickle_from_file('results.pickle')
else:
results = []
for isrc,(ra,dec,brickname) in enumerate([
(0.2266, 3.9822, '0001p040'),
(7.8324, 1.2544, '0078p012'),
(1.1020, 3.9040, '0011p040'),
(7.3252, 4.6847, '0073p047'),
(3.1874, 3.9724, '0031p040'),
(9.5112, 4.6934, '0094p047'),
(4.4941, 1.1058, '0043p010'),
(3.8900, 0.6041, '0038p005'),
(8.1934, 4.0124, '0081p040'),
(6.8125, 0.5463, '0068p005'),
]):
#if isrc < 7:
# continue
if isrc not in [4,6,7,8,9]:
continue
outdir = 'out_%.4f_%.4f' % (ra,dec)
datadir = outdir.replace('out_', 'data_')
#cmd = 'ssh cori "cd legacypipe2/py && python legacypipe/runbrick.py --radec %.4f %.4f --width 100 --height 100 --survey-dir fakedr9 --outdir %s --stage image_coadds --skip-calibs && python legacyanalysis/create_testcase.py --survey-dir fakedr9 %s/coadd/*/*/*-ccds.fits %s %s"' % (ra, dec, outdir, outdir, datadir, brickname)
#cmd = 'ssh cori "cd legacypipe2/py && python legacypipe/runbrick.py --radec %.4f %.4f --width 100 --height 100 --survey-dir fakedr9 --outdir %s --stage image_coadds --skip-calibs && python legacyanalysis/create_testcase.py --survey-dir fakedr9 --outlier-dir %s %s/coadd/*/*/*-ccds.fits %s %s"' % (ra, dec, outdir, outdir, outdir, datadir, brickname)
outbrick = ('custom-%06i%s%05i' %
(int(1000*ra), 'm' if dec < 0 else 'p',
int(1000*np.abs(dec))))
cmd = 'ssh cori "cd legacypipe2/py && python legacyanalysis/create_testcase.py --survey-dir fakedr9 --outlier-dir %s --outlier-brick %s %s/coadd/*/*/*-ccds.fits %s %s"' % (outdir, outbrick, outdir, datadir, brickname)
#os.system(cmd)
cmd = 'rsync -arv cori:legacypipe2/py/%s .' % datadir
#os.system(cmd)
#continue
survey = LegacySurveyData(datadir)
b = Duck()
b.ra = ra
b.dec = dec
W,H = 80,80
wcs = wcs_for_brick(b, W=W, H=H)
targetrd = np.array([wcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
ccds = survey.ccds_touching_wcs(wcs)
print(len(ccds), 'CCDs')
ims = [survey.get_image_object(ccd) for ccd in ccds]
keepims = []
for im in ims:
h,w = im.shape
if h >= H and w >= W:
keepims.append(im)
ims = keepims
gims = [im for im in ims if im.band == 'g']
rims = [im for im in ims if im.band == 'r']
zims = [im for im in ims if im.band == 'z']
nk = min([len(gims), len(rims), len(zims), 5])
ims = gims[:nk] + rims[:nk] + zims[:nk]
print('Keeping', len(ims), 'images')
tims = [im.get_tractor_image(pixPsf=True, hybridPsf=True, normalizePsf=True, splinesky=True, radecpoly=targetrd) for im in ims]
bands = 'grz'
devsrc = DevGalaxy(RaDecPos(ra,dec), NanoMaggies(**dict([(b,10.) for b in bands])), EllipseESoft(0., 0., 0.))
tr = Tractor(tims, [devsrc])
tr.freezeParam('images')
tr.optimize_loop()
print('Fit DeV source:', devsrc)
devmods = list(tr.getModelImages())
showboth(tims, devmods, wcs, bands);
s = flux_string(devsrc.brightness)
plt.suptitle('DeV model: ' + s + '\ndchisq 0.')
plt.savefig('src%02i-dev.png' % isrc)
devchi = 2. * tr.getLogLikelihood()
dasrc = DevAgnGalaxy(devsrc.pos.copy(), devsrc.brightness.copy(), devsrc.shape.copy(), NanoMaggies(**dict([(b,1.) for b in bands])))
#dasrc = DevAgnGalaxy(RaDecPos(ra,dec), NanoMaggies(**dict([(b,10.) for b in bands])), EllipseESoft(0., 0., 0.), NanoMaggies(**dict([(b,1.) for b in bands])))
tr = Tractor(tims, [dasrc])
tr.freezeParam('images')
tr.optimize_loop()
print('Fit Dev+PSF source:', dasrc)
damods = list(tr.getModelImages())
showboth(tims, damods, wcs, bands)
s1 = flux_string(dasrc.brightnessDev)
s2 = flux_string(dasrc.brightnessPsf)
pcts = [100. * dasrc.brightnessPsf.getFlux(b) / dasrc.brightnessDev.getFlux(b) for b in bands]
s3 = ', '.join(['%.2f' % p for p in pcts])
dachi = 2. * tr.getLogLikelihood()
plt.suptitle('DeV + Point Source model: DeV %s, PSF %s' % (s1, s2) + ' (%s %%)' % s3 +
'\ndchisq %.1f' % (dachi - devchi))
plt.savefig('src%02i-devcore.png' % isrc)
#sersrc = SersicGalaxy(RaDecPos(ra, dec), NanoMaggies(**dict([(b,10.) for b in bands])), EllipseESoft(0.5, 0., 0.), SersicIndex(4.0))
sersrc = SersicGalaxy(devsrc.pos.copy(), devsrc.brightness.copy(), devsrc.shape.copy(), SersicIndex(4.0))
tr = Tractor(tims, [sersrc])
tr.freezeParam('images')
r = tr.optimize_loop()
print('Opt:', r)
print('Fit Ser source:', sersrc)
if sersrc.sersicindex.getValue() >= 6.0:
sersrc.freezeParam('sersicindex')
r = tr.optimize_loop()
print('Re-fit Ser source:', sersrc)
sermods = list(tr.getModelImages())
showboth(tims, sermods, wcs, bands)
s = flux_string(sersrc.brightness)
serchi = 2. * tr.getLogLikelihood()
plt.suptitle('Sersic model: %s, index %.2f' % (s, sersrc.sersicindex.getValue()) +
'\ndchisq %.1f' % (serchi - devchi))
plt.savefig('src%02i-ser.png' % isrc)
#sasrc = SersicAgnGalaxy(RaDecPos(ra, dec), NanoMaggies(**dict([(b,10.) for b in bands])), EllipseESoft(0.5, 0., 0.), SersicIndex(4.0), NanoMaggies(**dict([(b,1.) for b in bands])))
si = sersrc.sersicindex.getValue()
if si > 6.0:
si = 4.0
si = SersicIndex(si)
sasrc = SersicAgnGalaxy(sersrc.pos.copy(), sersrc.brightness.copy(), sersrc.shape.copy(), si, NanoMaggies(**dict([(b,1.) for b in bands])))
tr = Tractor(tims, [sasrc])
tr.freezeParam('images')
r = tr.optimize_loop()
print('Fit Ser+PSF source:', sasrc)
if sasrc.sersicindex.getValue() >= 6.0:
sasrc.freezeParam('sersicindex')
r = tr.optimize_loop()
print('Re-fit Ser+PSF source:', sasrc)
samods = list(tr.getModelImages())
showboth(tims, samods, wcs, bands)
s1 = flux_string(sasrc.brightness)
s2 = flux_string(sasrc.brightnessPsf)
pcts = [100. * sasrc.brightnessPsf.getFlux(b) / sasrc.brightness.getFlux(b) for b in bands]
s3 = ', '.join(['%.2f' % p for p in pcts])
sachi = 2. * tr.getLogLikelihood()
plt.suptitle('Sersic + Point Source model: Ser %s, index %.2f, PSF %s' % (s1, sasrc.sersicindex.getValue(), s2) +
' (%s %%)' % s3 +
'\ndchisq %.1f' % (sachi - devchi))
plt.savefig('src%02i-sercore.png' % isrc)
ri = (ra, dec, brickname, devsrc, devchi, dasrc, dachi,
sersrc, serchi, sasrc, sachi)
if len(results) > isrc:
results[isrc] = ri
else:
results.append(ri)
pickle_to_file(results, 'results.pickle')
def bayes_figs(DES, detmaps, detivs, good, wcs, img):
# First, build empirical SED prior "library" from DES sources
DES.flux_g = np.maximum(0, DES.flux_auto_g)
DES.flux_r = np.maximum(0, DES.flux_auto_r)
DES.flux_i = np.maximum(0, DES.flux_auto_i)
flux = DES.flux_g + DES.flux_r + DES.flux_i
K = np.flatnonzero(flux > 0)
DES.cut(K)
flux = flux[K]
DES.f_g = DES.flux_g / flux
DES.f_r = DES.flux_r / flux
DES.f_i = DES.flux_i / flux
print('Kept', len(DES), 'with positive flux')
nbins=21
edge = 1. / (nbins-1) / 2.
#N,xe,ye = loghist(DES.f_g, DES.f_r, range=((0-edge,1+edge),(0-edge,1+edge)), nbins=nbins);
N,xe,ye = np.histogram2d(DES.f_g, DES.f_r,
range=((0-edge,1+edge),(0-edge,1+edge)),
bins=nbins)
N = N.T
print(np.sum(N > 0), np.sum(N > N.sum()*0.001))
NN = N.copy()
NN[N < N.sum()*0.001] = np.nan
plt.figure(figsize=(3,3))
plt.subplots_adjust(left=0.2, right=0.98, bottom=0.15, top=0.98)
plt.clf()
x0,x1 = -edge, 1+edge
y0,y1 = x0,x1
plt.imshow(NN, interpolation='nearest', origin='lower', extent=(x0,x1,y0,y1),
cmap=antigray, zorder=20)
plt.plot([x0, x0, x1, x0], [y0,y1,y0,y0], 'k-', zorder=30)
plt.gca().set_frame_on(False)
p = Polygon(np.array([[x0, x0, x1, x0], [y0, y1, y0, y0]]).T, color=(0.9,0.9,1),
zorder=15)
plt.gca().add_artist(p)
plt.xlabel('flux fraction g')
plt.ylabel('flux fraction r')
plt.savefig('bayes-prior-sed.pdf')
#iy,ix = np.nonzero(N)
iy,ix = np.nonzero(N > N.sum()*0.001)
print(len(iy), 'significant bins')
# Find f_{g,r} histogram midpoints
mx = (xe[:-1] + xe[1:]) / 2.
my = (ye[:-1] + ye[1:]) / 2.
fg = mx[ix]
fr = my[iy]
fi = 1. - (fg + fr)
fn = N[iy,ix]
seds = np.clip(np.vstack((fg,fr,fi)).T, 0., 1.)
weights = fn / np.sum(fn)
ok = np.flatnonzero(np.sum(seds, axis=1) == 1)
seds = seds[ok,:]
weights = weights[ok]
print(len(weights), 'color-color bins are populated; max weight', weights.max())
plt.clf()
plothist(DES.mag_auto_g - DES.mag_auto_r, DES.mag_auto_r - DES.mag_auto_i,
range=((-0.5, 3),(-0.5, 2)), nbins=20,
imshowargs=dict(cmap=antigray), dohot=False, docolorbar=False)
plt.xlabel('g - r (mag)')
plt.ylabel('r - i (mag)')
plt.savefig('bayes-data-cc.pdf')
plt.clf()
fg_grid,fr_grid = np.meshgrid(xe, ye)
fi_grid = 1. - (fg_grid + fr_grid)
gr_grid = -2.5 * np.log10(fg_grid / fr_grid)
ri_grid = -2.5 * np.log10(fr_grid / fi_grid)
good_grid = (fi_grid >= 0) * np.isfinite(gr_grid) * np.isfinite(ri_grid)
h,w = good_grid.shape
cm = antigray
ng = 0
for j0 in range(h-1):
for i0 in range(w-1):
j1 = j0+1
i1 = i0+1
if not(good_grid[j0,i0] and good_grid[j0,i1] and good_grid[j1,i0] and good_grid[j1,i1]):
continue
if N[j0,i0] == 0:
continue
ng += N[j0,i0]
xx = [gr_grid[j0,i0], gr_grid[j0,i1], gr_grid[j1,i1], gr_grid[j1,i0], gr_grid[j0,i0]]
yy = [ri_grid[j0,i0], ri_grid[j0,i1], ri_grid[j1,i1], ri_grid[j1,i0], ri_grid[j0,i0]]
xy = np.vstack((xx, yy)).T
poly = Polygon(xy, color=cm(N[j0,i0]/N.max()))
plt.gca().add_artist(poly)
#plt.gca().set_fc('0.5')
plt.axis([-0.5, 3, -0.5,2])
plt.xlabel('g - r (mag)')
plt.ylabel('r - i (mag)')
print(ng, 'of', N.sum(), 'plotted')
plt.savefig('bayes-prior-cc.pdf')
H,W = detmaps[0].shape
bayesfn = 'lprb.pickle'
if os.path.exists(bayesfn):
print('Reading cached', bayesfn)
lprb = unpickle_from_file(bayesfn)
else:
# Number of bands
J = 3
# Build detection-map & iv arrays
D = np.zeros((J,H,W))
Div = np.zeros((J,H,W))
for i,(d,div) in enumerate(zip(detmaps,detivs)):
D[i,:,:] = d
Div[i,:,:] = div
alpha = 1.
t0 = time.process_time()
lprb = log_pratio_bayes(seds, weights, D, Div, alpha)
t1 = time.process_time()
print('Bayes took', t1-t0, 'CPU-seconds')
pickle_to_file(lprb, bayesfn)
print('Writing cache', bayesfn)
# Bayes took 276.118402 CPU-seconds
sz = 20
bx,by = detect_sources(lprb, 2000)
bsources = fits_table()
bsources.x = bx
bsources.y = by
bsources.lprb = lprb[bsources.y, bsources.x]
bsources.cut((bsources.x > sz) * (bsources.x < (W-sz)) * (bsources.y > sz) * (bsources.y < (H-sz)))
bsources.cut(good[bsources.y, bsources.x])
print('Kept', len(bsources))
g_det, r_det, i_det = detmaps
bsources.g_flux = g_det[bsources.y, bsources.x]
bsources.r_flux = r_det[bsources.y, bsources.x]
bsources.i_flux = i_det[bsources.y, bsources.x]
bsources.ra,bsources.dec = wcs.pixelxy2radec(bsources.x+1, bsources.y+1)
bsources.g_mag = -2.5*(np.log10(bsources.g_flux) - 9)
bsources.r_mag = -2.5*(np.log10(bsources.r_flux) - 9)
bsources.i_mag = -2.5*(np.log10(bsources.i_flux) - 9)
bsources.gr = bsources.g_mag - bsources.r_mag
bsources.ri = bsources.r_mag - bsources.i_mag
I = np.argsort(-bsources.lprb)
bsources.cut(I)
# save for later...
bsources_orig = bsources.copy()
# g + r + i detections
xg,yg = detect_sources(detmaps[0] * np.sqrt(detivs[0]), 50.)
xr,yr = detect_sources(detmaps[1] * np.sqrt(detivs[1]), 50.)
xi,yi = detect_sources(detmaps[2] * np.sqrt(detivs[2]), 50.)
print('Detected', len(xg),len(xr),len(xi), 'gri')
xm,ym = xg.copy(),yg.copy()
for xx,yy in [(xr,yr),(xi,yi)]:
I,J,d = match_xy(xm,ym, xx,yy, 5.)
print('Matched:', len(I))
U = np.ones(len(xx), bool)
U[J] = False
print('Unmatched:', np.sum(U))
xm = np.hstack((xm, xx[U]))
ym = np.hstack((ym, yy[U]))
print('Total of', len(xm), 'g+r+i')
sources = fits_table()
sources.x = xm
sources.y = ym
iy,ix = np.round(sources.y).astype(int), np.round(sources.x).astype(int)
sources.sn_g = (detmaps[0] * np.sqrt(detivs[0]))[iy,ix]
sources.sn_r = (detmaps[1] * np.sqrt(detivs[1]))[iy,ix]
sources.sn_i = (detmaps[2] * np.sqrt(detivs[2]))[iy,ix]
sources.g_flux = g_det[sources.y, sources.x]
sources.r_flux = r_det[sources.y, sources.x]
sources.i_flux = i_det[sources.y, sources.x]
sources.ra,sources.dec = wcs.pixelxy2radec(sources.x+1, sources.y+1)
sources.g_mag = -2.5*(np.log10(sources.g_flux) - 9)
sources.r_mag = -2.5*(np.log10(sources.r_flux) - 9)
sources.i_mag = -2.5*(np.log10(sources.i_flux) - 9)
sources.gr = sources.g_mag - sources.r_mag
sources.ri = sources.r_mag - sources.i_mag
sources.sn_max = np.maximum(sources.sn_g, np.maximum(sources.sn_r, sources.sn_i))
sources.cut((sources.x > sz) * (sources.x < (W-sz)) * (sources.y > sz) * (sources.y < (H-sz)))
sources.cut(good[sources.y, sources.x])
print('Kept', len(sources))
I = np.argsort(-sources.sn_max)
sources.cut(I)
sources_orig = sources.copy()
N = min(len(sources), len(bsources))
sources = sources[:N]
bsources = bsources[:N]
print('Cut both to', N)
# Define unmatched sources as ones that are not in the other's hot region
hot = binary_fill_holes(np.logical_or(detmaps[0] * np.sqrt(detivs[0]) > 50.,
np.logical_or(detmaps[1] * np.sqrt(detivs[1]) > 50.,
detmaps[2] * np.sqrt(detivs[2]) > 50.)))
Bhot = binary_fill_holes(lprb > 2000.)
UB = np.flatnonzero((hot[bsources.y, bsources.x] == False))
US = np.flatnonzero((Bhot[sources.y, sources.x] == False))
print(len(UB), 'unmatched Bayesian', len(US), 'g+r+i')
MS = np.flatnonzero(Bhot[sources.y, sources.x])
plt.figure(figsize=(6,4))
plt.subplots_adjust(left=0.1, right=0.98, bottom=0.12, top=0.98)
plt.clf()
# for the legend only
p1 = plt.plot(-10, -10, 'k.')
plt.plot(sources.gr[MS], sources.ri[MS], 'k.', alpha=0.1,
label='Both')
p2 = plt.plot(sources.gr[US], sources.ri[US], 'o', mec='r', mfc='none',
label='g+r+i only')
p3 = plt.plot(bsources.gr[UB], bsources.ri[UB], 'bx',
label='Bayesian only')
plt.xlabel('g - r (mag)')
plt.ylabel('r - i (mag)')
plt.legend((p1[0],p2[0],p3[0]),
('Detected by both', 'Only detected by g+r+i', 'Only detected by Bayesian'),
loc='lower left')
plt.axis([-5, 5, -3, 3])
plt.savefig('bayes-vs-gri.pdf')
plt.figure(figsize=(4,4))
plt.subplots_adjust(left=0.01, right=0.99, bottom=0.01, top=0.99)
plt.clf()
show_sources(bsources[UB], img, R=10, C=10, divider=1)
#plt.suptitle('Bayesian only');
plt.savefig('bayes-only.pdf')
plt.clf()
show_sources(sources[US], img, R=10, C=10, divider=1)
#plt.suptitle('Bayesian only');
plt.savefig('gri-only.pdf')
#####
sources = sources_orig.copy()
# What if we demand 2-band detection?
#bsources = bsources_orig
# Detect at a higher threshold to get ~2400 sources
Bhot = binary_fill_holes(lprb > 4000.)
bx,by = detect_sources(lprb, 4000.)
bsources = fits_table()
bsources.x = bx
bsources.y = by
bsources.lprb = lprb[bsources.y, bsources.x]
bsources.cut((bsources.x > sz) * (bsources.x < (W-sz)) * (bsources.y > sz) * (bsources.y < (H-sz)))
bsources.cut(good[bsources.y, bsources.x])
print('Kept', len(bsources))
bsources.g_flux = g_det[bsources.y, bsources.x]
bsources.r_flux = r_det[bsources.y, bsources.x]
bsources.i_flux = i_det[bsources.y, bsources.x]
bsources.ra,bsources.dec = wcs.pixelxy2radec(bsources.x+1, bsources.y+1)
bsources.g_mag = -2.5*(np.log10(bsources.g_flux) - 9)
bsources.r_mag = -2.5*(np.log10(bsources.r_flux) - 9)
bsources.i_mag = -2.5*(np.log10(bsources.i_flux) - 9)
bsources.gr = bsources.g_mag - bsources.r_mag
bsources.ri = bsources.r_mag - bsources.i_mag
I = np.argsort(-bsources.lprb)
bsources.cut(I)
sthresh = 50.
hotg = binary_fill_holes(detmaps[0] * np.sqrt(detivs[0]) > sthresh)
hotr = binary_fill_holes(detmaps[1] * np.sqrt(detivs[1]) > sthresh)
hoti = binary_fill_holes(detmaps[2] * np.sqrt(detivs[2]) > sthresh)
hotsum = hotg*1 + hotr*1 + hoti*1
hot = (hotsum >= 2)
K = np.flatnonzero(hot[sources.y,sources.x])
# 2386 detected in 2 or more bands
print(len(K), 'detected in 2 or more bands')
sources.cut(K)
N = min(len(sources), len(bsources))
sources = sources[:N]
bsources = bsources[:N]
print('Cut both to', N)
UB = np.flatnonzero((hot[bsources.y, bsources.x] == False))
US = np.flatnonzero((Bhot[sources.y, sources.x] == False))
print(len(UB), 'unmatched Bayesian', len(US), '2 of (g+r+i)')
plt.figure(figsize=(6,4))
plt.subplots_adjust(left=0.1, right=0.98, bottom=0.12, top=0.98)
plt.clf()
plt.plot(sources.gr[US], sources.ri[US], 'o', mec='r', mfc='none',
label='g+r+i only')
plt.plot(bsources.gr[UB], bsources.ri[UB], 'kx',
label='Bayesian only')
plt.xlabel('g - r (mag)')
plt.ylabel('r - i (mag)')
plt.legend()
plt.axis([-5, 5, -3, 3])
plt.savefig('bayes-vs-2gri.pdf')
plt.figure(figsize=(4,4))
plt.subplots_adjust(left=0.01, right=0.99, bottom=0.01, top=0.99)
plt.clf()
show_sources(bsources[UB], img, R=10, C=10, divider=1)
plt.savefig('bayes-2only.pdf')
plt.clf()
show_sources(sources[US], img, R=10, C=10, divider=1)
plt.savefig('2gri-only.pdf')
