def sky_fiber_plots(survey,
brickname,
skyfibers,
basefn,
bands=['g', 'r', 'z']):
from legacypipe.survey import get_rgb
import fitsio
import pylab as plt
rgbkwargs = dict(mnmx=(-1, 100.), arcsinh=1.)
imgs = []
for band in bands:
fn = survey.find_file('image', brick=brickname, band=band)
imgs.append(fitsio.read(fn))
rgb = get_rgb(imgs, bands, **rgbkwargs)
ima = dict(interpolation='nearest', origin='lower')
plt.clf()
plt.imshow(rgb, **ima)
plt.plot(skyfibers.x, skyfibers.y, 'o', mfc='none', mec='r', mew=2, ms=10)
plt.title('Sky fiber positions')
plt.savefig(basefn + '-1.png')
plt.clf()
plt.subplots_adjust(hspace=0, wspace=0)
SZ = 25
fig = plt.gcf()
fh, fw = fig.get_figheight(), fig.get_figwidth()
C = int(np.ceil(np.sqrt(len(skyfibers) * fw / fh)))
R = int(np.ceil(len(skyfibers) / float(C)))
k = 1
H, W = imgs[0].shape
for x, y in zip(skyfibers.x, skyfibers.y):
if x < SZ or y < SZ or x >= W - SZ or y >= H - SZ:
continue
plt.subplot(R, C, k)
k += 1
plt.imshow(rgb[y - SZ:y + SZ + 1, x - SZ:x + SZ + 1, :], **ima)
plt.xticks([])
plt.yticks([])
plt.suptitle('Sky fiber locations')
plt.savefig(basefn + '-2.png')
plt.clf()
ccmap = dict(z='m')
for band in bands:
flux = skyfibers.get('apflux_%s' % band)
plt.plot(flux.T, color=ccmap.get(band, band), alpha=0.1)
plt.ylim(-10, 10)
#plt.xticks(np.arange(len(apertures_arcsec)),
# ['%g' % ap for ap in apertures_arcsec])
plt.xlabel('Aperture') # (arcsec radius)')
plt.ylabel('Aperture flux (nanomaggies)')
plt.title('Sky fiber: aperture flux')
plt.savefig(basefn + '-3.png')
def main():
survey = LegacySurveyData()
brickname = '2351p137'
# RA,Dec = 235.0442, 13.7125
bx, by = 3300, 1285
sz = 50
bbox = [bx - sz, bx + sz, by - sz, by + sz]
objid = 1394
bands = ['g', 'r', 'z']
from legacypipe.runbrick import stage_tims, _get_mod, rgbkwargs, rgbkwargs_resid
from legacypipe.survey import get_rgb, imsave_jpeg
from legacypipe.coadds import make_coadds
from astrometry.util.multiproc import multiproc
from astrometry.util.fits import fits_table
from legacypipe.catalog import read_fits_catalog
# brick = survey.get_brick_by_name(brickname)
# # Get WCS object describing brick
# targetwcs = wcs_for_brick(brick)
# (x0,x1,y0,y1) = bbox
# W = x1-x0
# H = y1-y0
# targetwcs = targetwcs.get_subimage(x0, y0, W, H)
# H,W = targetwcs.shape
mp = multiproc()
P = stage_tims(brickname=brickname,
survey=survey,
target_extent=bbox,
pixPsf=True,
hybridPsf=True,
depth_cut=False,
mp=mp)
print('Got', P.keys())
tims = P['tims']
targetwcs = P['targetwcs']
H, W = targetwcs.shape
# Read Tractor catalog
fn = survey.find_file('tractor', brick=brickname)
print('Trying to read catalog', fn)
cat = fits_table(fn)
print('Read', len(cat), 'sources')
ok, xx, yy = targetwcs.radec2pixelxy(cat.ra, cat.dec)
I = np.flatnonzero((xx > 0) * (xx < W) * (yy > 0) * (yy < H))
cat.cut(I)
print('Cut to', len(cat), 'sources within box')
I = np.flatnonzero(cat.objid != objid)
cat.cut(I)
print('Cut to', len(cat), 'sources with objid !=', objid)
#cat.about()
# Convert FITS catalog into tractor source objects
print('Creating tractor sources...')
srcs = read_fits_catalog(cat, fluxPrefix='')
print('Sources:')
for src in srcs:
print(' ', src)
print('Rendering model images...')
mods = [_get_mod((tim, srcs)) for tim in tims]
print('Producing coadds...')
C = make_coadds(tims, bands, targetwcs, mods=mods, mp=mp)
print('Coadds:', dir(C))
coadd_list = [('image', C.coimgs, rgbkwargs),
('model', C.comods, rgbkwargs),
('resid', C.coresids, rgbkwargs_resid)]
#C.coimgs, C.comods, C.coresids
for name, ims, rgbkw in coadd_list:
rgb = get_rgb(ims, bands, **rgbkw)
kwa = {}
#with survey.write_output(name + '-jpeg', brick=brickname) as out:
# imsave_jpeg(out.fn, rgb, origin='lower', **kwa)
# print('Wrote', out.fn)
outfn = name + '.jpg'
imsave_jpeg(outfn, rgb, origin='lower', **kwa)
del rgb
def top_levels(mp, opt):
from map.views import save_jpeg, trymakedirs
if opt.kind in [
'decaps2', 'decaps2-model', 'decaps2-resid', 'mzls+bass-dr4',
'mzls+bass-dr4-model', 'mzls+bass-dr4-resid', 'decals-dr5',
'decals-dr5-model', 'decals-dr5-resid', 'mzls+bass-dr6',
'mzls+bass-dr6-model', 'mzls+bass-dr6-resid', 'eboss',
'unwise-neo2', 'sdss2'
]:
import pylab as plt
from decals import settings
from legacypipe.survey import get_rgb
import fitsio
from scipy.ndimage.filters import gaussian_filter
from map.views import trymakedirs
from map.views import _unwise_to_rgb
tag = opt.kind
rgbkwargs = {}
if opt.kind == 'unwise-neo2':
bands = [1, 2]
get_rgb = _unwise_to_rgb
elif opt.kind == 'sdss2':
bands = 'gri'
get_rgb = sdss_rgb
else:
bands = 'grz'
get_rgb = dr2_rgb
ver = tileversions.get(opt.kind, [1])[-1]
print('Version', ver)
basescale = 5
pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s',
'%(zoom)i', '%(x)i', '%(y)i.jpg')
patdata = dict(ver=ver)
tilesize = 256
tiles = 2**basescale
side = tiles * tilesize
basepat = 'base-%s-%i-%%s.fits' % (opt.kind, basescale)
basefns = [basepat % band for band in bands]
if not all([os.path.exists(fn) for fn in basefns]):
bases = [np.zeros((side, side), np.float32) for band in bands]
args = []
xy = []
if opt.y1 is None:
opt.y1 = tiles
if opt.x0 is None:
opt.x0 = 0
if opt.x1 is None:
opt.x1 = tiles
for y in range(opt.y0, opt.y1):
for x in range(opt.x0, opt.x1):
args.append((opt.kind, basescale, x, y, False, True))
xy.append((x, y))
tiles = mp.map(_one_tile, args)
for ims, (x, y) in zip(tiles, xy):
#for a,(x,y) in zip(args, xy):
#print('_one_tile args:', a)
#ims = _one_tile(a)
#print('-> ', ims)
if ims is None:
continue
for im, base in zip(ims, bases):
if im is None:
continue
base[y * tilesize:(y + 1) * tilesize,
x * tilesize:(x + 1) * tilesize] = im
for fn, base in zip(basefns, bases):
fitsio.write(fn, base, clobber=True)
else:
print('Reading', basefns)
bases = [fitsio.read(fn) for fn in basefns]
for scale in range(basescale, -1, -1):
print('Scale', scale)
tiles = 2**scale
for y in range(tiles):
for x in range(tiles):
ims = [
base[y * tilesize:(y + 1) * tilesize,
x * tilesize:(x + 1) * tilesize] for base in bases
]
rgb = get_rgb(ims, bands, **rgbkwargs)
pp = patdata.copy()
pp.update(zoom=scale, x=x, y=y)
fn = pat % pp
trymakedirs(fn)
save_jpeg(fn, rgb)
print('Wrote', fn)
for i, base in enumerate(bases):
base = (base[::2, ::2] + base[1::2, ::2] + base[1::2, 1::2] +
base[::2, 1::2]) / 4.
bases[i] = base
elif opt.kind in [
'unwise',
'unwise-neo1',
'unwise-w3w4',
]:
import pylab as plt
from decals import settings
from map.views import _unwise_to_rgb, save_jpeg, trymakedirs
import fitsio
if opt.kind == 'unwise-w3w4':
tag = 'unwise-w3w4'
bands = [3, 4]
bounce = _bounce_map_unwise_w3w4
elif opt.kind == 'unwise-neo1':
tag = 'unwise-neo1'
bands = [1, 2]
bounce = _bounce_map_unwise_neo1
else:
tag = 'unwise-w1w2'
bands = [1, 2]
bounce = _bounce_map_unwise_w1w2
pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s',
'%(zoom)i', '%(x)i', '%(y)i.jpg')
ver = 1
patdata = dict(ver=ver)
basescale = 4
tilesize = 256
tiles = 2**basescale
side = tiles * tilesize
basepat = 'base-%s-%i-%%s.fits' % (opt.kind, basescale)
basefns = [basepat % band for band in bands]
if not all([os.path.exists(fn) for fn in basefns]):
bases = [np.zeros((side, side), np.float32) for band in bands]
args = []
for y in range(tiles):
for x in range(tiles):
#print 'Base tile', x, y
args.append((req, ver, basescale, x, y))
tiles = mp.map(bounce, args)
for ims, arg in zip(tiles, args):
x, y = arg[-2:]
for im, base in zip(ims, bases):
if im is None:
continue
base[y * tilesize:(y + 1) * tilesize,
x * tilesize:(x + 1) * tilesize] = im
for fn, base in zip(basefns, bases):
fitsio.write(fn, base, clobber=True)
else:
print('Reading', basefns)
bases = [fitsio.read(fn) for fn in basefns]
if False:
# Messin' around
plt.figure(figsize=(8, 8))
plt.subplots_adjust(left=0, right=1, bottom=0, top=1)
#for S in [1000, 3000, 10000]:
#for Q in [10, 25, 50]:
S, Q = 3000, 25
im = _unwise_to_rgb([w1base, w2base], S=S, Q=Q)
#plt.clf()
#plt.imshow(im)
plt.imsave('base-S%i-Q%s.png' % (S, Q), im)
# Try converting to galactic coords...
from astrometry.util.util import anwcs_create_mercator_2
print('Base images:', w1base.shape)
zoom = basescale
h, w = w1base.shape
zoomscale = 2.**zoom * (256. / h)
print('Zoomscale', zoomscale)
wcs = anwcs_create_mercator_2(180., 0., w / 2., h / 2.,
zoomscale, w, h, 1)
wcs2 = anwcs_create_mercator_2(0., 0., w / 2., h / 2.,
zoomscale, w, h, 1)
print('WCS:')
for x, y in [(1, 1), (1, h), (w, 1), (w, h), (w / 2, 1),
(w / 2, h / 2)]:
print('x,y', (x, y), '-> RA,Dec',
wcs.pixelxy2radec(x, y)[-2:])
ok, ras, nil = wcs2.pixelxy2radec(np.arange(w), np.ones(w))
ok, nil, decs = wcs2.pixelxy2radec(np.ones(h), np.arange(h))
print('RAs', ras.shape)
print('Decs', decs.shape)
lls = ras
bbs = decs
ll, bb = np.meshgrid(lls, bbs)
print('LL,BB', ll.shape, bb.shape)
from astrometry.util.starutil_numpy import lbtoradec
ra, dec = lbtoradec(ll, bb)
print('RA,Dec', ra.shape, dec.shape)
ok, xx, yy = wcs.radec2pixelxy(ra, dec)
print('xx,yy', xx.shape, yy.shape)
lb1 = w1base[np.clip(np.round(yy - 1).astype(int), 0, h - 1),
np.clip(np.round(xx - 1).astype(int), 0, w - 1)]
lb2 = w2base[np.clip(np.round(yy - 1).astype(int), 0, h - 1),
np.clip(np.round(xx - 1).astype(int), 0, w - 1)]
lbim = _unwise_to_rgb(lb1, lb2, S=S, Q=Q)
plt.imsave('lb.png', lbim)
sys.exit(0)
from scipy.ndimage.filters import gaussian_filter
for scale in range(basescale - 1, -1, -1):
for i, base in enumerate(bases):
base = gaussian_filter(base, 1.)
base = (base[::2, ::2] + base[1::2, ::2] + base[1::2, 1::2] +
base[::2, 1::2]) / 4.
bases[i] = base
tiles = 2**scale
for y in range(tiles):
for x in range(tiles):
ims = [
base[y * tilesize:(y + 1) * tilesize,
x * tilesize:(x + 1) * tilesize] for base in bases
]
tile = _unwise_to_rgb(ims, bands=bands)
pp = patdata.copy()
pp.update(zoom=scale, x=x, y=y)
fn = pat % pp
trymakedirs(fn)
save_jpeg(fn, tile)
print('Wrote', fn)
if opt.kind in ['depth-g', 'depth-r', 'depth-z']:
import pylab as plt
from decals import settings
from scipy.ndimage.filters import gaussian_filter
from map.views import trymakedirs
tag = 'decam-' + opt.kind
band = opt.kind[-1]
ver = 1
basescale = 5
pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s',
'%(zoom)i', '%(x)i', '%(y)i.jpg')
patdata = dict(ver=ver)
tilesize = 256
tiles = 2**basescale
side = tiles * tilesize
base = np.zeros((side, side, 3), np.float32)
for y in range(tiles):
for x in range(tiles):
dat = patdata.copy()
dat.update(zoom=basescale, x=x, y=y)
fn = pat % dat
if not os.path.exists(fn):
print('Does not exist:', fn)
continue
img = plt.imread(fn)
base[y * tilesize:(y + 1) * tilesize,
x * tilesize:(x + 1) * tilesize, :] = img
for scale in range(basescale - 1, -1, -1):
newbase = []
for i in range(3):
b = gaussian_filter(base[:, :, i], 1.)
b = (b[::2, ::2] + b[1::2, ::2] + b[1::2, 1::2] +
b[::2, 1::2]) / 4.
newbase.append(b)
base = np.dstack(newbase)
tiles = 2**scale
for y in range(tiles):
for x in range(tiles):
img = base[y * tilesize:(y + 1) * tilesize,
x * tilesize:(x + 1) * tilesize, :]
pp = patdata.copy()
pp.update(zoom=scale, x=x, y=y)
fn = pat % pp
trymakedirs(fn)
plt.imsave(fn,
np.clip(np.round(img).astype(np.uint8), 0, 255))
print('Wrote', fn)
### HACK... factor this out...
if opt.kind in [
'sdss',
'decals-dr2',
'decals-dr2-model',
'decals-dr2-resid',
'decals-dr3',
'decals-dr3-model',
'decals-dr3-resid',
]:
import pylab as plt
from decals import settings
from legacypipe.survey import get_rgb
import fitsio
from scipy.ndimage.filters import gaussian_filter
from map.views import trymakedirs
tag = opt.kind
bouncemap = {
'sdss': _bounce_sdssco,
'decals-dr3': _bounce_decals_dr3,
'decals-dr3-model': _bounce_decals_dr3,
'decals-dr3-resid': _bounce_decals_dr3,
}
bounce = bouncemap[opt.kind]
rgbkwargs = dict(mnmx=(-1, 100.), arcsinh=1.)
bands = 'grz'
if opt.kind in [
'decals-dr2',
'decals-dr2-model',
'decals-dr3',
'decals-dr3-model',
]:
get_rgb = dr2_rgb
rgbkwargs = {}
elif opt.kind == 'sdssco':
rgbfunc = sdss_rgb
rgbkwargs = {}
ver = tileversions.get(opt.kind, [1])[-1]
print('Version', ver)
basescale = 5
pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s',
'%(zoom)i', '%(x)i', '%(y)i.jpg')
patdata = dict(ver=ver)
tilesize = 256
tiles = 2**basescale
side = tiles * tilesize
basepat = 'base-%s-%i-%%s.fits' % (opt.kind, basescale)
basefns = [basepat % band for band in bands]
if not all([os.path.exists(fn) for fn in basefns]):
bases = [np.zeros((side, side), np.float32) for band in bands]
args = []
kwa = dict()
if 'model' in opt.kind:
kwa.update(model=True, add_gz=True)
elif 'resid' in opt.kind:
kwa.update(resid=True, model_gz=True)
xy = []
if opt.y1 is None:
opt.y1 = tiles
if opt.x0 is None:
opt.x0 = 0
if opt.x1 is None:
opt.x1 = tiles
for y in range(opt.y0, opt.y1):
for x in range(opt.x0, opt.x1):
args.append(((req, ver, basescale, x, y), kwa))
xy.append((x, y))
tiles = mp.map(bounce, args)
for ims, (x, y) in zip(tiles, xy):
if ims is None:
continue
for im, base in zip(ims, bases):
if im is None:
continue
base[y * tilesize:(y + 1) * tilesize,
x * tilesize:(x + 1) * tilesize] = im
for fn, base in zip(basefns, bases):
fitsio.write(fn, base, clobber=True)
else:
print('Reading', basefns)
bases = [fitsio.read(fn) for fn in basefns]
#for scale in range(basescale-1, -1, -1):
for scale in range(basescale, -1, -1):
print('Scale', scale)
tiles = 2**scale
for y in range(tiles):
for x in range(tiles):
ims = [
base[y * tilesize:(y + 1) * tilesize,
x * tilesize:(x + 1) * tilesize] for base in bases
]
if opt.kind == 'sdss':
bands = 'gri'
rgb = sdss_rgb(ims, bands)
else:
rgb = get_rgb(ims, bands, **rgbkwargs)
pp = patdata.copy()
pp.update(zoom=scale, x=x, y=y)
fn = pat % pp
trymakedirs(fn)
save_jpeg(fn, rgb)
print('Wrote', fn)
for i, base in enumerate(bases):
#base = gaussian_filter(base, 1.)
base = (base[::2, ::2] + base[1::2, ::2] + base[1::2, 1::2] +
base[::2, 1::2]) / 4.
bases[i] = base
def _tractor_coadds(galaxycat,
targetwcs,
tims,
mods,
version_header,
objid=None,
brickname=None,
survey=None,
mp=None,
verbose=False,
bands=['g', 'r', 'z']):
"""Generate individual-band FITS and color coadds for each central using
Tractor.
"""
from legacypipe.coadds import make_coadds, write_coadd_images
from legacypipe.runbrick import rgbkwargs, rgbkwargs_resid
from legacypipe.survey import get_rgb, imsave_jpeg
if brickname is None:
brickname = galaxycat['brickname']
if verbose:
print('Producing coadds...')
C = make_coadds(tims,
bands,
targetwcs,
mods=mods,
mp=mp,
callback=write_coadd_images,
callback_args=(survey, brickname, version_header, tims,
targetwcs))
# Move (rename) the coadds into the desired output directory.
for suffix in ('chi2', 'image', 'invvar', 'model'):
for band in bands:
oldfile = os.path.join(
survey.output_dir, 'coadd', brickname[:3], brickname,
'legacysurvey-{}-{}-{}.fits.fz'.format(brickname, suffix,
band))
newfile = os.path.join(
survey.output_dir,
'{}-{}-{}.fits.fz'.format(objid, suffix, band))
shutil.copy(oldfile, newfile)
if True:
shutil.rmtree(os.path.join(survey.output_dir, 'coadd'))
# Build png postage stamps of the coadds.
coadd_list = [('image', C.coimgs, rgbkwargs),
('model', C.comods, rgbkwargs),
('resid', C.coresids, rgbkwargs_resid)]
for name, ims, rgbkw in coadd_list:
rgb = get_rgb(ims, bands, **rgbkw)
kwa = {}
outfn = os.path.join(survey.output_dir,
'{}-{}.jpg'.format(objid, name))
if verbose:
print('Writing {}'.format(outfn))
imsave_jpeg(outfn, rgb, origin='lower', **kwa)
del rgb
def make_coadds(tims,
bands,
targetwcs,
mods=None,
xy=None,
apertures=None,
apxy=None,
ngood=False,
detmaps=False,
psfsize=False,
allmasks=True,
max=False,
sbscale=True,
callback=None,
callback_args=None,
plots=False,
ps=None,
lanczos=True,
mp=None,
satur_val=10.):
from astrometry.util.ttime import Time
t0 = Time()
if callback_args is None:
callback_args = []
class Duck(object):
pass
C = Duck()
W = int(targetwcs.get_width())
H = int(targetwcs.get_height())
# always, for patching SATUR, etc pixels?
unweighted = True
C.coimgs = []
# the pixelwise inverse-variances (weights) of the "coimgs".
C.cowimgs = []
if detmaps:
C.galdetivs = []
C.psfdetivs = []
if mods is not None:
C.comods = []
C.coresids = []
if apertures is not None:
unweighted = True
C.AP = fits_table()
if allmasks is not None:
C.allmasks = []
if max:
C.maximgs = []
if xy:
ix, iy = xy
C.T = fits_table()
C.T.nobs = np.zeros((len(ix), len(bands)), np.int16)
C.T.anymask = np.zeros((len(ix), len(bands)), np.int16)
C.T.allmask = np.zeros((len(ix), len(bands)), np.int16)
if psfsize:
C.T.psfsize = np.zeros((len(ix), len(bands)), np.float32)
if detmaps:
C.T.psfdepth = np.zeros((len(ix), len(bands)), np.float32)
C.T.galdepth = np.zeros((len(ix), len(bands)), np.float32)
if lanczos:
debug('Doing Lanczos resampling')
for tim in tims:
# surface-brightness correction
tim.sbscale = (targetwcs.pixel_scale() / tim.subwcs.pixel_scale())**2
# We create one iterator per band to do the tim resampling. These all run in
# parallel when multi-processing.
imaps = []
for band in bands:
args = []
for itim, tim in enumerate(tims):
if tim.band != band:
continue
if mods is None:
mo = None
else:
mo = mods[itim]
args.append((itim, tim, mo, lanczos, targetwcs, sbscale))
if mp is not None:
imaps.append(mp.imap_unordered(_resample_one, args))
else:
imaps.append(map(_resample_one, args))
# Args for aperture photometry
apargs = []
if xy:
# To save the memory of 2 x float64 maps, we instead do arg min/max maps
# append a 0 to the list of mjds so that mjds[-1] gives 0.
mjds = np.array([tim.time.toMjd() for tim in tims] + [0])
mjd_argmins = np.empty((H, W), np.int16)
mjd_argmaxs = np.empty((H, W), np.int16)
mjd_argmins[:, :] = -1
mjd_argmaxs[:, :] = -1
if plots:
allresids = []
tinyw = 1e-30
for iband, (band, timiter) in enumerate(zip(bands, imaps)):
debug('Computing coadd for band', band)
# coadded weight map (moo)
cow = np.zeros((H, W), np.float32)
# coadded weighted image map
cowimg = np.zeros((H, W), np.float32)
kwargs = dict(cowimg=cowimg, cow=cow)
if detmaps:
# detection map inverse-variance (depth map)
psfdetiv = np.zeros((H, W), np.float32)
C.psfdetivs.append(psfdetiv)
kwargs.update(psfdetiv=psfdetiv)
# galaxy detection map inverse-variance (galdepth map)
galdetiv = np.zeros((H, W), np.float32)
C.galdetivs.append(galdetiv)
kwargs.update(galdetiv=galdetiv)
if mods is not None:
# model image
cowmod = np.zeros((H, W), np.float32)
# chi-squared image
cochi2 = np.zeros((H, W), np.float32)
kwargs.update(cowmod=cowmod, cochi2=cochi2)
if unweighted:
# unweighted image
coimg = np.zeros((H, W), np.float32)
if mods is not None:
# unweighted model
comod = np.zeros((H, W), np.float32)
# number of exposures
con = np.zeros((H, W), np.int16)
# inverse-variance
coiv = np.zeros((H, W), np.float32)
kwargs.update(coimg=coimg, coiv=coiv)
# Note that we have 'congood' as well as 'nobs':
# * 'congood' is used for the 'nexp' *image*.
# It counts the number of "good" (unmasked) exposures
# * 'nobs' is used for the per-source measurements
# It counts the total number of exposures, including masked pixels
#
# (you want to know the number of observations within the
# source footprint, not just the peak pixel which may be
# saturated, etc.)
if ngood:
congood = np.zeros((H, W), np.int16)
kwargs.update(congood=congood)
if xy or allmasks:
# These match the type of the "DQ" images.
# "any" mask
ormask = np.zeros((H, W), np.int16)
# "all" mask
andmask = np.empty((H, W), np.int16)
from functools import reduce
allbits = reduce(np.bitwise_or, DQ_BITS.values())
andmask[:, :] = allbits
kwargs.update(ormask=ormask, andmask=andmask)
if xy:
# number of observations
nobs = np.zeros((H, W), np.int16)
kwargs.update(nobs=nobs)
if psfsize:
psfsizemap = np.zeros((H, W), np.float32)
# like "cow", but constant invvar per-CCD;
# only required for psfsizemap
flatcow = np.zeros((H, W), np.float32)
kwargs.update(psfsize=psfsizemap)
if max:
maximg = np.zeros((H, W), np.float32)
C.maximgs.append(maximg)
for R in timiter:
if R is None:
continue
itim, Yo, Xo, iv, im, mo, dq = R
tim = tims[itim]
if plots:
from legacypipe.runbrick import rgbkwargs
from legacypipe.survey import get_rgb
import pylab as plt
# # Make one grayscale, brick-space plot per image
# thisimg = np.zeros((H,W), np.float32)
# thisimg[Yo,Xo] = im
# rgb = get_rgb([thisimg], [band], **rgbkwargs)
# rgb = rgb.sum(axis=2)
# fn = ps.getnext()
# plt.imsave(fn, rgb, origin='lower', cmap='gray')
#plt.clf()
#plt.imshow(rgb, interpolation='nearest', origin='lower', cmap='gray')
#plt.xticks([]); plt.yticks([])
#ps.savefig()
# Image, Model, and Resids
plt.clf()
plt.subplot(2, 2, 1)
thisimg = np.zeros((H, W), np.float32)
thisimg[Yo, Xo] = im
rgb = get_rgb([thisimg], [band], **rgbkwargs)
iplane = dict(g=2, r=1, z=0)[band]
rgbimg = rgb[:, :, iplane]
plt.imshow(rgbimg,
interpolation='nearest',
origin='lower',
cmap='gray')
plt.xticks([])
plt.yticks([])
if mods is not None:
plt.subplot(2, 2, 2)
thismod = np.zeros((H, W), np.float32)
thismod[Yo, Xo] = mo
rgb = get_rgb([thismod], [band], **rgbkwargs)
rgbmod = rgb[:, :, iplane]
plt.imshow(rgbmod,
interpolation='nearest',
origin='lower',
cmap='gray')
plt.xticks([])
plt.yticks([])
plt.subplot(2, 2, 3)
thisres = np.zeros((H, W), np.float32)
thisres[Yo, Xo] = (im - mo) * np.sqrt(iv)
plt.imshow(thisres,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-20,
vmax=20)
plt.xticks([])
plt.yticks([])
else:
if unweighted and (dq is not None):
# HACK -- copy-n-pasted code from below.
okbits = 0
#for bitname in ['satur', 'bleed']:
for bitname in ['satur']:
okbits |= DQ_BITS[bitname]
brightpix = ((dq & okbits) != 0)
myim = im.copy()
if satur_val is not None:
# HACK -- force SATUR pix to be bright
myim[brightpix] = satur_val
#for bitname in ['interp']:
for bitname in ['interp', 'bleed']:
okbits |= DQ_BITS[bitname]
goodpix = ((dq & ~okbits) == 0)
thisgood = np.zeros((H, W), np.float32)
thisgood[Yo, Xo] = goodpix
plt.subplot(2, 2, 2)
plt.imshow(thisgood,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=0,
vmax=1)
plt.xticks([])
plt.yticks([])
plt.title('goodpix')
thisim = np.zeros((H, W), np.float32)
thisim[Yo, Xo] = goodpix * myim
rgb = get_rgb([thisim], [band], **rgbkwargs)
iplane = dict(g=2, r=1, z=0)[band]
rgbimg = rgb[:, :, iplane]
plt.subplot(2, 2, 3)
plt.imshow(rgbimg,
interpolation='nearest',
origin='lower',
cmap='gray')
plt.xticks([])
plt.yticks([])
plt.title('goodpix rgb')
rgbmod = None
thisres = None
plt.subplot(2, 2, 4)
thisiv = np.zeros((H, W), np.float32)
thisiv[Yo, Xo] = iv
plt.imshow(thisiv,
interpolation='nearest',
origin='lower',
cmap='gray')
plt.xticks([])
plt.yticks([])
plt.title('invvar')
plt.suptitle(tim.name + ': %.2f' % (tim.time.toYear()))
ps.savefig()
allresids.append(
(tim.time.toYear(), tim.name, rgbimg, rgbmod, thisres))
# invvar-weighted image
cowimg[Yo, Xo] += iv * im
cow[Yo, Xo] += iv
if unweighted:
if dq is None:
goodpix = 1
else:
# include SATUR pixels if no other
# pixels exists
okbits = 0
for bitname in ['satur']:
okbits |= DQ_BITS[bitname]
brightpix = ((dq & okbits) != 0)
if satur_val is not None:
# HACK -- force SATUR pix to be bright
im[brightpix] = satur_val
# Include these pixels if none other exist??
for bitname in ['interp']: #, 'bleed']:
okbits |= DQ_BITS[bitname]
goodpix = ((dq & ~okbits) == 0)
coimg[Yo, Xo] += goodpix * im
con[Yo, Xo] += goodpix
coiv[Yo, Xo] += goodpix * 1. / (tim.sig1 *
tim.sbscale)**2 # ...ish
if xy:
if dq is not None:
ormask[Yo, Xo] |= dq
andmask[Yo, Xo] &= dq
# raw exposure count
nobs[Yo, Xo] += 1
# mjd_min/max
update = np.logical_or(
mjd_argmins[Yo, Xo] == -1, (mjd_argmins[Yo, Xo] > -1) *
(mjds[itim] < mjds[mjd_argmins[Yo, Xo]]))
mjd_argmins[Yo[update], Xo[update]] = itim
update = np.logical_or(
mjd_argmaxs[Yo, Xo] == -1, (mjd_argmaxs[Yo, Xo] > -1) *
(mjds[itim] > mjds[mjd_argmaxs[Yo, Xo]]))
mjd_argmaxs[Yo[update], Xo[update]] = itim
del update
if psfsize:
# psfnorm is in units of 1/pixels.
# (eg, psfnorm for a gaussian is ~ 1/psf_sigma)
# Neff is in pixels**2
neff = 1. / tim.psfnorm**2
# Narcsec is in arcsec**2
narcsec = neff * tim.wcs.pixel_scale()**2
# Make smooth maps -- don't ignore CRs, saturated pix, etc
#psfsizemap[Yo,Xo] += (iv>0) * (1/tim.sig1**2) * (1. / narcsec)
#flatcow[Yo,Xo] += (iv>0) * (1/tim.sig1**2)
iv1 = 1. / tim.sig1**2
psfsizemap[Yo, Xo] += iv1 * (1. / narcsec)
flatcow[Yo, Xo] += iv1
if detmaps:
# point-source depth
detsig1 = tim.sig1 / tim.psfnorm
psfdetiv[Yo, Xo] += (iv > 0) * (1. / detsig1**2)
# Galaxy detection map
gdetsig1 = tim.sig1 / tim.galnorm
galdetiv[Yo, Xo] += (iv > 0) * (1. / gdetsig1**2)
if ngood:
congood[Yo, Xo] += (iv > 0)
if mods is not None:
# straight-up
comod[Yo, Xo] += goodpix * mo
# invvar-weighted
cowmod[Yo, Xo] += iv * mo
# chi-squared
cochi2[Yo, Xo] += iv * (im - mo)**2
del mo
del goodpix
if max:
maximg[Yo, Xo] = np.maximum(maximg[Yo, Xo], im * (iv > 0))
del Yo, Xo, im, iv
# END of loop over tims
# Per-band:
cowimg /= np.maximum(cow, tinyw)
C.coimgs.append(cowimg)
C.cowimgs.append(cow)
if mods is not None:
cowmod /= np.maximum(cow, tinyw)
C.comods.append(cowmod)
coresid = cowimg - cowmod
coresid[cow == 0] = 0.
C.coresids.append(coresid)
if allmasks:
C.allmasks.append(andmask)
if unweighted:
coimg /= np.maximum(con, 1)
del con
if plots:
plt.clf()
plt.subplot(2, 2, 1)
mn, mx = cowimg.min(), cowimg.max()
plt.imshow(cowimg,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=mn,
vmax=mx)
plt.xticks([])
plt.yticks([])
plt.title('weighted img')
plt.subplot(2, 2, 2)
mycow = cow.copy()
# mark zero as special color
#mycow[mycow == 0] = np.nan
plt.imshow(mycow,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=0)
plt.xticks([])
plt.yticks([])
plt.title('weights')
plt.subplot(2, 2, 3)
plt.imshow(coimg,
interpolation='nearest',
origin='lower',
cmap='gray')
plt.xticks([])
plt.yticks([])
plt.title('unweighted img')
mycowimg = cowimg.copy()
mycowimg[cow == 0] = coimg[cow == 0]
plt.subplot(2, 2, 4)
plt.imshow(mycowimg,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=mn,
vmax=mx)
plt.xticks([])
plt.yticks([])
plt.title('patched img')
plt.suptitle('band %s' % band)
ps.savefig()
cowimg[cow == 0] = coimg[cow == 0]
if mods is not None:
cowmod[cow == 0] = comod[cow == 0]
if xy:
C.T.nobs[:, iband] = nobs[iy, ix]
C.T.anymask[:, iband] = ormask[iy, ix]
C.T.allmask[:, iband] = andmask[iy, ix]
# unless there were no images there...
C.T.allmask[nobs[iy, ix] == 0, iband] = 0
if detmaps:
C.T.psfdepth[:, iband] = psfdetiv[iy, ix]
C.T.galdepth[:, iband] = galdetiv[iy, ix]
if psfsize:
# psfsizemap is accumulated in units of iv * (1 / arcsec**2)
# take out the weighting
psfsizemap /= np.maximum(flatcow, tinyw)
# Correction factor to get back to equivalent of Gaussian sigma
tosigma = 1. / (2. * np.sqrt(np.pi))
# Conversion factor to FWHM (2.35)
tofwhm = 2. * np.sqrt(2. * np.log(2.))
# Scale back to units of linear arcsec.
psfsizemap[:, :] = (1. / np.sqrt(psfsizemap)) * tosigma * tofwhm
psfsizemap[flatcow == 0] = 0.
if xy:
C.T.psfsize[:, iband] = psfsizemap[iy, ix]
if apertures is not None:
# Aperture photometry, using the unweighted "coimg" and
# "coiv" arrays.
with np.errstate(divide='ignore'):
imsigma = 1.0 / np.sqrt(coiv)
imsigma[coiv == 0] = 0
for irad, rad in enumerate(apertures):
apargs.append((irad, band, rad, coimg, imsigma, True, apxy))
if mods is not None:
apargs.append(
(irad, band, rad, coresid, None, False, apxy))
if callback is not None:
callback(band, *callback_args, **kwargs)
# END of loop over bands
t2 = Time()
debug('coadds: images:', t2 - t0)
if plots:
I = np.argsort([a[0] for a in allresids])
cols = int(np.ceil(np.sqrt(len(I))))
rows = int(np.ceil(len(I) / float(cols)))
allresids = [allresids[i] for i in I]
plt.clf()
for i, (y, n, img, mod, res) in enumerate(allresids):
plt.subplot(rows, cols, i + 1)
plt.imshow(img,
interpolation='nearest',
origin='lower',
cmap='gray')
plt.xticks([])
plt.yticks([])
plt.title('%.1f: %s' % (y, n))
plt.suptitle('Data')
ps.savefig()
if mods is not None:
plt.clf()
for i, (y, n, img, mod, res) in enumerate(allresids):
plt.subplot(rows, cols, i + 1)
plt.imshow(mod,
interpolation='nearest',
origin='lower',
cmap='gray')
plt.xticks([])
plt.yticks([])
plt.title('%.1f: %s' % (y, n))
plt.suptitle('Model')
ps.savefig()
plt.clf()
for i, (y, n, img, mod, res) in enumerate(allresids):
plt.subplot(rows, cols, i + 1)
plt.imshow(res,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-20,
vmax=20)
plt.xticks([])
plt.yticks([])
plt.title('%.1f: %s' % (y, n))
plt.suptitle('Resids')
ps.savefig()
if xy is not None:
C.T.mjd_min = mjds[mjd_argmins[iy, ix]]
C.T.mjd_max = mjds[mjd_argmaxs[iy, ix]]
del mjd_argmins
del mjd_argmaxs
if apertures is not None:
# Aperture phot, in parallel
if mp is not None:
apresults = mp.map(_apphot_one, apargs)
else:
apresults = map(_apphot_one, apargs)
del apargs
apresults = iter(apresults)
for iband, band in enumerate(bands):
apimg = []
apimgerr = []
if mods is not None:
apres = []
for irad, rad in enumerate(apertures):
# py2
if hasattr(apresults, 'next'):
(airad, aband, isimg, ap_img, ap_err) = apresults.next()
# py3
else:
(airad, aband, isimg, ap_img,
ap_err) = apresults.__next__()
assert (airad == irad)
assert (aband == band)
assert (isimg)
apimg.append(ap_img)
apimgerr.append(ap_err)
if mods is not None:
# py2
if hasattr(apresults, 'next'):
(airad, aband, isimg, ap_img,
ap_err) = apresults.next()
else:
(airad, aband, isimg, ap_img,
ap_err) = apresults.__next__()
assert (airad == irad)
assert (aband == band)
assert (not isimg)
apres.append(ap_img)
assert (ap_err is None)
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
C.AP.set('apflux_img_%s' % band, ap)
ap = 1. / (np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
C.AP.set('apflux_img_ivar_%s' % band, ap)
if mods is not None:
ap = np.vstack(apres).T
ap[np.logical_not(np.isfinite(ap))] = 0.
C.AP.set('apflux_resid_%s' % band, ap)
t3 = Time()
debug('coadds apphot:', t3 - t2)
return C
def _tractor_coadds(sample,
targetwcs,
tims,
mods,
version_header,
objid=None,
brickname=None,
survey=None,
mp=None,
log=None,
bands=['g', 'r', 'z']):
"""Generate individual-band FITS and color coadds for each central using
Tractor.
"""
from legacypipe.coadds import make_coadds, write_coadd_images
#from legacypipe.runbrick import rgbkwargs, rgbkwargs_resid
from legacypipe.survey import get_rgb, imsave_jpeg
if brickname is None:
brickname = sample['brickname']
print('Producing coadds...', flush=True, file=log)
if log:
with redirect_stdout(log), redirect_stderr(log):
C = make_coadds(tims,
bands,
targetwcs,
mods=mods,
mp=mp,
callback=write_coadd_images,
callback_args=(survey, brickname, version_header,
tims, targetwcs))
else:
C = make_coadds(tims,
bands,
targetwcs,
mods=mods,
mp=mp,
callback=write_coadd_images,
callback_args=(survey, brickname, version_header, tims,
targetwcs))
# Move (rename) the coadds into the desired output directory.
for suffix in np.atleast_1d('model'):
for band in bands:
shutil.copy(
os.path.join(
survey.output_dir, 'coadd', brickname[:3], brickname,
'legacysurvey-{}-{}-{}.fits.fz'.format(
brickname, suffix, band)),
os.path.join(
survey.output_dir,
'{}-{}-nocentral-{}.fits.fz'.format(objid, suffix, band)))
if True:
shutil.rmtree(os.path.join(survey.output_dir, 'coadd'))
# Build png postage stamps of the coadds.
rgbkwargs = dict(mnmx=(-1, 100), arcsinh=1)
#rgbkwargs_resid = dict(mnmx=(0.1, 2), arcsinh=1)
rgbkwargs_resid = dict(mnmx=(-1, 100), arcsinh=1)
#coadd_list = [('image-central', C.coimgs, rgbkwargs),
# ('model-central', C.comods, rgbkwargs),
# ('resid-central', C.coresids, rgbkwargs_resid)]
coadd_list = [('model-nocentral', C.comods, rgbkwargs),
('image-central', C.coresids, rgbkwargs_resid)]
for name, ims, rgbkw in coadd_list:
rgb = get_rgb(ims, bands, **rgbkw)
kwa = {}
outfn = os.path.join(survey.output_dir,
'{}-{}.jpg'.format(objid, name))
print('Writing {}'.format(outfn), flush=True, file=log)
imsave_jpeg(outfn, rgb, origin='lower', **kwa)
del rgb
def _make_coadds_plots_1(im, band, mods, mo, iv, unweighted,
dq, satur_val, allresids, ps, H, W,
tim, Yo, Xo):
from legacypipe.survey import get_rgb
import pylab as plt
# # Make one grayscale, brick-space plot per image
# thisimg = np.zeros((H,W), np.float32)
# thisimg[Yo,Xo] = im
# rgb = get_rgb([thisimg], [band])
# rgb = rgb.sum(axis=2)
# fn = ps.getnext()
# plt.imsave(fn, rgb, origin='lower', cmap='gray')
#plt.clf()
#plt.imshow(rgb, interpolation='nearest', origin='lower', cmap='gray')
#plt.xticks([]); plt.yticks([])
#ps.savefig()
# Image, Model, and Resids
plt.clf()
plt.subplot(2,2,1)
thisimg = np.zeros((H,W), np.float32)
thisimg[Yo,Xo] = im
rgb = get_rgb([thisimg], [band])
iplane = dict(g=2, r=1, z=0)[band]
rgbimg = rgb[:,:,iplane]
plt.imshow(rgbimg, interpolation='nearest', origin='lower', cmap='gray')
plt.xticks([]); plt.yticks([])
if mods is not None:
plt.subplot(2,2,2)
thismod = np.zeros((H,W), np.float32)
thismod[Yo,Xo] = mo
rgb = get_rgb([thismod], [band])
rgbmod = rgb[:,:,iplane]
plt.imshow(rgbmod, interpolation='nearest', origin='lower', cmap='gray')
plt.xticks([]); plt.yticks([])
plt.subplot(2,2,3)
thisres = np.zeros((H,W), np.float32)
thisres[Yo,Xo] = (im - mo) * np.sqrt(iv)
plt.imshow(thisres, interpolation='nearest', origin='lower', cmap='gray',
vmin=-20, vmax=20)
plt.xticks([]); plt.yticks([])
else:
if unweighted and (dq is not None):
# HACK -- copy-n-pasted code from below.
okbits = 0
#for bitname in ['satur', 'bleed']:
for bitname in ['satur']:
okbits |= DQ_BITS[bitname]
brightpix = ((dq & okbits) != 0)
myim = im.copy()
if satur_val is not None:
# HACK -- force SATUR pix to be bright
myim[brightpix] = satur_val
#for bitname in ['interp']:
for bitname in ['interp', 'bleed']:
okbits |= DQ_BITS[bitname]
goodpix = ((dq & ~okbits) == 0)
thisgood = np.zeros((H,W), np.float32)
thisgood[Yo,Xo] = goodpix
plt.subplot(2,2,2)
plt.imshow(thisgood, interpolation='nearest', origin='lower', cmap='gray', vmin=0, vmax=1)
plt.xticks([]); plt.yticks([])
plt.title('goodpix')
thisim = np.zeros((H,W), np.float32)
thisim[Yo,Xo] = goodpix * myim
rgb = get_rgb([thisim], [band])
iplane = dict(g=2, r=1, z=0)[band]
rgbimg = rgb[:,:,iplane]
plt.subplot(2,2,3)
plt.imshow(rgbimg, interpolation='nearest', origin='lower', cmap='gray')
plt.xticks([]); plt.yticks([])
plt.title('goodpix rgb')
rgbmod=None
thisres=None
plt.subplot(2,2,4)
thisiv = np.zeros((H,W), np.float32)
thisiv[Yo,Xo] = iv
plt.imshow(thisiv, interpolation='nearest', origin='lower', cmap='gray')
plt.xticks([]); plt.yticks([])
plt.title('invvar')
plt.suptitle(tim.name + ': %.2f' % (tim.time.toYear()))
ps.savefig()
allresids.append((tim.time.toYear(), tim.name, rgbimg,rgbmod,thisres))
def stage_fit_on_coadds(survey=None,
targetwcs=None,
pixscale=None,
bands=None,
tims=None,
brickname=None,
version_header=None,
coadd_tiers=None,
apodize=True,
subsky=True,
ubercal_sky=False,
subsky_radii=None,
nsatur=None,
fitoncoadds_reweight_ivar=True,
plots=False,
plots2=False,
ps=None,
coadd_bw=False,
W=None,
H=None,
brick=None,
blobs=None,
lanczos=True,
ccds=None,
write_metrics=True,
mp=None,
record_event=None,
**kwargs):
from legacypipe.coadds import make_coadds
from legacypipe.bits import DQ_BITS
from legacypipe.survey import LegacySurveyWcs
from legacypipe.coadds import get_coadd_headers
from tractor.image import Image
from tractor.basics import LinearPhotoCal
from tractor.sky import ConstantSky
from tractor.psf import PixelizedPSF
from tractor.tractortime import TAITime
import astropy.time
import fitsio
if plots or plots2:
import pylab as plt
from legacypipe.survey import get_rgb
# Custom sky-subtraction for large galaxies.
skydict = {}
if not subsky:
if ubercal_sky:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('fitoncoadds-{}'.format(brickname))
tims, skydict = ubercal_skysub(tims,
targetwcs,
survey,
brickname,
bands,
mp,
subsky_radii=subsky_radii,
plots=True,
plots2=False,
ps=ps,
verbose=True)
else:
print('Skipping sky-subtraction entirely.')
# Create coadds and then build custom tims from them.
for tim in tims:
ie = tim.inverr
if np.any(ie < 0):
print('Negative inverse error in image {}'.format(tim.name))
CC = []
if coadd_tiers:
# Sort by band and sort them into tiers.
tiers = [[] for i in range(coadd_tiers)]
for b in bands:
btims = []
seeing = []
for tim in tims:
if tim.band != b:
continue
btims.append(tim)
seeing.append(tim.psf_fwhm * tim.imobj.pixscale)
I = np.argsort(seeing)
btims = [btims[i] for i in I]
seeing = [seeing[i] for i in I]
N = min(coadd_tiers, len(btims))
splits = np.round(np.arange(N + 1) * float(len(btims)) /
N).astype(int)
print('Splitting', len(btims), 'images into', N, 'tiers: splits:',
splits)
print('Seeing limits:',
[seeing[min(s,
len(seeing) - 1)] for s in splits])
for s0, s1, tt in zip(splits, splits[1:], tiers):
tt.extend(btims[s0:s1])
for itier, tier in enumerate(tiers):
print('Producing coadds for tier', (itier + 1))
C = make_coadds(
tier,
bands,
targetwcs,
detmaps=True,
ngood=True,
lanczos=lanczos,
allmasks=True,
anymasks=True,
psf_images=True,
nsatur=2,
mp=mp,
plots=plots2,
ps=ps, # note plots2 here!
callback=None)
if plots:
plt.clf()
for iband, (band, psf) in enumerate(zip(bands, C.psf_imgs)):
plt.subplot(1, len(bands), iband + 1)
plt.imshow(psf, interpolation='nearest', origin='lower')
plt.title('Coadd PSF image: band %s' % band)
plt.suptitle('Tier %i' % (itier + 1))
ps.savefig()
# for band,img in zip(bands, C.coimgs):
# plt.clf()
# plt.imshow(img,
plt.clf()
plt.imshow(get_rgb(C.coimgs, bands), origin='lower')
plt.title('Tier %i' % (itier + 1))
ps.savefig()
CC.append(C)
else:
C = make_coadds(
tims,
bands,
targetwcs,
detmaps=True,
ngood=True,
lanczos=lanczos,
allmasks=True,
anymasks=True,
psf_images=True,
mp=mp,
plots=plots2,
ps=ps, # note plots2 here!
callback=None)
CC.append(C)
cotims = []
for C in CC:
if plots2:
for band, iv in zip(bands, C.cowimgs):
pass
# plt.clf()
# plt.imshow(np.sqrt(iv), interpolation='nearest', origin='lower')
# plt.title('Coadd Inverr: band %s' % band)
# ps.savefig()
for band, psf in zip(bands, C.psf_imgs):
plt.clf()
plt.imshow(psf, interpolation='nearest', origin='lower')
plt.title('Coadd PSF image: band %s' % band)
ps.savefig()
for band, img, iv in zip(bands, C.coimgs, C.cowimgs):
from scipy.ndimage.filters import gaussian_filter
# plt.clf()
# plt.hist((img * np.sqrt(iv))[iv>0], bins=50, range=(-5,8), log=True)
# plt.title('Coadd pixel values (sigmas): band %s' % band)
# ps.savefig()
psf_sigma = np.mean([
(tim.psf_sigma * tim.imobj.pixscale / pixscale)
for tim in tims if tim.band == band
])
gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
psfnorm = gnorm #np.sqrt(np.sum(psfimg**2))
detim = gaussian_filter(img, psf_sigma) / psfnorm**2
cosig1 = 1. / np.sqrt(np.median(iv[iv > 0]))
detsig1 = cosig1 / psfnorm
# plt.clf()
# plt.subplot(2,1,1)
# plt.hist(detim.ravel() / detsig1, bins=50, range=(-5,8), log=True)
# plt.title('Coadd detection map values / sig1 (sigmas): band %s' % band)
# plt.subplot(2,1,2)
# plt.hist(detim.ravel() / detsig1, bins=50, range=(-5,8))
# ps.savefig()
# # as in detection.py
# detiv = np.zeros_like(detim) + (1. / detsig1**2)
# detiv[iv == 0] = 0.
# detiv = gaussian_filter(detiv, psf_sigma)
#
# plt.clf()
# plt.hist((detim * np.sqrt(detiv)).ravel(), bins=50, range=(-5,8), log=True)
# plt.title('Coadd detection map values / detie (sigmas): band %s' % band)
# ps.savefig()
for iband, (band, img, iv, allmask, anymask, psfimg) in enumerate(
zip(bands, C.coimgs, C.cowimgs, C.allmasks, C.anymasks,
C.psf_imgs)):
mjd = np.mean(
[tim.imobj.mjdobs for tim in tims if tim.band == band])
mjd_tai = astropy.time.Time(mjd, format='mjd', scale='utc').tai.mjd
tai = TAITime(None, mjd=mjd_tai)
twcs = LegacySurveyWcs(targetwcs, tai)
#print('PSF sigmas (in pixels) for band', band, ':',
# ['%.2f' % tim.psf_sigma for tim in tims if tim.band == band])
print(
'PSF sigmas in coadd pixels:', ', '.join([
'%.2f' % (tim.psf_sigma * tim.imobj.pixscale / pixscale)
for tim in tims if tim.band == band
]))
psf_sigma = np.mean([
(tim.psf_sigma * tim.imobj.pixscale / pixscale) for tim in tims
if tim.band == band
])
print('Using average PSF sigma', psf_sigma)
psf = PixelizedPSF(psfimg)
gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
psfnorm = np.sqrt(np.sum(psfimg**2))
print('Gaussian PSF norm', gnorm, 'vs pixelized', psfnorm)
# if plots:
# from collections import Counter
# plt.clf()
# plt.imshow(mask, interpolation='nearest', origin='lower')
# plt.colorbar()
# plt.title('allmask')
# ps.savefig()
# print('allmask for band', band, ': values:', Counter(mask.ravel()))
# Scale invvar to take into account that we have resampled (~double-counted) pixels
tim_pixscale = np.mean(
[tim.imobj.pixscale for tim in tims if tim.band == band])
cscale = tim_pixscale / pixscale
print('average tim pixel scale / coadd scale:', cscale)
iv /= cscale**2
if fitoncoadds_reweight_ivar:
# We first tried setting the invvars constant per tim -- this
# makes things worse, since we *remove* the lowered invvars at
# the cores of galaxies.
#
# Here we're hacking the relative weights -- squaring the
# weights but then making the median the same, ie, squaring
# the dynamic range or relative weights -- ie, downweighting
# the cores even more than they already are from source
# Poisson terms.
median_iv = np.median(iv[iv > 0])
assert (median_iv > 0)
iv = iv * np.sqrt(iv) / np.sqrt(median_iv)
assert (np.all(np.isfinite(iv)))
assert (np.all(iv >= 0))
cotim = Image(img,
invvar=iv,
wcs=twcs,
psf=psf,
photocal=LinearPhotoCal(1., band=band),
sky=ConstantSky(0.),
name='coadd-' + band)
cotim.band = band
cotim.subwcs = targetwcs
cotim.psf_sigma = psf_sigma
cotim.sig1 = 1. / np.sqrt(np.median(iv[iv > 0]))
# Often, SATUR masks on galaxies / stars are surrounded by BLEED pixels. Soak these into
# the SATUR mask.
from scipy.ndimage.morphology import binary_dilation
anymask |= np.logical_and(((anymask & DQ_BITS['bleed']) > 0),
binary_dilation(
((anymask & DQ_BITS['satur']) > 0),
iterations=10)) * DQ_BITS['satur']
# Saturated in any image -> treat as saturated in coadd
# (otherwise you get weird systematics in the weighted coadds, and weird source detection!)
mask = allmask
mask[(anymask & DQ_BITS['satur'] > 0)] |= DQ_BITS['satur']
if coadd_tiers:
# nsatur -- reset SATUR bit
mask &= ~DQ_BITS['satur']
mask |= DQ_BITS['satur'] * C.satmaps[iband]
cotim.dq = mask
cotim.dq_saturation_bits = DQ_BITS['satur']
cotim.psfnorm = gnorm
cotim.galnorm = 1.0 # bogus!
cotim.imobj = Duck()
cotim.imobj.fwhm = 2.35 * psf_sigma
cotim.imobj.pixscale = pixscale
cotim.time = tai
cotim.primhdr = fitsio.FITSHDR()
get_coadd_headers(cotim.primhdr, tims, band, coadd_headers=skydict)
cotims.append(cotim)
if plots:
plt.clf()
bitmap = dict([(v, k) for k, v in DQ_BITS.items()])
k = 1
for i in range(12):
bitval = 1 << i
if not bitval in bitmap:
continue
# only 9 bits are actually used
plt.subplot(3, 3, k)
k += 1
plt.imshow((cotim.dq & bitval) > 0,
vmin=0,
vmax=1.5,
cmap='hot',
origin='lower')
plt.title(bitmap[bitval])
plt.suptitle('Coadd mask planes %s band' % band)
ps.savefig()
plt.clf()
h, w = cotim.shape
rgb = np.zeros((h, w, 3), np.uint8)
rgb[:, :, 0] = (cotim.dq & DQ_BITS['satur'] > 0) * 255
rgb[:, :, 1] = (cotim.dq & DQ_BITS['bleed'] > 0) * 255
plt.imshow(rgb, origin='lower')
plt.suptitle('Coadd DQ band %s: red = SATUR, green = BLEED' %
band)
ps.savefig()
# Save an image of the coadd PSF
# copy version_header before modifying it.
hdr = fitsio.FITSHDR()
for r in version_header.records():
hdr.add_record(r)
hdr.add_record(
dict(name='IMTYPE',
value='coaddpsf',
comment='LegacySurveys image type'))
hdr.add_record(
dict(name='BAND', value=band,
comment='Band of this coadd/PSF'))
hdr.add_record(
dict(name='PSF_SIG',
value=psf_sigma,
comment='Average PSF sigma (coadd pixels)'))
hdr.add_record(
dict(name='PIXSCAL',
value=pixscale,
comment='Pixel scale of this PSF (arcsec)'))
hdr.add_record(
dict(name='INPIXSC',
value=tim_pixscale,
comment='Native image pixscale scale (average, arcsec)'))
hdr.add_record(
dict(name='MJD', value=mjd, comment='Average MJD for coadd'))
hdr.add_record(
dict(name='MJD_TAI',
value=mjd_tai,
comment='Average MJD (in TAI) for coadd'))
with survey.write_output('copsf', brick=brickname,
band=band) as out:
out.fits.write(psfimg, header=hdr)
# EVIL
return dict(tims=cotims, coadd_headers=skydict)
def ubercal_skysub(tims,
targetwcs,
survey,
brickname,
bands,
mp,
subsky_radii=None,
plots=False,
plots2=False,
ps=None,
verbose=False):
"""With the ubercal option, we (1) read the full-field mosaics ('bandtims') for
a given bandpass and put them all on the same 'system' using the overlapping
pixels; (2) apply the derived corrections to the in-field 'tims'; (3) build
the coadds (per bandpass) from the 'tims'; and (4) subtract the median sky
from the mosaic (after aggressively masking objects and reference sources).
"""
from tractor.sky import ConstantSky
from legacypipe.reference import get_reference_sources, get_reference_map
from legacypipe.coadds import make_coadds
from legacypipe.survey import get_rgb, imsave_jpeg
from astropy.stats import sigma_clipped_stats
if plots or plots2:
import os
import matplotlib.pyplot as plt
if plots:
plt.figure(figsize=(8, 6))
mods = []
for tim in tims:
imcopy = tim.getImage().copy()
tim.sky.addTo(imcopy, -1)
mods.append(imcopy)
C = make_coadds(tims,
bands,
targetwcs,
mods=mods,
callback=None,
mp=mp)
imsave_jpeg(os.path.join(survey.output_dir, 'metrics', 'cus',
'{}-pipelinesky.jpg'.format(ps.basefn)),
get_rgb(C.comods, bands),
origin='lower')
skydict = {}
if subsky_radii is not None:
skydict.update(
{'NSKYANN': (len(subsky_radii) // 2, 'number of sky annuli')})
for irad, (rin, rout) in enumerate(
zip(subsky_radii[0::2], subsky_radii[1::2])):
skydict.update({
'SKYRIN{:02d}'.format(irad):
(np.float32(rin), 'inner sky radius {} [arcsec]'.format(irad))
})
skydict.update({
'SKYROT{:02d}'.format(irad):
(np.float32(rout), 'outer sky radius {} [arcsec]'.format(irad))
})
allbands = np.array([tim.band for tim in tims])
# we need the full-field mosaics if doing annular sky-subtraction
if subsky_radii is not None:
allbandtims = []
else:
allbandtims = None
for band in sorted(set(allbands)):
print('Working on band {}'.format(band))
I = np.where(allbands == band)[0]
bandtims = [
tims[ii].imobj.get_tractor_image(gaussPsf=True,
pixPsf=False,
subsky=False,
dq=True,
apodize=False) for ii in I
]
# Derive the ubercal correction and then apply it.
x = coadds_ubercal(bandtims,
coaddtims=[tims[ii] for ii in I],
plots=plots,
plots2=plots2,
ps=ps,
verbose=True)
for ii, corr in zip(I, x):
skydict.update({
'SKCCD{:03d}'.format(ii):
(tims[ii].name, 'ubersky CCD {:03d}'.format(ii))
})
skydict.update({
'SKCOR{:03d}'.format(ii):
(corr, 'ubersky corr CCD {:03d}'.format(ii))
})
# Apply the correction and return the tims
for jj, (correction, ii) in enumerate(zip(x, I)):
tims[ii].data += correction
tims[ii].sky = ConstantSky(0.0)
# Also correct the full-field mosaics
bandtims[jj].data += correction
bandtims[jj].sky = ConstantSky(0.0)
## Check--
#for jj, correction in enumerate(x):
# fulltims[jj].data += correction
#newcorrection = coadds_ubercal(fulltims)
#print(newcorrection)
if allbandtims is not None:
allbandtims = allbandtims + bandtims
# Estimate the sky background from an annulus surrounding the object
# (assumed to be at the center of the mosaic, targetwcs.crval).
if subsky_radii is not None:
from astrometry.util.util import Tan
# the inner and outer radii / annuli are nested in subsky_radii
allrin = subsky_radii[0::2]
allrout = subsky_radii[1::2]
pixscale = targetwcs.pixel_scale()
bigH = float(np.ceil(2 * np.max(allrout) / pixscale))
bigW = bigH
# if doing annulur sky-subtraction we need a bigger mosaic
bigtargetwcs = Tan(targetwcs.crval[0], targetwcs.crval[1],
bigW / 2. + 0.5, bigH / 2. + 0.5,
-pixscale / 3600.0, 0., 0., pixscale / 3600.0,
float(bigW), float(bigH))
C = make_coadds(allbandtims,
bands,
bigtargetwcs,
callback=None,
sbscale=False,
mp=mp)
_, x0, y0 = bigtargetwcs.radec2pixelxy(bigtargetwcs.crval[0],
bigtargetwcs.crval[1])
xcen, ycen = np.round(x0 - 1).astype('int'), np.round(y0 -
1).astype('int')
ymask, xmask = np.ogrid[-ycen:bigH - ycen, -xcen:bigW - xcen]
refs, _ = get_reference_sources(survey,
bigtargetwcs,
bigtargetwcs.pixel_scale(), ['r'],
tycho_stars=True,
gaia_stars=True,
large_galaxies=True,
star_clusters=True)
refmask = get_reference_map(bigtargetwcs, refs) == 0 # True=skypix
for coimg, coiv, band in zip(C.coimgs, C.cowimgs, bands):
skypix = _build_objmask(coimg, coiv, refmask * (coiv > 0))
for irad, (rin, rout) in enumerate(zip(allrin, allrout)):
inmask = (xmask**2 + ymask**2) <= (rin / pixscale)**2
outmask = (xmask**2 + ymask**2) <= (rout / pixscale)**2
skymask = (outmask * 1 - inmask * 1) == 1 # True=skypix
# Find and mask objects, then get the sky.
skypix_annulus = np.logical_and(skypix, skymask)
#import matplotlib.pyplot as plt ; plt.imshow(skypix_annulus, origin='lower') ; plt.savefig('junk3.png')
#import pdb ; pdb.set_trace()
if np.sum(skypix_annulus) == 0:
raise ValueError('No pixels in sky!')
_skymean, _skymedian, _skysig = sigma_clipped_stats(
coimg, mask=np.logical_not(skypix_annulus), sigma=3.0)
skydict.update({
'{}SKYMN{:02d}'.format(band.upper(), irad):
(np.float32(_skymean),
'mean {} sky in annulus {}'.format(band, irad))
})
skydict.update({
'{}SKYMD{:02d}'.format(band.upper(), irad):
(np.float32(_skymedian),
'median {} sky in annulus {}'.format(band, irad))
})
skydict.update({
'{}SKYSG{:02d}'.format(band.upper(), irad):
(np.float32(_skysig),
'sigma {} sky in annulus {}'.format(band, irad))
})
skydict.update({
'{}SKYNP{:02d}'.format(band.upper(), irad):
(np.sum(skypix_annulus),
'npix {} sky in annulus {}'.format(band, irad))
})
# the reference annulus is the first one
if irad == 0:
skymean, skymedian, skysig = _skymean, _skymedian, _skysig
skypix_mask = skypix_annulus
I = np.where(allbands == band)[0]
for ii in I:
tims[ii].data -= skymedian
#print('Tim', tims[ii], 'after subtracting skymedian: median', np.median(tims[ii].data))
else:
# regular mosaic
C = make_coadds(tims,
bands,
targetwcs,
callback=None,
sbscale=False,
mp=mp)
refs, _ = get_reference_sources(survey,
targetwcs,
targetwcs.pixel_scale(), ['r'],
tycho_stars=True,
gaia_stars=True,
large_galaxies=True,
star_clusters=True)
refmask = get_reference_map(targetwcs, refs) == 0 # True=skypix
for coimg, coiv, band in zip(C.coimgs, C.cowimgs, bands):
skypix = refmask * (coiv > 0)
skypix_mask = _build_objmask(coimg, coiv, skypix)
skymean, skymedian, skysig = sigma_clipped_stats(
coimg, mask=np.logical_not(skypix_mask), sigma=3.0)
skydict.update({
'{}SKYMN00'.format(band.upper(), irad):
(np.float32(_skymean), 'mean {} sky'.format(band))
})
skydict.update({
'{}SKYMD00'.format(band.upper(), irad):
(np.float32(_skymedian), 'median {} sky'.format(band))
})
skydict.update({
'{}SKYSG00'.format(band.upper(), irad):
(np.float32(_skysig), 'sigma {} sky'.format(band))
})
skydict.update({
'{}SKYNP00'.format(band.upper(), irad):
(np.sum(skypix_mask), 'npix {} sky'.format(band))
})
I = np.where(allbands == band)[0]
for ii in I:
tims[ii].data -= skymedian
# print('Tim', tims[ii], 'after subtracting skymedian: median', np.median(tims[ii].data))
#print('Band', band, 'Coadd sky:', skymedian)
if plots2:
plt.clf()
plt.hist(coimg.ravel(), bins=50, range=(-3, 3), density=True)
plt.axvline(skymedian, color='k')
for ii in I:
#print('Tim', tims[ii], 'median', np.median(tims[ii].data))
plt.hist((tims[ii].data - skymedian).ravel(),
bins=50,
range=(-3, 3),
histtype='step',
density=True)
plt.title('Band %s: tim pix & skymedian' % band)
ps.savefig()
# Produce skymedian-subtracted, masked image for later RGB plot
coimg -= skymedian
coimg[~skypix_mask] = 0.
#coimg[np.logical_not(skymask * (coiv > 0))] = 0.
if plots2:
plt.clf()
plt.imshow(get_rgb(C.coimgs, bands),
origin='lower',
interpolation='nearest')
ps.savefig()
for band in bands:
for tim in tims:
if tim.band != band:
continue
plt.clf()
C = make_coadds([tim],
bands,
targetwcs,
callback=None,
sbscale=False,
mp=mp)
plt.imshow(get_rgb(C.coimgs, bands).sum(axis=2),
cmap='gray',
interpolation='nearest',
origin='lower')
plt.title('Band %s: tim %s' % (band, tim.name))
ps.savefig()
if plots:
import pylab as plt
import matplotlib.patches as patches
# skysub QA
C = make_coadds(tims, bands, targetwcs, callback=None, mp=mp)
imsave_jpeg(os.path.join(survey.output_dir, 'metrics', 'cus',
'{}-customsky.jpg'.format(ps.basefn)),
get_rgb(C.coimgs, bands),
origin='lower')
# ccdpos QA
refs, _ = get_reference_sources(survey,
targetwcs,
targetwcs.pixel_scale(), ['r'],
tycho_stars=False,
gaia_stars=False,
large_galaxies=True,
star_clusters=False)
pixscale = targetwcs.pixel_scale()
width = targetwcs.get_width() * pixscale / 3600 # [degrees]
bb, bbcc = targetwcs.radec_bounds(), targetwcs.radec_center(
) # [degrees]
pad = 0.5 * width # [degrees]
delta = np.max((np.diff(bb[0:2]), np.diff(bb[2:4]))) / 2 + pad / 2
xlim = bbcc[0] - delta, bbcc[0] + delta
ylim = bbcc[1] - delta, bbcc[1] + delta
plt.clf()
_, allax = plt.subplots(1,
3,
figsize=(12, 5),
sharey=True,
sharex=True)
for ax, band in zip(allax, ('g', 'r', 'z')):
ax.set_xlabel('RA (deg)')
ax.text(0.9,
0.05,
band,
ha='center',
va='bottom',
transform=ax.transAxes,
fontsize=18)
if band == 'g':
ax.set_ylabel('Dec (deg)')
ax.get_xaxis().get_major_formatter().set_useOffset(False)
# individual CCDs
these = np.where([tim.band == band for tim in tims])[0]
col = plt.cm.Set1(np.linspace(0, 1, len(tims)))
for ii, indx in enumerate(these):
tim = tims[indx]
#wcs = tim.subwcs
wcs = tim.imobj.get_wcs()
cc = wcs.radec_bounds()
ax.add_patch(
patches.Rectangle((cc[0], cc[2]),
cc[1] - cc[0],
cc[3] - cc[2],
fill=False,
lw=2,
edgecolor=col[these[ii]],
label='ccd{:02d}'.format(these[ii])))
ax.legend(ncol=2, frameon=False, loc='upper left', fontsize=10)
# output mosaic footprint
cc = targetwcs.radec_bounds()
ax.add_patch(
patches.Rectangle((cc[0], cc[2]),
cc[1] - cc[0],
cc[3] - cc[2],
fill=False,
lw=2,
edgecolor='k'))
if subsky_radii is not None:
racen, deccen = targetwcs.crval
for rad in subsky_radii:
ax.add_patch(
patches.Circle((racen, deccen),
rad / 3600,
fill=False,
edgecolor='black',
lw=2))
else:
for gal in refs:
ax.add_patch(
patches.Circle((gal.ra, gal.dec),
gal.radius,
fill=False,
edgecolor='black',
lw=2))
ax.set_ylim(ylim)
ax.set_xlim(xlim)
ax.invert_xaxis()
ax.set_aspect('equal')
plt.subplots_adjust(bottom=0.12,
wspace=0.05,
left=0.12,
right=0.97,
top=0.95)
plt.savefig(
os.path.join(survey.output_dir, 'metrics', 'cus',
'{}-ccdpos.jpg'.format(ps.basefn)))
if plots2:
plt.clf()
for coimg, band in zip(C.coimgs, bands):
plt.hist(coimg.ravel(),
bins=50,
range=(-0.5, 0.5),
histtype='step',
label=band)
plt.legend()
plt.title('After adjustment: coadds (sb scaled)')
ps.savefig()
return tims, skydict
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--catalog', help='Catalog to render')
parser.add_argument('--brick-coadd', help='Produce a coadd of the images overlapping the given brickname.')
parser.add_argument('--ccds', help='Use this table of CCDs')
parser.add_argument('--brick-wcs', help='File containing a WCS header describing the coadd WCS to render.')
parser.add_argument('--brick-wcs-ext', type=int, help='FITS file extension containing a WCS header describing the coadd WCS to render.')
parser.add_argument('--outlier-mask-brick', help='Comma-separated list of bricknames from which outlier masks should be read.')
parser.add_argument('--out', help='Filename pattern ("BAND" will be replaced by band name) of output images.')
parser.add_argument('--resid', help='Filename pattern ("BAND" will be replaced by band name) of residual images.')
parser.add_argument('--jpeg', help='Write RGB image to this filename')
parser.add_argument('--resid-jpeg', help='Write RGB residual image to this filename')
opt = parser.parse_args()
if opt.catalog is None:
print('Need catalog!')
return -1
cat = fits_table(opt.catalog)
if opt.ccds is None:
if opt.brick_coadd is None:
print('Need brick catalog!')
return -1
brickname = opt.brick_coadd
survey = LegacySurveyData()
if opt.brick_wcs is None:
print('FIXME')
return -1
else:
wcs = Tan(opt.brick_wcs, opt.brick_wcs_ext)
tcat = read_fits_catalog(cat)
if opt.ccds:
ccdfn = opt.ccds
else:
ccdfn = survey.find_file('ccds-table', brick=brickname)
print('Reading', ccdfn)
ccds = fits_table(ccdfn)
H,W = wcs.shape
targetrd = np.array([wcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
tims = []
for ccd in ccds:
im = survey.get_image_object(ccd)
#slc = slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1)
#tim = im.get_tractor_image(slc=slc)
tim = im.get_tractor_image(radecpoly=targetrd)
print('Read', tim)
tims.append(tim)
if opt.outlier_mask_brick is not None:
bricks = opt.outlier_mask_brick.split(',')
for b in bricks:
print('Reading outlier mask for brick', b,
':', survey.find_file('outliers_mask', brick=b, output=False))
ok = read_outlier_mask_file(survey, tims, b,
subimage=True, output=False)
tr = Tractor(tims, tcat)
mods = list(tr.getModelImages())
bands = 'grz'
def write_model(band, cowimg=None, cowmod=None, **kwargs):
if cowmod is None:
print('No model for', band)
return
outfn = opt.out.replace('BAND', band)
fitsio.write(outfn, cowmod, clobber=True)
print('Wrote model for', band, 'to', outfn)
if opt.resid:
outfn = opt.resid.replace('BAND', band)
fitsio.write(outfn, cowimg - cowmod, clobber=True)
print('Wrote resid for', band, 'to', outfn)
C = make_coadds(tims, bands, wcs, mods=mods,
callback=write_model)
if opt.jpeg:
from legacypipe.survey import get_rgb
import pylab as plt
plt.imsave(opt.jpeg, get_rgb(C.comods, bands), origin='lower')
if opt.resid_jpeg:
from legacypipe.survey import get_rgb
import pylab as plt
plt.imsave(opt.resid_jpeg,
get_rgb([im-mod for im,mod in zip(C.coimgs, C.comods)], bands),
origin='lower')
def top_levels(mp, opt):
from map.views import save_jpeg, trymakedirs
if opt.kind in ['decaps2', 'decaps2-model', 'decaps2-resid',
'mzls+bass-dr4', 'mzls+bass-dr4-model', 'mzls+bass-dr4-resid',
'decals-dr5', 'decals-dr5-model', 'decals-dr5-resid',
'decals-dr7', 'decals-dr7-model', 'decals-dr7-resid',
'mzls+bass-dr6', 'mzls+bass-dr6-model', 'mzls+bass-dr6-resid',
'des-dr1',
'eboss',
'unwise-neo2', 'unwise-neo3', 'unwise-neo4', 'unwise-cat-model',
'galex', 'sdss2', 'wssa',
'ls-dr56', 'ls-dr67'] or True:
import pylab as plt
from viewer import settings
import fitsio
from scipy.ndimage.filters import gaussian_filter
from map.views import _unwise_to_rgb
tag = opt.kind
from map.views import get_layer
layer = get_layer(opt.kind)
bands = layer.get_bands()
print('Layer:', layer)
#print('Survey:', layer.survey)
#print(' cache_dir:', layer.survey.cache_dir)
print('Bands', bands)
rgbkwargs = {}
if opt.kind in ['unwise-neo2', 'unwise-neo3', 'unwise-neo4', 'unwise-cat-model']:
bands = [1, 2]
elif opt.kind == 'sdss2':
bands = 'gri'
elif opt.kind == 'galex':
bands = ['n','f']
elif opt.kind == 'wssa':
bands = ['x']
#else:
# bands = 'grz'
ver = tileversions.get(opt.kind, [1])[-1]
print('Version', ver)
basescale = 5
pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s',
'%(zoom)i', '%(x)i', '%(y)i.jpg')
patdata = dict(ver=ver)
tilesize = 256
tiles = 2**basescale
side = tiles * tilesize
basepat = 'base-%s-%i-%%s.fits' % (opt.kind, basescale)
basefns = [basepat % band for band in bands]
if not all([os.path.exists(fn) for fn in basefns]):
bases = [np.zeros((side, side), np.float32) for band in bands]
args = []
xy = []
if opt.y1 is None:
opt.y1 = tiles
if opt.x0 is None:
opt.x0 = 0
if opt.x1 is None:
opt.x1 = tiles
for y in range(opt.y0, opt.y1):
for x in range(opt.x0, opt.x1):
args.append((opt.kind, basescale, x, y, False, True))
xy.append((x,y))
tiles = mp.map(_one_tile, args)
for ims,(x,y) in zip(tiles, xy):
#for a,(x,y) in zip(args, xy):
#print('_one_tile args:', a)
#ims = _one_tile(a)
#print('-> ', ims)
if ims is None:
continue
for im,base in zip(ims, bases):
if im is None:
continue
base[y*tilesize:(y+1)*tilesize,
x*tilesize:(x+1)*tilesize] = im
for fn,base in zip(basefns, bases):
fitsio.write(fn, base, clobber=True)
else:
print('Reading', basefns)
bases = [fitsio.read(fn) for fn in basefns]
for scale in range(basescale, -1, -1):
print('Scale', scale)
tiles = 2**scale
for y in range(tiles):
for x in range(tiles):
ims = [base[y*tilesize:(y+1)*tilesize,
x*tilesize:(x+1)*tilesize] for base in bases]
rgb = layer.get_rgb(ims, bands, **rgbkwargs)
pp = patdata.copy()
pp.update(zoom=scale, x=x, y=y)
fn = pat % pp
trymakedirs(fn)
save_jpeg(fn, rgb)
print('Wrote', fn)
for i,base in enumerate(bases):
if 'vlass' in opt.kind:
base = np.maximum(np.maximum(base[::2,::2], base[1::2,::2]),
np.maximum(base[1::2,1::2], base[::2,1::2]))
else:
base = (base[::2,::2] + base[1::2,::2] + base[1::2,1::2] + base[::2,1::2])/4.
bases[i] = base
elif opt.kind in ['unwise', 'unwise-neo1', 'unwise-w3w4',]:
import pylab as plt
from viewer import settings
from map.views import _unwise_to_rgb
import fitsio
if opt.kind == 'unwise-w3w4':
tag = 'unwise-w3w4'
bands = [3,4]
bounce = _bounce_map_unwise_w3w4
elif opt.kind == 'unwise-neo1':
tag = 'unwise-neo1'
bands = [1,2]
bounce = _bounce_map_unwise_neo1
else:
tag = 'unwise-w1w2'
bands = [1,2]
bounce = _bounce_map_unwise_w1w2
pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s',
'%(zoom)i', '%(x)i', '%(y)i.jpg')
ver = 1
patdata = dict(ver=ver)
basescale = 4
tilesize = 256
tiles = 2**basescale
side = tiles * tilesize
basepat = 'base-%s-%i-%%s.fits' % (opt.kind, basescale)
basefns = [basepat % band for band in bands]
if not all([os.path.exists(fn) for fn in basefns]):
bases = [np.zeros((side, side), np.float32) for band in bands]
args = []
for y in range(tiles):
for x in range(tiles):
#print 'Base tile', x, y
args.append((req, ver, basescale, x, y))
tiles = mp.map(bounce, args)
for ims,arg in zip(tiles,args):
x,y = arg[-2:]
for im,base in zip(ims, bases):
if im is None:
continue
base[y*tilesize:(y+1)*tilesize,
x*tilesize:(x+1)*tilesize] = im
for fn,base in zip(basefns, bases):
fitsio.write(fn, base, clobber=True)
else:
print('Reading', basefns)
bases = [fitsio.read(fn) for fn in basefns]
if False:
# Messin' around
plt.figure(figsize=(8,8))
plt.subplots_adjust(left=0, right=1, bottom=0, top=1)
#for S in [1000, 3000, 10000]:
#for Q in [10, 25, 50]:
S,Q = 3000,25
im = _unwise_to_rgb([w1base, w2base], S=S, Q=Q)
#plt.clf()
#plt.imshow(im)
plt.imsave('base-S%i-Q%s.png' % (S,Q), im)
# Try converting to galactic coords...
from astrometry.util.util import anwcs_create_mercator_2
print('Base images:', w1base.shape)
zoom = basescale
h,w = w1base.shape
zoomscale = 2.**zoom * (256./h)
print('Zoomscale', zoomscale)
wcs = anwcs_create_mercator_2(180., 0., w/2., h/2.,
zoomscale, w, h, 1)
wcs2 = anwcs_create_mercator_2(0., 0., w/2., h/2.,
zoomscale, w, h, 1)
print('WCS:')
for x,y in [(1,1), (1,h), (w,1), (w,h), (w/2,1), (w/2,h/2)]:
print('x,y', (x,y), '-> RA,Dec', wcs.pixelxy2radec(x,y)[-2:])
ok,ras,nil = wcs2.pixelxy2radec(np.arange(w), np.ones(w))
ok,nil,decs = wcs2.pixelxy2radec(np.ones(h), np.arange(h))
print('RAs', ras.shape)
print('Decs', decs.shape)
lls = ras
bbs = decs
ll,bb = np.meshgrid(lls, bbs)
print('LL,BB', ll.shape, bb.shape)
from astrometry.util.starutil_numpy import lbtoradec
ra,dec = lbtoradec(ll,bb)
print('RA,Dec', ra.shape, dec.shape)
ok,xx,yy = wcs.radec2pixelxy(ra, dec)
print('xx,yy', xx.shape, yy.shape)
lb1 = w1base[np.clip(np.round(yy-1).astype(int), 0, h-1),
np.clip(np.round(xx-1).astype(int), 0, w-1)]
lb2 = w2base[np.clip(np.round(yy-1).astype(int), 0, h-1),
np.clip(np.round(xx-1).astype(int), 0, w-1)]
lbim = _unwise_to_rgb(lb1, lb2, S=S,Q=Q)
plt.imsave('lb.png', lbim)
sys.exit(0)
from scipy.ndimage.filters import gaussian_filter
for scale in range(basescale-1, -1, -1):
for i,base in enumerate(bases):
base = gaussian_filter(base, 1.)
base = (base[::2,::2] + base[1::2,::2] + base[1::2,1::2] + base[::2,1::2])/4.
bases[i] = base
tiles = 2**scale
for y in range(tiles):
for x in range(tiles):
ims = [base[y*tilesize:(y+1)*tilesize,
x*tilesize:(x+1)*tilesize] for base in bases]
tile = _unwise_to_rgb(ims, bands=bands)
pp = patdata.copy()
pp.update(zoom=scale, x=x, y=y)
fn = pat % pp
trymakedirs(fn)
save_jpeg(fn, tile)
print('Wrote', fn)
if opt.kind in ['depth-g', 'depth-r', 'depth-z']:
import pylab as plt
from viewer import settings
from scipy.ndimage.filters import gaussian_filter
from map.views import trymakedirs
tag = 'decam-' + opt.kind
#band = opt.kind[-1]
ver = 1
basescale = 5
pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s',
'%(zoom)i', '%(x)i', '%(y)i.jpg')
patdata = dict(ver=ver)
tilesize = 256
tiles = 2**basescale
side = tiles * tilesize
base = np.zeros((side, side, 3), np.float32)
for y in range(tiles):
for x in range(tiles):
dat = patdata.copy()
dat.update(zoom=basescale, x=x, y=y)
fn = pat % dat
if not os.path.exists(fn):
print('Does not exist:', fn)
continue
img = plt.imread(fn)
base[y*tilesize:(y+1)*tilesize,
x*tilesize:(x+1)*tilesize,:] = img
for scale in range(basescale-1, -1, -1):
newbase = []
for i in range(3):
b = gaussian_filter(base[:,:,i], 1.)
b = (b[ ::2, ::2] + b[1::2, ::2] +
b[1::2,1::2] + b[ ::2,1::2])/4.
newbase.append(b)
base = np.dstack(newbase)
tiles = 2**scale
for y in range(tiles):
for x in range(tiles):
img = base[y*tilesize:(y+1)*tilesize,
x*tilesize:(x+1)*tilesize, :]
pp = patdata.copy()
pp.update(zoom=scale, x=x, y=y)
fn = pat % pp
trymakedirs(fn)
plt.imsave(fn, np.clip(np.round(img).astype(np.uint8), 0, 255))
print('Wrote', fn)
### HACK... factor this out...
if opt.kind in ['sdss',
'decals-dr2', 'decals-dr2-model', 'decals-dr2-resid',
'decals-dr3', 'decals-dr3-model', 'decals-dr3-resid',
]:
import pylab as plt
from viewer import settings
from legacypipe.survey import get_rgb
import fitsio
from scipy.ndimage.filters import gaussian_filter
tag = opt.kind
bouncemap = {
'sdss': _bounce_sdssco,
'decals-dr3': _bounce_decals_dr3,
'decals-dr3-model': _bounce_decals_dr3,
'decals-dr3-resid': _bounce_decals_dr3,
}
bounce = bouncemap[opt.kind]
rgbkwargs = dict(mnmx=(-1,100.), arcsinh=1.)
bands = 'grz'
if opt.kind in ['decals-dr2', 'decals-dr2-model',
'decals-dr3', 'decals-dr3-model',]:
get_rgb = dr2_rgb
rgbkwargs = {}
elif opt.kind == 'sdssco':
rgbkwargs = {}
ver = tileversions.get(opt.kind, [1])[-1]
print('Version', ver)
basescale = 5
pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s',
'%(zoom)i', '%(x)i', '%(y)i.jpg')
patdata = dict(ver=ver)
tilesize = 256
tiles = 2**basescale
side = tiles * tilesize
basepat = 'base-%s-%i-%%s.fits' % (opt.kind, basescale)
basefns = [basepat % band for band in bands]
if not all([os.path.exists(fn) for fn in basefns]):
bases = [np.zeros((side, side), np.float32) for band in bands]
args = []
kwa = dict()
if 'model' in opt.kind:
kwa.update(model=True, add_gz=True)
elif 'resid' in opt.kind:
kwa.update(resid=True, model_gz=True)
xy = []
if opt.y1 is None:
opt.y1 = tiles
if opt.x0 is None:
opt.x0 = 0
if opt.x1 is None:
opt.x1 = tiles
for y in range(opt.y0, opt.y1):
for x in range(opt.x0, opt.x1):
args.append(((req, ver, basescale, x, y),kwa))
xy.append((x,y))
tiles = mp.map(bounce, args)
for ims,(x,y) in zip(tiles, xy):
if ims is None:
continue
for im,base in zip(ims, bases):
if im is None:
continue
base[y*tilesize:(y+1)*tilesize,
x*tilesize:(x+1)*tilesize] = im
for fn,base in zip(basefns, bases):
fitsio.write(fn, base, clobber=True)
else:
print('Reading', basefns)
bases = [fitsio.read(fn) for fn in basefns]
#for scale in range(basescale-1, -1, -1):
for scale in range(basescale, -1, -1):
print('Scale', scale)
tiles = 2**scale
for y in range(tiles):
for x in range(tiles):
ims = [base[y*tilesize:(y+1)*tilesize,
x*tilesize:(x+1)*tilesize] for base in bases]
if opt.kind == 'sdss':
bands = 'gri'
rgb = sdss_rgb(ims, bands)
else:
rgb = get_rgb(ims, bands, **rgbkwargs)
pp = patdata.copy()
pp.update(zoom=scale, x=x, y=y)
fn = pat % pp
trymakedirs(fn)
save_jpeg(fn, rgb)
print('Wrote', fn)
for i,base in enumerate(bases):
#base = gaussian_filter(base, 1.)
base = (base[::2,::2] + base[1::2,::2] + base[1::2,1::2] + base[::2,1::2])/4.
bases[i] = base
