def sky_fiber_plots(survey, brickname, skyfibers, basefn, bands=['g', 'r', 'z']): from legacypipe.survey import get_rgb import fitsio import pylab as plt rgbkwargs = dict(mnmx=(-1, 100.), arcsinh=1.) imgs = [] for band in bands: fn = survey.find_file('image', brick=brickname, band=band) imgs.append(fitsio.read(fn)) rgb = get_rgb(imgs, bands, **rgbkwargs) ima = dict(interpolation='nearest', origin='lower') plt.clf() plt.imshow(rgb, **ima) plt.plot(skyfibers.x, skyfibers.y, 'o', mfc='none', mec='r', mew=2, ms=10) plt.title('Sky fiber positions') plt.savefig(basefn + '-1.png') plt.clf() plt.subplots_adjust(hspace=0, wspace=0) SZ = 25 fig = plt.gcf() fh, fw = fig.get_figheight(), fig.get_figwidth() C = int(np.ceil(np.sqrt(len(skyfibers) * fw / fh))) R = int(np.ceil(len(skyfibers) / float(C))) k = 1 H, W = imgs[0].shape for x, y in zip(skyfibers.x, skyfibers.y): if x < SZ or y < SZ or x >= W - SZ or y >= H - SZ: continue plt.subplot(R, C, k) k += 1 plt.imshow(rgb[y - SZ:y + SZ + 1, x - SZ:x + SZ + 1, :], **ima) plt.xticks([]) plt.yticks([]) plt.suptitle('Sky fiber locations') plt.savefig(basefn + '-2.png') plt.clf() ccmap = dict(z='m') for band in bands: flux = skyfibers.get('apflux_%s' % band) plt.plot(flux.T, color=ccmap.get(band, band), alpha=0.1) plt.ylim(-10, 10) #plt.xticks(np.arange(len(apertures_arcsec)), # ['%g' % ap for ap in apertures_arcsec]) plt.xlabel('Aperture') # (arcsec radius)') plt.ylabel('Aperture flux (nanomaggies)') plt.title('Sky fiber: aperture flux') plt.savefig(basefn + '-3.png')
def main(): survey = LegacySurveyData() brickname = '2351p137' # RA,Dec = 235.0442, 13.7125 bx, by = 3300, 1285 sz = 50 bbox = [bx - sz, bx + sz, by - sz, by + sz] objid = 1394 bands = ['g', 'r', 'z'] from legacypipe.runbrick import stage_tims, _get_mod, rgbkwargs, rgbkwargs_resid from legacypipe.survey import get_rgb, imsave_jpeg from legacypipe.coadds import make_coadds from astrometry.util.multiproc import multiproc from astrometry.util.fits import fits_table from legacypipe.catalog import read_fits_catalog # brick = survey.get_brick_by_name(brickname) # # Get WCS object describing brick # targetwcs = wcs_for_brick(brick) # (x0,x1,y0,y1) = bbox # W = x1-x0 # H = y1-y0 # targetwcs = targetwcs.get_subimage(x0, y0, W, H) # H,W = targetwcs.shape mp = multiproc() P = stage_tims(brickname=brickname, survey=survey, target_extent=bbox, pixPsf=True, hybridPsf=True, depth_cut=False, mp=mp) print('Got', P.keys()) tims = P['tims'] targetwcs = P['targetwcs'] H, W = targetwcs.shape # Read Tractor catalog fn = survey.find_file('tractor', brick=brickname) print('Trying to read catalog', fn) cat = fits_table(fn) print('Read', len(cat), 'sources') ok, xx, yy = targetwcs.radec2pixelxy(cat.ra, cat.dec) I = np.flatnonzero((xx > 0) * (xx < W) * (yy > 0) * (yy < H)) cat.cut(I) print('Cut to', len(cat), 'sources within box') I = np.flatnonzero(cat.objid != objid) cat.cut(I) print('Cut to', len(cat), 'sources with objid !=', objid) #cat.about() # Convert FITS catalog into tractor source objects print('Creating tractor sources...') srcs = read_fits_catalog(cat, fluxPrefix='') print('Sources:') for src in srcs: print(' ', src) print('Rendering model images...') mods = [_get_mod((tim, srcs)) for tim in tims] print('Producing coadds...') C = make_coadds(tims, bands, targetwcs, mods=mods, mp=mp) print('Coadds:', dir(C)) coadd_list = [('image', C.coimgs, rgbkwargs), ('model', C.comods, rgbkwargs), ('resid', C.coresids, rgbkwargs_resid)] #C.coimgs, C.comods, C.coresids for name, ims, rgbkw in coadd_list: rgb = get_rgb(ims, bands, **rgbkw) kwa = {} #with survey.write_output(name + '-jpeg', brick=brickname) as out: # imsave_jpeg(out.fn, rgb, origin='lower', **kwa) # print('Wrote', out.fn) outfn = name + '.jpg' imsave_jpeg(outfn, rgb, origin='lower', **kwa) del rgb
def top_levels(mp, opt): from map.views import save_jpeg, trymakedirs if opt.kind in [ 'decaps2', 'decaps2-model', 'decaps2-resid', 'mzls+bass-dr4', 'mzls+bass-dr4-model', 'mzls+bass-dr4-resid', 'decals-dr5', 'decals-dr5-model', 'decals-dr5-resid', 'mzls+bass-dr6', 'mzls+bass-dr6-model', 'mzls+bass-dr6-resid', 'eboss', 'unwise-neo2', 'sdss2' ]: import pylab as plt from decals import settings from legacypipe.survey import get_rgb import fitsio from scipy.ndimage.filters import gaussian_filter from map.views import trymakedirs from map.views import _unwise_to_rgb tag = opt.kind rgbkwargs = {} if opt.kind == 'unwise-neo2': bands = [1, 2] get_rgb = _unwise_to_rgb elif opt.kind == 'sdss2': bands = 'gri' get_rgb = sdss_rgb else: bands = 'grz' get_rgb = dr2_rgb ver = tileversions.get(opt.kind, [1])[-1] print('Version', ver) basescale = 5 pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s', '%(zoom)i', '%(x)i', '%(y)i.jpg') patdata = dict(ver=ver) tilesize = 256 tiles = 2**basescale side = tiles * tilesize basepat = 'base-%s-%i-%%s.fits' % (opt.kind, basescale) basefns = [basepat % band for band in bands] if not all([os.path.exists(fn) for fn in basefns]): bases = [np.zeros((side, side), np.float32) for band in bands] args = [] xy = [] if opt.y1 is None: opt.y1 = tiles if opt.x0 is None: opt.x0 = 0 if opt.x1 is None: opt.x1 = tiles for y in range(opt.y0, opt.y1): for x in range(opt.x0, opt.x1): args.append((opt.kind, basescale, x, y, False, True)) xy.append((x, y)) tiles = mp.map(_one_tile, args) for ims, (x, y) in zip(tiles, xy): #for a,(x,y) in zip(args, xy): #print('_one_tile args:', a) #ims = _one_tile(a) #print('-> ', ims) if ims is None: continue for im, base in zip(ims, bases): if im is None: continue base[y * tilesize:(y + 1) * tilesize, x * tilesize:(x + 1) * tilesize] = im for fn, base in zip(basefns, bases): fitsio.write(fn, base, clobber=True) else: print('Reading', basefns) bases = [fitsio.read(fn) for fn in basefns] for scale in range(basescale, -1, -1): print('Scale', scale) tiles = 2**scale for y in range(tiles): for x in range(tiles): ims = [ base[y * tilesize:(y + 1) * tilesize, x * tilesize:(x + 1) * tilesize] for base in bases ] rgb = get_rgb(ims, bands, **rgbkwargs) pp = patdata.copy() pp.update(zoom=scale, x=x, y=y) fn = pat % pp trymakedirs(fn) save_jpeg(fn, rgb) print('Wrote', fn) for i, base in enumerate(bases): base = (base[::2, ::2] + base[1::2, ::2] + base[1::2, 1::2] + base[::2, 1::2]) / 4. bases[i] = base elif opt.kind in [ 'unwise', 'unwise-neo1', 'unwise-w3w4', ]: import pylab as plt from decals import settings from map.views import _unwise_to_rgb, save_jpeg, trymakedirs import fitsio if opt.kind == 'unwise-w3w4': tag = 'unwise-w3w4' bands = [3, 4] bounce = _bounce_map_unwise_w3w4 elif opt.kind == 'unwise-neo1': tag = 'unwise-neo1' bands = [1, 2] bounce = _bounce_map_unwise_neo1 else: tag = 'unwise-w1w2' bands = [1, 2] bounce = _bounce_map_unwise_w1w2 pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s', '%(zoom)i', '%(x)i', '%(y)i.jpg') ver = 1 patdata = dict(ver=ver) basescale = 4 tilesize = 256 tiles = 2**basescale side = tiles * tilesize basepat = 'base-%s-%i-%%s.fits' % (opt.kind, basescale) basefns = [basepat % band for band in bands] if not all([os.path.exists(fn) for fn in basefns]): bases = [np.zeros((side, side), np.float32) for band in bands] args = [] for y in range(tiles): for x in range(tiles): #print 'Base tile', x, y args.append((req, ver, basescale, x, y)) tiles = mp.map(bounce, args) for ims, arg in zip(tiles, args): x, y = arg[-2:] for im, base in zip(ims, bases): if im is None: continue base[y * tilesize:(y + 1) * tilesize, x * tilesize:(x + 1) * tilesize] = im for fn, base in zip(basefns, bases): fitsio.write(fn, base, clobber=True) else: print('Reading', basefns) bases = [fitsio.read(fn) for fn in basefns] if False: # Messin' around plt.figure(figsize=(8, 8)) plt.subplots_adjust(left=0, right=1, bottom=0, top=1) #for S in [1000, 3000, 10000]: #for Q in [10, 25, 50]: S, Q = 3000, 25 im = _unwise_to_rgb([w1base, w2base], S=S, Q=Q) #plt.clf() #plt.imshow(im) plt.imsave('base-S%i-Q%s.png' % (S, Q), im) # Try converting to galactic coords... from astrometry.util.util import anwcs_create_mercator_2 print('Base images:', w1base.shape) zoom = basescale h, w = w1base.shape zoomscale = 2.**zoom * (256. / h) print('Zoomscale', zoomscale) wcs = anwcs_create_mercator_2(180., 0., w / 2., h / 2., zoomscale, w, h, 1) wcs2 = anwcs_create_mercator_2(0., 0., w / 2., h / 2., zoomscale, w, h, 1) print('WCS:') for x, y in [(1, 1), (1, h), (w, 1), (w, h), (w / 2, 1), (w / 2, h / 2)]: print('x,y', (x, y), '-> RA,Dec', wcs.pixelxy2radec(x, y)[-2:]) ok, ras, nil = wcs2.pixelxy2radec(np.arange(w), np.ones(w)) ok, nil, decs = wcs2.pixelxy2radec(np.ones(h), np.arange(h)) print('RAs', ras.shape) print('Decs', decs.shape) lls = ras bbs = decs ll, bb = np.meshgrid(lls, bbs) print('LL,BB', ll.shape, bb.shape) from astrometry.util.starutil_numpy import lbtoradec ra, dec = lbtoradec(ll, bb) print('RA,Dec', ra.shape, dec.shape) ok, xx, yy = wcs.radec2pixelxy(ra, dec) print('xx,yy', xx.shape, yy.shape) lb1 = w1base[np.clip(np.round(yy - 1).astype(int), 0, h - 1), np.clip(np.round(xx - 1).astype(int), 0, w - 1)] lb2 = w2base[np.clip(np.round(yy - 1).astype(int), 0, h - 1), np.clip(np.round(xx - 1).astype(int), 0, w - 1)] lbim = _unwise_to_rgb(lb1, lb2, S=S, Q=Q) plt.imsave('lb.png', lbim) sys.exit(0) from scipy.ndimage.filters import gaussian_filter for scale in range(basescale - 1, -1, -1): for i, base in enumerate(bases): base = gaussian_filter(base, 1.) base = (base[::2, ::2] + base[1::2, ::2] + base[1::2, 1::2] + base[::2, 1::2]) / 4. bases[i] = base tiles = 2**scale for y in range(tiles): for x in range(tiles): ims = [ base[y * tilesize:(y + 1) * tilesize, x * tilesize:(x + 1) * tilesize] for base in bases ] tile = _unwise_to_rgb(ims, bands=bands) pp = patdata.copy() pp.update(zoom=scale, x=x, y=y) fn = pat % pp trymakedirs(fn) save_jpeg(fn, tile) print('Wrote', fn) if opt.kind in ['depth-g', 'depth-r', 'depth-z']: import pylab as plt from decals import settings from scipy.ndimage.filters import gaussian_filter from map.views import trymakedirs tag = 'decam-' + opt.kind band = opt.kind[-1] ver = 1 basescale = 5 pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s', '%(zoom)i', '%(x)i', '%(y)i.jpg') patdata = dict(ver=ver) tilesize = 256 tiles = 2**basescale side = tiles * tilesize base = np.zeros((side, side, 3), np.float32) for y in range(tiles): for x in range(tiles): dat = patdata.copy() dat.update(zoom=basescale, x=x, y=y) fn = pat % dat if not os.path.exists(fn): print('Does not exist:', fn) continue img = plt.imread(fn) base[y * tilesize:(y + 1) * tilesize, x * tilesize:(x + 1) * tilesize, :] = img for scale in range(basescale - 1, -1, -1): newbase = [] for i in range(3): b = gaussian_filter(base[:, :, i], 1.) b = (b[::2, ::2] + b[1::2, ::2] + b[1::2, 1::2] + b[::2, 1::2]) / 4. newbase.append(b) base = np.dstack(newbase) tiles = 2**scale for y in range(tiles): for x in range(tiles): img = base[y * tilesize:(y + 1) * tilesize, x * tilesize:(x + 1) * tilesize, :] pp = patdata.copy() pp.update(zoom=scale, x=x, y=y) fn = pat % pp trymakedirs(fn) plt.imsave(fn, np.clip(np.round(img).astype(np.uint8), 0, 255)) print('Wrote', fn) ### HACK... factor this out... if opt.kind in [ 'sdss', 'decals-dr2', 'decals-dr2-model', 'decals-dr2-resid', 'decals-dr3', 'decals-dr3-model', 'decals-dr3-resid', ]: import pylab as plt from decals import settings from legacypipe.survey import get_rgb import fitsio from scipy.ndimage.filters import gaussian_filter from map.views import trymakedirs tag = opt.kind bouncemap = { 'sdss': _bounce_sdssco, 'decals-dr3': _bounce_decals_dr3, 'decals-dr3-model': _bounce_decals_dr3, 'decals-dr3-resid': _bounce_decals_dr3, } bounce = bouncemap[opt.kind] rgbkwargs = dict(mnmx=(-1, 100.), arcsinh=1.) bands = 'grz' if opt.kind in [ 'decals-dr2', 'decals-dr2-model', 'decals-dr3', 'decals-dr3-model', ]: get_rgb = dr2_rgb rgbkwargs = {} elif opt.kind == 'sdssco': rgbfunc = sdss_rgb rgbkwargs = {} ver = tileversions.get(opt.kind, [1])[-1] print('Version', ver) basescale = 5 pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s', '%(zoom)i', '%(x)i', '%(y)i.jpg') patdata = dict(ver=ver) tilesize = 256 tiles = 2**basescale side = tiles * tilesize basepat = 'base-%s-%i-%%s.fits' % (opt.kind, basescale) basefns = [basepat % band for band in bands] if not all([os.path.exists(fn) for fn in basefns]): bases = [np.zeros((side, side), np.float32) for band in bands] args = [] kwa = dict() if 'model' in opt.kind: kwa.update(model=True, add_gz=True) elif 'resid' in opt.kind: kwa.update(resid=True, model_gz=True) xy = [] if opt.y1 is None: opt.y1 = tiles if opt.x0 is None: opt.x0 = 0 if opt.x1 is None: opt.x1 = tiles for y in range(opt.y0, opt.y1): for x in range(opt.x0, opt.x1): args.append(((req, ver, basescale, x, y), kwa)) xy.append((x, y)) tiles = mp.map(bounce, args) for ims, (x, y) in zip(tiles, xy): if ims is None: continue for im, base in zip(ims, bases): if im is None: continue base[y * tilesize:(y + 1) * tilesize, x * tilesize:(x + 1) * tilesize] = im for fn, base in zip(basefns, bases): fitsio.write(fn, base, clobber=True) else: print('Reading', basefns) bases = [fitsio.read(fn) for fn in basefns] #for scale in range(basescale-1, -1, -1): for scale in range(basescale, -1, -1): print('Scale', scale) tiles = 2**scale for y in range(tiles): for x in range(tiles): ims = [ base[y * tilesize:(y + 1) * tilesize, x * tilesize:(x + 1) * tilesize] for base in bases ] if opt.kind == 'sdss': bands = 'gri' rgb = sdss_rgb(ims, bands) else: rgb = get_rgb(ims, bands, **rgbkwargs) pp = patdata.copy() pp.update(zoom=scale, x=x, y=y) fn = pat % pp trymakedirs(fn) save_jpeg(fn, rgb) print('Wrote', fn) for i, base in enumerate(bases): #base = gaussian_filter(base, 1.) base = (base[::2, ::2] + base[1::2, ::2] + base[1::2, 1::2] + base[::2, 1::2]) / 4. bases[i] = base
def _tractor_coadds(galaxycat, targetwcs, tims, mods, version_header, objid=None, brickname=None, survey=None, mp=None, verbose=False, bands=['g', 'r', 'z']): """Generate individual-band FITS and color coadds for each central using Tractor. """ from legacypipe.coadds import make_coadds, write_coadd_images from legacypipe.runbrick import rgbkwargs, rgbkwargs_resid from legacypipe.survey import get_rgb, imsave_jpeg if brickname is None: brickname = galaxycat['brickname'] if verbose: print('Producing coadds...') C = make_coadds(tims, bands, targetwcs, mods=mods, mp=mp, callback=write_coadd_images, callback_args=(survey, brickname, version_header, tims, targetwcs)) # Move (rename) the coadds into the desired output directory. for suffix in ('chi2', 'image', 'invvar', 'model'): for band in bands: oldfile = os.path.join( survey.output_dir, 'coadd', brickname[:3], brickname, 'legacysurvey-{}-{}-{}.fits.fz'.format(brickname, suffix, band)) newfile = os.path.join( survey.output_dir, '{}-{}-{}.fits.fz'.format(objid, suffix, band)) shutil.copy(oldfile, newfile) if True: shutil.rmtree(os.path.join(survey.output_dir, 'coadd')) # Build png postage stamps of the coadds. coadd_list = [('image', C.coimgs, rgbkwargs), ('model', C.comods, rgbkwargs), ('resid', C.coresids, rgbkwargs_resid)] for name, ims, rgbkw in coadd_list: rgb = get_rgb(ims, bands, **rgbkw) kwa = {} outfn = os.path.join(survey.output_dir, '{}-{}.jpg'.format(objid, name)) if verbose: print('Writing {}'.format(outfn)) imsave_jpeg(outfn, rgb, origin='lower', **kwa) del rgb
def make_coadds(tims, bands, targetwcs, mods=None, xy=None, apertures=None, apxy=None, ngood=False, detmaps=False, psfsize=False, allmasks=True, max=False, sbscale=True, callback=None, callback_args=None, plots=False, ps=None, lanczos=True, mp=None, satur_val=10.): from astrometry.util.ttime import Time t0 = Time() if callback_args is None: callback_args = [] class Duck(object): pass C = Duck() W = int(targetwcs.get_width()) H = int(targetwcs.get_height()) # always, for patching SATUR, etc pixels? unweighted = True C.coimgs = [] # the pixelwise inverse-variances (weights) of the "coimgs". C.cowimgs = [] if detmaps: C.galdetivs = [] C.psfdetivs = [] if mods is not None: C.comods = [] C.coresids = [] if apertures is not None: unweighted = True C.AP = fits_table() if allmasks is not None: C.allmasks = [] if max: C.maximgs = [] if xy: ix, iy = xy C.T = fits_table() C.T.nobs = np.zeros((len(ix), len(bands)), np.int16) C.T.anymask = np.zeros((len(ix), len(bands)), np.int16) C.T.allmask = np.zeros((len(ix), len(bands)), np.int16) if psfsize: C.T.psfsize = np.zeros((len(ix), len(bands)), np.float32) if detmaps: C.T.psfdepth = np.zeros((len(ix), len(bands)), np.float32) C.T.galdepth = np.zeros((len(ix), len(bands)), np.float32) if lanczos: debug('Doing Lanczos resampling') for tim in tims: # surface-brightness correction tim.sbscale = (targetwcs.pixel_scale() / tim.subwcs.pixel_scale())**2 # We create one iterator per band to do the tim resampling. These all run in # parallel when multi-processing. imaps = [] for band in bands: args = [] for itim, tim in enumerate(tims): if tim.band != band: continue if mods is None: mo = None else: mo = mods[itim] args.append((itim, tim, mo, lanczos, targetwcs, sbscale)) if mp is not None: imaps.append(mp.imap_unordered(_resample_one, args)) else: imaps.append(map(_resample_one, args)) # Args for aperture photometry apargs = [] if xy: # To save the memory of 2 x float64 maps, we instead do arg min/max maps # append a 0 to the list of mjds so that mjds[-1] gives 0. mjds = np.array([tim.time.toMjd() for tim in tims] + [0]) mjd_argmins = np.empty((H, W), np.int16) mjd_argmaxs = np.empty((H, W), np.int16) mjd_argmins[:, :] = -1 mjd_argmaxs[:, :] = -1 if plots: allresids = [] tinyw = 1e-30 for iband, (band, timiter) in enumerate(zip(bands, imaps)): debug('Computing coadd for band', band) # coadded weight map (moo) cow = np.zeros((H, W), np.float32) # coadded weighted image map cowimg = np.zeros((H, W), np.float32) kwargs = dict(cowimg=cowimg, cow=cow) if detmaps: # detection map inverse-variance (depth map) psfdetiv = np.zeros((H, W), np.float32) C.psfdetivs.append(psfdetiv) kwargs.update(psfdetiv=psfdetiv) # galaxy detection map inverse-variance (galdepth map) galdetiv = np.zeros((H, W), np.float32) C.galdetivs.append(galdetiv) kwargs.update(galdetiv=galdetiv) if mods is not None: # model image cowmod = np.zeros((H, W), np.float32) # chi-squared image cochi2 = np.zeros((H, W), np.float32) kwargs.update(cowmod=cowmod, cochi2=cochi2) if unweighted: # unweighted image coimg = np.zeros((H, W), np.float32) if mods is not None: # unweighted model comod = np.zeros((H, W), np.float32) # number of exposures con = np.zeros((H, W), np.int16) # inverse-variance coiv = np.zeros((H, W), np.float32) kwargs.update(coimg=coimg, coiv=coiv) # Note that we have 'congood' as well as 'nobs': # * 'congood' is used for the 'nexp' *image*. # It counts the number of "good" (unmasked) exposures # * 'nobs' is used for the per-source measurements # It counts the total number of exposures, including masked pixels # # (you want to know the number of observations within the # source footprint, not just the peak pixel which may be # saturated, etc.) if ngood: congood = np.zeros((H, W), np.int16) kwargs.update(congood=congood) if xy or allmasks: # These match the type of the "DQ" images. # "any" mask ormask = np.zeros((H, W), np.int16) # "all" mask andmask = np.empty((H, W), np.int16) from functools import reduce allbits = reduce(np.bitwise_or, DQ_BITS.values()) andmask[:, :] = allbits kwargs.update(ormask=ormask, andmask=andmask) if xy: # number of observations nobs = np.zeros((H, W), np.int16) kwargs.update(nobs=nobs) if psfsize: psfsizemap = np.zeros((H, W), np.float32) # like "cow", but constant invvar per-CCD; # only required for psfsizemap flatcow = np.zeros((H, W), np.float32) kwargs.update(psfsize=psfsizemap) if max: maximg = np.zeros((H, W), np.float32) C.maximgs.append(maximg) for R in timiter: if R is None: continue itim, Yo, Xo, iv, im, mo, dq = R tim = tims[itim] if plots: from legacypipe.runbrick import rgbkwargs from legacypipe.survey import get_rgb import pylab as plt # # Make one grayscale, brick-space plot per image # thisimg = np.zeros((H,W), np.float32) # thisimg[Yo,Xo] = im # rgb = get_rgb([thisimg], [band], **rgbkwargs) # rgb = rgb.sum(axis=2) # fn = ps.getnext() # plt.imsave(fn, rgb, origin='lower', cmap='gray') #plt.clf() #plt.imshow(rgb, interpolation='nearest', origin='lower', cmap='gray') #plt.xticks([]); plt.yticks([]) #ps.savefig() # Image, Model, and Resids plt.clf() plt.subplot(2, 2, 1) thisimg = np.zeros((H, W), np.float32) thisimg[Yo, Xo] = im rgb = get_rgb([thisimg], [band], **rgbkwargs) iplane = dict(g=2, r=1, z=0)[band] rgbimg = rgb[:, :, iplane] plt.imshow(rgbimg, interpolation='nearest', origin='lower', cmap='gray') plt.xticks([]) plt.yticks([]) if mods is not None: plt.subplot(2, 2, 2) thismod = np.zeros((H, W), np.float32) thismod[Yo, Xo] = mo rgb = get_rgb([thismod], [band], **rgbkwargs) rgbmod = rgb[:, :, iplane] plt.imshow(rgbmod, interpolation='nearest', origin='lower', cmap='gray') plt.xticks([]) plt.yticks([]) plt.subplot(2, 2, 3) thisres = np.zeros((H, W), np.float32) thisres[Yo, Xo] = (im - mo) * np.sqrt(iv) plt.imshow(thisres, interpolation='nearest', origin='lower', cmap='gray', vmin=-20, vmax=20) plt.xticks([]) plt.yticks([]) else: if unweighted and (dq is not None): # HACK -- copy-n-pasted code from below. okbits = 0 #for bitname in ['satur', 'bleed']: for bitname in ['satur']: okbits |= DQ_BITS[bitname] brightpix = ((dq & okbits) != 0) myim = im.copy() if satur_val is not None: # HACK -- force SATUR pix to be bright myim[brightpix] = satur_val #for bitname in ['interp']: for bitname in ['interp', 'bleed']: okbits |= DQ_BITS[bitname] goodpix = ((dq & ~okbits) == 0) thisgood = np.zeros((H, W), np.float32) thisgood[Yo, Xo] = goodpix plt.subplot(2, 2, 2) plt.imshow(thisgood, interpolation='nearest', origin='lower', cmap='gray', vmin=0, vmax=1) plt.xticks([]) plt.yticks([]) plt.title('goodpix') thisim = np.zeros((H, W), np.float32) thisim[Yo, Xo] = goodpix * myim rgb = get_rgb([thisim], [band], **rgbkwargs) iplane = dict(g=2, r=1, z=0)[band] rgbimg = rgb[:, :, iplane] plt.subplot(2, 2, 3) plt.imshow(rgbimg, interpolation='nearest', origin='lower', cmap='gray') plt.xticks([]) plt.yticks([]) plt.title('goodpix rgb') rgbmod = None thisres = None plt.subplot(2, 2, 4) thisiv = np.zeros((H, W), np.float32) thisiv[Yo, Xo] = iv plt.imshow(thisiv, interpolation='nearest', origin='lower', cmap='gray') plt.xticks([]) plt.yticks([]) plt.title('invvar') plt.suptitle(tim.name + ': %.2f' % (tim.time.toYear())) ps.savefig() allresids.append( (tim.time.toYear(), tim.name, rgbimg, rgbmod, thisres)) # invvar-weighted image cowimg[Yo, Xo] += iv * im cow[Yo, Xo] += iv if unweighted: if dq is None: goodpix = 1 else: # include SATUR pixels if no other # pixels exists okbits = 0 for bitname in ['satur']: okbits |= DQ_BITS[bitname] brightpix = ((dq & okbits) != 0) if satur_val is not None: # HACK -- force SATUR pix to be bright im[brightpix] = satur_val # Include these pixels if none other exist?? for bitname in ['interp']: #, 'bleed']: okbits |= DQ_BITS[bitname] goodpix = ((dq & ~okbits) == 0) coimg[Yo, Xo] += goodpix * im con[Yo, Xo] += goodpix coiv[Yo, Xo] += goodpix * 1. / (tim.sig1 * tim.sbscale)**2 # ...ish if xy: if dq is not None: ormask[Yo, Xo] |= dq andmask[Yo, Xo] &= dq # raw exposure count nobs[Yo, Xo] += 1 # mjd_min/max update = np.logical_or( mjd_argmins[Yo, Xo] == -1, (mjd_argmins[Yo, Xo] > -1) * (mjds[itim] < mjds[mjd_argmins[Yo, Xo]])) mjd_argmins[Yo[update], Xo[update]] = itim update = np.logical_or( mjd_argmaxs[Yo, Xo] == -1, (mjd_argmaxs[Yo, Xo] > -1) * (mjds[itim] > mjds[mjd_argmaxs[Yo, Xo]])) mjd_argmaxs[Yo[update], Xo[update]] = itim del update if psfsize: # psfnorm is in units of 1/pixels. # (eg, psfnorm for a gaussian is ~ 1/psf_sigma) # Neff is in pixels**2 neff = 1. / tim.psfnorm**2 # Narcsec is in arcsec**2 narcsec = neff * tim.wcs.pixel_scale()**2 # Make smooth maps -- don't ignore CRs, saturated pix, etc #psfsizemap[Yo,Xo] += (iv>0) * (1/tim.sig1**2) * (1. / narcsec) #flatcow[Yo,Xo] += (iv>0) * (1/tim.sig1**2) iv1 = 1. / tim.sig1**2 psfsizemap[Yo, Xo] += iv1 * (1. / narcsec) flatcow[Yo, Xo] += iv1 if detmaps: # point-source depth detsig1 = tim.sig1 / tim.psfnorm psfdetiv[Yo, Xo] += (iv > 0) * (1. / detsig1**2) # Galaxy detection map gdetsig1 = tim.sig1 / tim.galnorm galdetiv[Yo, Xo] += (iv > 0) * (1. / gdetsig1**2) if ngood: congood[Yo, Xo] += (iv > 0) if mods is not None: # straight-up comod[Yo, Xo] += goodpix * mo # invvar-weighted cowmod[Yo, Xo] += iv * mo # chi-squared cochi2[Yo, Xo] += iv * (im - mo)**2 del mo del goodpix if max: maximg[Yo, Xo] = np.maximum(maximg[Yo, Xo], im * (iv > 0)) del Yo, Xo, im, iv # END of loop over tims # Per-band: cowimg /= np.maximum(cow, tinyw) C.coimgs.append(cowimg) C.cowimgs.append(cow) if mods is not None: cowmod /= np.maximum(cow, tinyw) C.comods.append(cowmod) coresid = cowimg - cowmod coresid[cow == 0] = 0. C.coresids.append(coresid) if allmasks: C.allmasks.append(andmask) if unweighted: coimg /= np.maximum(con, 1) del con if plots: plt.clf() plt.subplot(2, 2, 1) mn, mx = cowimg.min(), cowimg.max() plt.imshow(cowimg, interpolation='nearest', origin='lower', cmap='gray', vmin=mn, vmax=mx) plt.xticks([]) plt.yticks([]) plt.title('weighted img') plt.subplot(2, 2, 2) mycow = cow.copy() # mark zero as special color #mycow[mycow == 0] = np.nan plt.imshow(mycow, interpolation='nearest', origin='lower', cmap='gray', vmin=0) plt.xticks([]) plt.yticks([]) plt.title('weights') plt.subplot(2, 2, 3) plt.imshow(coimg, interpolation='nearest', origin='lower', cmap='gray') plt.xticks([]) plt.yticks([]) plt.title('unweighted img') mycowimg = cowimg.copy() mycowimg[cow == 0] = coimg[cow == 0] plt.subplot(2, 2, 4) plt.imshow(mycowimg, interpolation='nearest', origin='lower', cmap='gray', vmin=mn, vmax=mx) plt.xticks([]) plt.yticks([]) plt.title('patched img') plt.suptitle('band %s' % band) ps.savefig() cowimg[cow == 0] = coimg[cow == 0] if mods is not None: cowmod[cow == 0] = comod[cow == 0] if xy: C.T.nobs[:, iband] = nobs[iy, ix] C.T.anymask[:, iband] = ormask[iy, ix] C.T.allmask[:, iband] = andmask[iy, ix] # unless there were no images there... C.T.allmask[nobs[iy, ix] == 0, iband] = 0 if detmaps: C.T.psfdepth[:, iband] = psfdetiv[iy, ix] C.T.galdepth[:, iband] = galdetiv[iy, ix] if psfsize: # psfsizemap is accumulated in units of iv * (1 / arcsec**2) # take out the weighting psfsizemap /= np.maximum(flatcow, tinyw) # Correction factor to get back to equivalent of Gaussian sigma tosigma = 1. / (2. * np.sqrt(np.pi)) # Conversion factor to FWHM (2.35) tofwhm = 2. * np.sqrt(2. * np.log(2.)) # Scale back to units of linear arcsec. psfsizemap[:, :] = (1. / np.sqrt(psfsizemap)) * tosigma * tofwhm psfsizemap[flatcow == 0] = 0. if xy: C.T.psfsize[:, iband] = psfsizemap[iy, ix] if apertures is not None: # Aperture photometry, using the unweighted "coimg" and # "coiv" arrays. with np.errstate(divide='ignore'): imsigma = 1.0 / np.sqrt(coiv) imsigma[coiv == 0] = 0 for irad, rad in enumerate(apertures): apargs.append((irad, band, rad, coimg, imsigma, True, apxy)) if mods is not None: apargs.append( (irad, band, rad, coresid, None, False, apxy)) if callback is not None: callback(band, *callback_args, **kwargs) # END of loop over bands t2 = Time() debug('coadds: images:', t2 - t0) if plots: I = np.argsort([a[0] for a in allresids]) cols = int(np.ceil(np.sqrt(len(I)))) rows = int(np.ceil(len(I) / float(cols))) allresids = [allresids[i] for i in I] plt.clf() for i, (y, n, img, mod, res) in enumerate(allresids): plt.subplot(rows, cols, i + 1) plt.imshow(img, interpolation='nearest', origin='lower', cmap='gray') plt.xticks([]) plt.yticks([]) plt.title('%.1f: %s' % (y, n)) plt.suptitle('Data') ps.savefig() if mods is not None: plt.clf() for i, (y, n, img, mod, res) in enumerate(allresids): plt.subplot(rows, cols, i + 1) plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray') plt.xticks([]) plt.yticks([]) plt.title('%.1f: %s' % (y, n)) plt.suptitle('Model') ps.savefig() plt.clf() for i, (y, n, img, mod, res) in enumerate(allresids): plt.subplot(rows, cols, i + 1) plt.imshow(res, interpolation='nearest', origin='lower', cmap='gray', vmin=-20, vmax=20) plt.xticks([]) plt.yticks([]) plt.title('%.1f: %s' % (y, n)) plt.suptitle('Resids') ps.savefig() if xy is not None: C.T.mjd_min = mjds[mjd_argmins[iy, ix]] C.T.mjd_max = mjds[mjd_argmaxs[iy, ix]] del mjd_argmins del mjd_argmaxs if apertures is not None: # Aperture phot, in parallel if mp is not None: apresults = mp.map(_apphot_one, apargs) else: apresults = map(_apphot_one, apargs) del apargs apresults = iter(apresults) for iband, band in enumerate(bands): apimg = [] apimgerr = [] if mods is not None: apres = [] for irad, rad in enumerate(apertures): # py2 if hasattr(apresults, 'next'): (airad, aband, isimg, ap_img, ap_err) = apresults.next() # py3 else: (airad, aband, isimg, ap_img, ap_err) = apresults.__next__() assert (airad == irad) assert (aband == band) assert (isimg) apimg.append(ap_img) apimgerr.append(ap_err) if mods is not None: # py2 if hasattr(apresults, 'next'): (airad, aband, isimg, ap_img, ap_err) = apresults.next() else: (airad, aband, isimg, ap_img, ap_err) = apresults.__next__() assert (airad == irad) assert (aband == band) assert (not isimg) apres.append(ap_img) assert (ap_err is None) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. C.AP.set('apflux_img_%s' % band, ap) ap = 1. / (np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. C.AP.set('apflux_img_ivar_%s' % band, ap) if mods is not None: ap = np.vstack(apres).T ap[np.logical_not(np.isfinite(ap))] = 0. C.AP.set('apflux_resid_%s' % band, ap) t3 = Time() debug('coadds apphot:', t3 - t2) return C
def _tractor_coadds(sample, targetwcs, tims, mods, version_header, objid=None, brickname=None, survey=None, mp=None, log=None, bands=['g', 'r', 'z']): """Generate individual-band FITS and color coadds for each central using Tractor. """ from legacypipe.coadds import make_coadds, write_coadd_images #from legacypipe.runbrick import rgbkwargs, rgbkwargs_resid from legacypipe.survey import get_rgb, imsave_jpeg if brickname is None: brickname = sample['brickname'] print('Producing coadds...', flush=True, file=log) if log: with redirect_stdout(log), redirect_stderr(log): C = make_coadds(tims, bands, targetwcs, mods=mods, mp=mp, callback=write_coadd_images, callback_args=(survey, brickname, version_header, tims, targetwcs)) else: C = make_coadds(tims, bands, targetwcs, mods=mods, mp=mp, callback=write_coadd_images, callback_args=(survey, brickname, version_header, tims, targetwcs)) # Move (rename) the coadds into the desired output directory. for suffix in np.atleast_1d('model'): for band in bands: shutil.copy( os.path.join( survey.output_dir, 'coadd', brickname[:3], brickname, 'legacysurvey-{}-{}-{}.fits.fz'.format( brickname, suffix, band)), os.path.join( survey.output_dir, '{}-{}-nocentral-{}.fits.fz'.format(objid, suffix, band))) if True: shutil.rmtree(os.path.join(survey.output_dir, 'coadd')) # Build png postage stamps of the coadds. rgbkwargs = dict(mnmx=(-1, 100), arcsinh=1) #rgbkwargs_resid = dict(mnmx=(0.1, 2), arcsinh=1) rgbkwargs_resid = dict(mnmx=(-1, 100), arcsinh=1) #coadd_list = [('image-central', C.coimgs, rgbkwargs), # ('model-central', C.comods, rgbkwargs), # ('resid-central', C.coresids, rgbkwargs_resid)] coadd_list = [('model-nocentral', C.comods, rgbkwargs), ('image-central', C.coresids, rgbkwargs_resid)] for name, ims, rgbkw in coadd_list: rgb = get_rgb(ims, bands, **rgbkw) kwa = {} outfn = os.path.join(survey.output_dir, '{}-{}.jpg'.format(objid, name)) print('Writing {}'.format(outfn), flush=True, file=log) imsave_jpeg(outfn, rgb, origin='lower', **kwa) del rgb
def _make_coadds_plots_1(im, band, mods, mo, iv, unweighted, dq, satur_val, allresids, ps, H, W, tim, Yo, Xo): from legacypipe.survey import get_rgb import pylab as plt # # Make one grayscale, brick-space plot per image # thisimg = np.zeros((H,W), np.float32) # thisimg[Yo,Xo] = im # rgb = get_rgb([thisimg], [band]) # rgb = rgb.sum(axis=2) # fn = ps.getnext() # plt.imsave(fn, rgb, origin='lower', cmap='gray') #plt.clf() #plt.imshow(rgb, interpolation='nearest', origin='lower', cmap='gray') #plt.xticks([]); plt.yticks([]) #ps.savefig() # Image, Model, and Resids plt.clf() plt.subplot(2,2,1) thisimg = np.zeros((H,W), np.float32) thisimg[Yo,Xo] = im rgb = get_rgb([thisimg], [band]) iplane = dict(g=2, r=1, z=0)[band] rgbimg = rgb[:,:,iplane] plt.imshow(rgbimg, interpolation='nearest', origin='lower', cmap='gray') plt.xticks([]); plt.yticks([]) if mods is not None: plt.subplot(2,2,2) thismod = np.zeros((H,W), np.float32) thismod[Yo,Xo] = mo rgb = get_rgb([thismod], [band]) rgbmod = rgb[:,:,iplane] plt.imshow(rgbmod, interpolation='nearest', origin='lower', cmap='gray') plt.xticks([]); plt.yticks([]) plt.subplot(2,2,3) thisres = np.zeros((H,W), np.float32) thisres[Yo,Xo] = (im - mo) * np.sqrt(iv) plt.imshow(thisres, interpolation='nearest', origin='lower', cmap='gray', vmin=-20, vmax=20) plt.xticks([]); plt.yticks([]) else: if unweighted and (dq is not None): # HACK -- copy-n-pasted code from below. okbits = 0 #for bitname in ['satur', 'bleed']: for bitname in ['satur']: okbits |= DQ_BITS[bitname] brightpix = ((dq & okbits) != 0) myim = im.copy() if satur_val is not None: # HACK -- force SATUR pix to be bright myim[brightpix] = satur_val #for bitname in ['interp']: for bitname in ['interp', 'bleed']: okbits |= DQ_BITS[bitname] goodpix = ((dq & ~okbits) == 0) thisgood = np.zeros((H,W), np.float32) thisgood[Yo,Xo] = goodpix plt.subplot(2,2,2) plt.imshow(thisgood, interpolation='nearest', origin='lower', cmap='gray', vmin=0, vmax=1) plt.xticks([]); plt.yticks([]) plt.title('goodpix') thisim = np.zeros((H,W), np.float32) thisim[Yo,Xo] = goodpix * myim rgb = get_rgb([thisim], [band]) iplane = dict(g=2, r=1, z=0)[band] rgbimg = rgb[:,:,iplane] plt.subplot(2,2,3) plt.imshow(rgbimg, interpolation='nearest', origin='lower', cmap='gray') plt.xticks([]); plt.yticks([]) plt.title('goodpix rgb') rgbmod=None thisres=None plt.subplot(2,2,4) thisiv = np.zeros((H,W), np.float32) thisiv[Yo,Xo] = iv plt.imshow(thisiv, interpolation='nearest', origin='lower', cmap='gray') plt.xticks([]); plt.yticks([]) plt.title('invvar') plt.suptitle(tim.name + ': %.2f' % (tim.time.toYear())) ps.savefig() allresids.append((tim.time.toYear(), tim.name, rgbimg,rgbmod,thisres))
def stage_fit_on_coadds(survey=None, targetwcs=None, pixscale=None, bands=None, tims=None, brickname=None, version_header=None, coadd_tiers=None, apodize=True, subsky=True, ubercal_sky=False, subsky_radii=None, nsatur=None, fitoncoadds_reweight_ivar=True, plots=False, plots2=False, ps=None, coadd_bw=False, W=None, H=None, brick=None, blobs=None, lanczos=True, ccds=None, write_metrics=True, mp=None, record_event=None, **kwargs): from legacypipe.coadds import make_coadds from legacypipe.bits import DQ_BITS from legacypipe.survey import LegacySurveyWcs from legacypipe.coadds import get_coadd_headers from tractor.image import Image from tractor.basics import LinearPhotoCal from tractor.sky import ConstantSky from tractor.psf import PixelizedPSF from tractor.tractortime import TAITime import astropy.time import fitsio if plots or plots2: import pylab as plt from legacypipe.survey import get_rgb # Custom sky-subtraction for large galaxies. skydict = {} if not subsky: if ubercal_sky: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('fitoncoadds-{}'.format(brickname)) tims, skydict = ubercal_skysub(tims, targetwcs, survey, brickname, bands, mp, subsky_radii=subsky_radii, plots=True, plots2=False, ps=ps, verbose=True) else: print('Skipping sky-subtraction entirely.') # Create coadds and then build custom tims from them. for tim in tims: ie = tim.inverr if np.any(ie < 0): print('Negative inverse error in image {}'.format(tim.name)) CC = [] if coadd_tiers: # Sort by band and sort them into tiers. tiers = [[] for i in range(coadd_tiers)] for b in bands: btims = [] seeing = [] for tim in tims: if tim.band != b: continue btims.append(tim) seeing.append(tim.psf_fwhm * tim.imobj.pixscale) I = np.argsort(seeing) btims = [btims[i] for i in I] seeing = [seeing[i] for i in I] N = min(coadd_tiers, len(btims)) splits = np.round(np.arange(N + 1) * float(len(btims)) / N).astype(int) print('Splitting', len(btims), 'images into', N, 'tiers: splits:', splits) print('Seeing limits:', [seeing[min(s, len(seeing) - 1)] for s in splits]) for s0, s1, tt in zip(splits, splits[1:], tiers): tt.extend(btims[s0:s1]) for itier, tier in enumerate(tiers): print('Producing coadds for tier', (itier + 1)) C = make_coadds( tier, bands, targetwcs, detmaps=True, ngood=True, lanczos=lanczos, allmasks=True, anymasks=True, psf_images=True, nsatur=2, mp=mp, plots=plots2, ps=ps, # note plots2 here! callback=None) if plots: plt.clf() for iband, (band, psf) in enumerate(zip(bands, C.psf_imgs)): plt.subplot(1, len(bands), iband + 1) plt.imshow(psf, interpolation='nearest', origin='lower') plt.title('Coadd PSF image: band %s' % band) plt.suptitle('Tier %i' % (itier + 1)) ps.savefig() # for band,img in zip(bands, C.coimgs): # plt.clf() # plt.imshow(img, plt.clf() plt.imshow(get_rgb(C.coimgs, bands), origin='lower') plt.title('Tier %i' % (itier + 1)) ps.savefig() CC.append(C) else: C = make_coadds( tims, bands, targetwcs, detmaps=True, ngood=True, lanczos=lanczos, allmasks=True, anymasks=True, psf_images=True, mp=mp, plots=plots2, ps=ps, # note plots2 here! callback=None) CC.append(C) cotims = [] for C in CC: if plots2: for band, iv in zip(bands, C.cowimgs): pass # plt.clf() # plt.imshow(np.sqrt(iv), interpolation='nearest', origin='lower') # plt.title('Coadd Inverr: band %s' % band) # ps.savefig() for band, psf in zip(bands, C.psf_imgs): plt.clf() plt.imshow(psf, interpolation='nearest', origin='lower') plt.title('Coadd PSF image: band %s' % band) ps.savefig() for band, img, iv in zip(bands, C.coimgs, C.cowimgs): from scipy.ndimage.filters import gaussian_filter # plt.clf() # plt.hist((img * np.sqrt(iv))[iv>0], bins=50, range=(-5,8), log=True) # plt.title('Coadd pixel values (sigmas): band %s' % band) # ps.savefig() psf_sigma = np.mean([ (tim.psf_sigma * tim.imobj.pixscale / pixscale) for tim in tims if tim.band == band ]) gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) psfnorm = gnorm #np.sqrt(np.sum(psfimg**2)) detim = gaussian_filter(img, psf_sigma) / psfnorm**2 cosig1 = 1. / np.sqrt(np.median(iv[iv > 0])) detsig1 = cosig1 / psfnorm # plt.clf() # plt.subplot(2,1,1) # plt.hist(detim.ravel() / detsig1, bins=50, range=(-5,8), log=True) # plt.title('Coadd detection map values / sig1 (sigmas): band %s' % band) # plt.subplot(2,1,2) # plt.hist(detim.ravel() / detsig1, bins=50, range=(-5,8)) # ps.savefig() # # as in detection.py # detiv = np.zeros_like(detim) + (1. / detsig1**2) # detiv[iv == 0] = 0. # detiv = gaussian_filter(detiv, psf_sigma) # # plt.clf() # plt.hist((detim * np.sqrt(detiv)).ravel(), bins=50, range=(-5,8), log=True) # plt.title('Coadd detection map values / detie (sigmas): band %s' % band) # ps.savefig() for iband, (band, img, iv, allmask, anymask, psfimg) in enumerate( zip(bands, C.coimgs, C.cowimgs, C.allmasks, C.anymasks, C.psf_imgs)): mjd = np.mean( [tim.imobj.mjdobs for tim in tims if tim.band == band]) mjd_tai = astropy.time.Time(mjd, format='mjd', scale='utc').tai.mjd tai = TAITime(None, mjd=mjd_tai) twcs = LegacySurveyWcs(targetwcs, tai) #print('PSF sigmas (in pixels) for band', band, ':', # ['%.2f' % tim.psf_sigma for tim in tims if tim.band == band]) print( 'PSF sigmas in coadd pixels:', ', '.join([ '%.2f' % (tim.psf_sigma * tim.imobj.pixscale / pixscale) for tim in tims if tim.band == band ])) psf_sigma = np.mean([ (tim.psf_sigma * tim.imobj.pixscale / pixscale) for tim in tims if tim.band == band ]) print('Using average PSF sigma', psf_sigma) psf = PixelizedPSF(psfimg) gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) psfnorm = np.sqrt(np.sum(psfimg**2)) print('Gaussian PSF norm', gnorm, 'vs pixelized', psfnorm) # if plots: # from collections import Counter # plt.clf() # plt.imshow(mask, interpolation='nearest', origin='lower') # plt.colorbar() # plt.title('allmask') # ps.savefig() # print('allmask for band', band, ': values:', Counter(mask.ravel())) # Scale invvar to take into account that we have resampled (~double-counted) pixels tim_pixscale = np.mean( [tim.imobj.pixscale for tim in tims if tim.band == band]) cscale = tim_pixscale / pixscale print('average tim pixel scale / coadd scale:', cscale) iv /= cscale**2 if fitoncoadds_reweight_ivar: # We first tried setting the invvars constant per tim -- this # makes things worse, since we *remove* the lowered invvars at # the cores of galaxies. # # Here we're hacking the relative weights -- squaring the # weights but then making the median the same, ie, squaring # the dynamic range or relative weights -- ie, downweighting # the cores even more than they already are from source # Poisson terms. median_iv = np.median(iv[iv > 0]) assert (median_iv > 0) iv = iv * np.sqrt(iv) / np.sqrt(median_iv) assert (np.all(np.isfinite(iv))) assert (np.all(iv >= 0)) cotim = Image(img, invvar=iv, wcs=twcs, psf=psf, photocal=LinearPhotoCal(1., band=band), sky=ConstantSky(0.), name='coadd-' + band) cotim.band = band cotim.subwcs = targetwcs cotim.psf_sigma = psf_sigma cotim.sig1 = 1. / np.sqrt(np.median(iv[iv > 0])) # Often, SATUR masks on galaxies / stars are surrounded by BLEED pixels. Soak these into # the SATUR mask. from scipy.ndimage.morphology import binary_dilation anymask |= np.logical_and(((anymask & DQ_BITS['bleed']) > 0), binary_dilation( ((anymask & DQ_BITS['satur']) > 0), iterations=10)) * DQ_BITS['satur'] # Saturated in any image -> treat as saturated in coadd # (otherwise you get weird systematics in the weighted coadds, and weird source detection!) mask = allmask mask[(anymask & DQ_BITS['satur'] > 0)] |= DQ_BITS['satur'] if coadd_tiers: # nsatur -- reset SATUR bit mask &= ~DQ_BITS['satur'] mask |= DQ_BITS['satur'] * C.satmaps[iband] cotim.dq = mask cotim.dq_saturation_bits = DQ_BITS['satur'] cotim.psfnorm = gnorm cotim.galnorm = 1.0 # bogus! cotim.imobj = Duck() cotim.imobj.fwhm = 2.35 * psf_sigma cotim.imobj.pixscale = pixscale cotim.time = tai cotim.primhdr = fitsio.FITSHDR() get_coadd_headers(cotim.primhdr, tims, band, coadd_headers=skydict) cotims.append(cotim) if plots: plt.clf() bitmap = dict([(v, k) for k, v in DQ_BITS.items()]) k = 1 for i in range(12): bitval = 1 << i if not bitval in bitmap: continue # only 9 bits are actually used plt.subplot(3, 3, k) k += 1 plt.imshow((cotim.dq & bitval) > 0, vmin=0, vmax=1.5, cmap='hot', origin='lower') plt.title(bitmap[bitval]) plt.suptitle('Coadd mask planes %s band' % band) ps.savefig() plt.clf() h, w = cotim.shape rgb = np.zeros((h, w, 3), np.uint8) rgb[:, :, 0] = (cotim.dq & DQ_BITS['satur'] > 0) * 255 rgb[:, :, 1] = (cotim.dq & DQ_BITS['bleed'] > 0) * 255 plt.imshow(rgb, origin='lower') plt.suptitle('Coadd DQ band %s: red = SATUR, green = BLEED' % band) ps.savefig() # Save an image of the coadd PSF # copy version_header before modifying it. hdr = fitsio.FITSHDR() for r in version_header.records(): hdr.add_record(r) hdr.add_record( dict(name='IMTYPE', value='coaddpsf', comment='LegacySurveys image type')) hdr.add_record( dict(name='BAND', value=band, comment='Band of this coadd/PSF')) hdr.add_record( dict(name='PSF_SIG', value=psf_sigma, comment='Average PSF sigma (coadd pixels)')) hdr.add_record( dict(name='PIXSCAL', value=pixscale, comment='Pixel scale of this PSF (arcsec)')) hdr.add_record( dict(name='INPIXSC', value=tim_pixscale, comment='Native image pixscale scale (average, arcsec)')) hdr.add_record( dict(name='MJD', value=mjd, comment='Average MJD for coadd')) hdr.add_record( dict(name='MJD_TAI', value=mjd_tai, comment='Average MJD (in TAI) for coadd')) with survey.write_output('copsf', brick=brickname, band=band) as out: out.fits.write(psfimg, header=hdr) # EVIL return dict(tims=cotims, coadd_headers=skydict)
def ubercal_skysub(tims, targetwcs, survey, brickname, bands, mp, subsky_radii=None, plots=False, plots2=False, ps=None, verbose=False): """With the ubercal option, we (1) read the full-field mosaics ('bandtims') for a given bandpass and put them all on the same 'system' using the overlapping pixels; (2) apply the derived corrections to the in-field 'tims'; (3) build the coadds (per bandpass) from the 'tims'; and (4) subtract the median sky from the mosaic (after aggressively masking objects and reference sources). """ from tractor.sky import ConstantSky from legacypipe.reference import get_reference_sources, get_reference_map from legacypipe.coadds import make_coadds from legacypipe.survey import get_rgb, imsave_jpeg from astropy.stats import sigma_clipped_stats if plots or plots2: import os import matplotlib.pyplot as plt if plots: plt.figure(figsize=(8, 6)) mods = [] for tim in tims: imcopy = tim.getImage().copy() tim.sky.addTo(imcopy, -1) mods.append(imcopy) C = make_coadds(tims, bands, targetwcs, mods=mods, callback=None, mp=mp) imsave_jpeg(os.path.join(survey.output_dir, 'metrics', 'cus', '{}-pipelinesky.jpg'.format(ps.basefn)), get_rgb(C.comods, bands), origin='lower') skydict = {} if subsky_radii is not None: skydict.update( {'NSKYANN': (len(subsky_radii) // 2, 'number of sky annuli')}) for irad, (rin, rout) in enumerate( zip(subsky_radii[0::2], subsky_radii[1::2])): skydict.update({ 'SKYRIN{:02d}'.format(irad): (np.float32(rin), 'inner sky radius {} [arcsec]'.format(irad)) }) skydict.update({ 'SKYROT{:02d}'.format(irad): (np.float32(rout), 'outer sky radius {} [arcsec]'.format(irad)) }) allbands = np.array([tim.band for tim in tims]) # we need the full-field mosaics if doing annular sky-subtraction if subsky_radii is not None: allbandtims = [] else: allbandtims = None for band in sorted(set(allbands)): print('Working on band {}'.format(band)) I = np.where(allbands == band)[0] bandtims = [ tims[ii].imobj.get_tractor_image(gaussPsf=True, pixPsf=False, subsky=False, dq=True, apodize=False) for ii in I ] # Derive the ubercal correction and then apply it. x = coadds_ubercal(bandtims, coaddtims=[tims[ii] for ii in I], plots=plots, plots2=plots2, ps=ps, verbose=True) for ii, corr in zip(I, x): skydict.update({ 'SKCCD{:03d}'.format(ii): (tims[ii].name, 'ubersky CCD {:03d}'.format(ii)) }) skydict.update({ 'SKCOR{:03d}'.format(ii): (corr, 'ubersky corr CCD {:03d}'.format(ii)) }) # Apply the correction and return the tims for jj, (correction, ii) in enumerate(zip(x, I)): tims[ii].data += correction tims[ii].sky = ConstantSky(0.0) # Also correct the full-field mosaics bandtims[jj].data += correction bandtims[jj].sky = ConstantSky(0.0) ## Check-- #for jj, correction in enumerate(x): # fulltims[jj].data += correction #newcorrection = coadds_ubercal(fulltims) #print(newcorrection) if allbandtims is not None: allbandtims = allbandtims + bandtims # Estimate the sky background from an annulus surrounding the object # (assumed to be at the center of the mosaic, targetwcs.crval). if subsky_radii is not None: from astrometry.util.util import Tan # the inner and outer radii / annuli are nested in subsky_radii allrin = subsky_radii[0::2] allrout = subsky_radii[1::2] pixscale = targetwcs.pixel_scale() bigH = float(np.ceil(2 * np.max(allrout) / pixscale)) bigW = bigH # if doing annulur sky-subtraction we need a bigger mosaic bigtargetwcs = Tan(targetwcs.crval[0], targetwcs.crval[1], bigW / 2. + 0.5, bigH / 2. + 0.5, -pixscale / 3600.0, 0., 0., pixscale / 3600.0, float(bigW), float(bigH)) C = make_coadds(allbandtims, bands, bigtargetwcs, callback=None, sbscale=False, mp=mp) _, x0, y0 = bigtargetwcs.radec2pixelxy(bigtargetwcs.crval[0], bigtargetwcs.crval[1]) xcen, ycen = np.round(x0 - 1).astype('int'), np.round(y0 - 1).astype('int') ymask, xmask = np.ogrid[-ycen:bigH - ycen, -xcen:bigW - xcen] refs, _ = get_reference_sources(survey, bigtargetwcs, bigtargetwcs.pixel_scale(), ['r'], tycho_stars=True, gaia_stars=True, large_galaxies=True, star_clusters=True) refmask = get_reference_map(bigtargetwcs, refs) == 0 # True=skypix for coimg, coiv, band in zip(C.coimgs, C.cowimgs, bands): skypix = _build_objmask(coimg, coiv, refmask * (coiv > 0)) for irad, (rin, rout) in enumerate(zip(allrin, allrout)): inmask = (xmask**2 + ymask**2) <= (rin / pixscale)**2 outmask = (xmask**2 + ymask**2) <= (rout / pixscale)**2 skymask = (outmask * 1 - inmask * 1) == 1 # True=skypix # Find and mask objects, then get the sky. skypix_annulus = np.logical_and(skypix, skymask) #import matplotlib.pyplot as plt ; plt.imshow(skypix_annulus, origin='lower') ; plt.savefig('junk3.png') #import pdb ; pdb.set_trace() if np.sum(skypix_annulus) == 0: raise ValueError('No pixels in sky!') _skymean, _skymedian, _skysig = sigma_clipped_stats( coimg, mask=np.logical_not(skypix_annulus), sigma=3.0) skydict.update({ '{}SKYMN{:02d}'.format(band.upper(), irad): (np.float32(_skymean), 'mean {} sky in annulus {}'.format(band, irad)) }) skydict.update({ '{}SKYMD{:02d}'.format(band.upper(), irad): (np.float32(_skymedian), 'median {} sky in annulus {}'.format(band, irad)) }) skydict.update({ '{}SKYSG{:02d}'.format(band.upper(), irad): (np.float32(_skysig), 'sigma {} sky in annulus {}'.format(band, irad)) }) skydict.update({ '{}SKYNP{:02d}'.format(band.upper(), irad): (np.sum(skypix_annulus), 'npix {} sky in annulus {}'.format(band, irad)) }) # the reference annulus is the first one if irad == 0: skymean, skymedian, skysig = _skymean, _skymedian, _skysig skypix_mask = skypix_annulus I = np.where(allbands == band)[0] for ii in I: tims[ii].data -= skymedian #print('Tim', tims[ii], 'after subtracting skymedian: median', np.median(tims[ii].data)) else: # regular mosaic C = make_coadds(tims, bands, targetwcs, callback=None, sbscale=False, mp=mp) refs, _ = get_reference_sources(survey, targetwcs, targetwcs.pixel_scale(), ['r'], tycho_stars=True, gaia_stars=True, large_galaxies=True, star_clusters=True) refmask = get_reference_map(targetwcs, refs) == 0 # True=skypix for coimg, coiv, band in zip(C.coimgs, C.cowimgs, bands): skypix = refmask * (coiv > 0) skypix_mask = _build_objmask(coimg, coiv, skypix) skymean, skymedian, skysig = sigma_clipped_stats( coimg, mask=np.logical_not(skypix_mask), sigma=3.0) skydict.update({ '{}SKYMN00'.format(band.upper(), irad): (np.float32(_skymean), 'mean {} sky'.format(band)) }) skydict.update({ '{}SKYMD00'.format(band.upper(), irad): (np.float32(_skymedian), 'median {} sky'.format(band)) }) skydict.update({ '{}SKYSG00'.format(band.upper(), irad): (np.float32(_skysig), 'sigma {} sky'.format(band)) }) skydict.update({ '{}SKYNP00'.format(band.upper(), irad): (np.sum(skypix_mask), 'npix {} sky'.format(band)) }) I = np.where(allbands == band)[0] for ii in I: tims[ii].data -= skymedian # print('Tim', tims[ii], 'after subtracting skymedian: median', np.median(tims[ii].data)) #print('Band', band, 'Coadd sky:', skymedian) if plots2: plt.clf() plt.hist(coimg.ravel(), bins=50, range=(-3, 3), density=True) plt.axvline(skymedian, color='k') for ii in I: #print('Tim', tims[ii], 'median', np.median(tims[ii].data)) plt.hist((tims[ii].data - skymedian).ravel(), bins=50, range=(-3, 3), histtype='step', density=True) plt.title('Band %s: tim pix & skymedian' % band) ps.savefig() # Produce skymedian-subtracted, masked image for later RGB plot coimg -= skymedian coimg[~skypix_mask] = 0. #coimg[np.logical_not(skymask * (coiv > 0))] = 0. if plots2: plt.clf() plt.imshow(get_rgb(C.coimgs, bands), origin='lower', interpolation='nearest') ps.savefig() for band in bands: for tim in tims: if tim.band != band: continue plt.clf() C = make_coadds([tim], bands, targetwcs, callback=None, sbscale=False, mp=mp) plt.imshow(get_rgb(C.coimgs, bands).sum(axis=2), cmap='gray', interpolation='nearest', origin='lower') plt.title('Band %s: tim %s' % (band, tim.name)) ps.savefig() if plots: import pylab as plt import matplotlib.patches as patches # skysub QA C = make_coadds(tims, bands, targetwcs, callback=None, mp=mp) imsave_jpeg(os.path.join(survey.output_dir, 'metrics', 'cus', '{}-customsky.jpg'.format(ps.basefn)), get_rgb(C.coimgs, bands), origin='lower') # ccdpos QA refs, _ = get_reference_sources(survey, targetwcs, targetwcs.pixel_scale(), ['r'], tycho_stars=False, gaia_stars=False, large_galaxies=True, star_clusters=False) pixscale = targetwcs.pixel_scale() width = targetwcs.get_width() * pixscale / 3600 # [degrees] bb, bbcc = targetwcs.radec_bounds(), targetwcs.radec_center( ) # [degrees] pad = 0.5 * width # [degrees] delta = np.max((np.diff(bb[0:2]), np.diff(bb[2:4]))) / 2 + pad / 2 xlim = bbcc[0] - delta, bbcc[0] + delta ylim = bbcc[1] - delta, bbcc[1] + delta plt.clf() _, allax = plt.subplots(1, 3, figsize=(12, 5), sharey=True, sharex=True) for ax, band in zip(allax, ('g', 'r', 'z')): ax.set_xlabel('RA (deg)') ax.text(0.9, 0.05, band, ha='center', va='bottom', transform=ax.transAxes, fontsize=18) if band == 'g': ax.set_ylabel('Dec (deg)') ax.get_xaxis().get_major_formatter().set_useOffset(False) # individual CCDs these = np.where([tim.band == band for tim in tims])[0] col = plt.cm.Set1(np.linspace(0, 1, len(tims))) for ii, indx in enumerate(these): tim = tims[indx] #wcs = tim.subwcs wcs = tim.imobj.get_wcs() cc = wcs.radec_bounds() ax.add_patch( patches.Rectangle((cc[0], cc[2]), cc[1] - cc[0], cc[3] - cc[2], fill=False, lw=2, edgecolor=col[these[ii]], label='ccd{:02d}'.format(these[ii]))) ax.legend(ncol=2, frameon=False, loc='upper left', fontsize=10) # output mosaic footprint cc = targetwcs.radec_bounds() ax.add_patch( patches.Rectangle((cc[0], cc[2]), cc[1] - cc[0], cc[3] - cc[2], fill=False, lw=2, edgecolor='k')) if subsky_radii is not None: racen, deccen = targetwcs.crval for rad in subsky_radii: ax.add_patch( patches.Circle((racen, deccen), rad / 3600, fill=False, edgecolor='black', lw=2)) else: for gal in refs: ax.add_patch( patches.Circle((gal.ra, gal.dec), gal.radius, fill=False, edgecolor='black', lw=2)) ax.set_ylim(ylim) ax.set_xlim(xlim) ax.invert_xaxis() ax.set_aspect('equal') plt.subplots_adjust(bottom=0.12, wspace=0.05, left=0.12, right=0.97, top=0.95) plt.savefig( os.path.join(survey.output_dir, 'metrics', 'cus', '{}-ccdpos.jpg'.format(ps.basefn))) if plots2: plt.clf() for coimg, band in zip(C.coimgs, bands): plt.hist(coimg.ravel(), bins=50, range=(-0.5, 0.5), histtype='step', label=band) plt.legend() plt.title('After adjustment: coadds (sb scaled)') ps.savefig() return tims, skydict
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--catalog', help='Catalog to render') parser.add_argument('--brick-coadd', help='Produce a coadd of the images overlapping the given brickname.') parser.add_argument('--ccds', help='Use this table of CCDs') parser.add_argument('--brick-wcs', help='File containing a WCS header describing the coadd WCS to render.') parser.add_argument('--brick-wcs-ext', type=int, help='FITS file extension containing a WCS header describing the coadd WCS to render.') parser.add_argument('--outlier-mask-brick', help='Comma-separated list of bricknames from which outlier masks should be read.') parser.add_argument('--out', help='Filename pattern ("BAND" will be replaced by band name) of output images.') parser.add_argument('--resid', help='Filename pattern ("BAND" will be replaced by band name) of residual images.') parser.add_argument('--jpeg', help='Write RGB image to this filename') parser.add_argument('--resid-jpeg', help='Write RGB residual image to this filename') opt = parser.parse_args() if opt.catalog is None: print('Need catalog!') return -1 cat = fits_table(opt.catalog) if opt.ccds is None: if opt.brick_coadd is None: print('Need brick catalog!') return -1 brickname = opt.brick_coadd survey = LegacySurveyData() if opt.brick_wcs is None: print('FIXME') return -1 else: wcs = Tan(opt.brick_wcs, opt.brick_wcs_ext) tcat = read_fits_catalog(cat) if opt.ccds: ccdfn = opt.ccds else: ccdfn = survey.find_file('ccds-table', brick=brickname) print('Reading', ccdfn) ccds = fits_table(ccdfn) H,W = wcs.shape targetrd = np.array([wcs.pixelxy2radec(x,y) for x,y in [(1,1),(W,1),(W,H),(1,H),(1,1)]]) tims = [] for ccd in ccds: im = survey.get_image_object(ccd) #slc = slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1) #tim = im.get_tractor_image(slc=slc) tim = im.get_tractor_image(radecpoly=targetrd) print('Read', tim) tims.append(tim) if opt.outlier_mask_brick is not None: bricks = opt.outlier_mask_brick.split(',') for b in bricks: print('Reading outlier mask for brick', b, ':', survey.find_file('outliers_mask', brick=b, output=False)) ok = read_outlier_mask_file(survey, tims, b, subimage=True, output=False) tr = Tractor(tims, tcat) mods = list(tr.getModelImages()) bands = 'grz' def write_model(band, cowimg=None, cowmod=None, **kwargs): if cowmod is None: print('No model for', band) return outfn = opt.out.replace('BAND', band) fitsio.write(outfn, cowmod, clobber=True) print('Wrote model for', band, 'to', outfn) if opt.resid: outfn = opt.resid.replace('BAND', band) fitsio.write(outfn, cowimg - cowmod, clobber=True) print('Wrote resid for', band, 'to', outfn) C = make_coadds(tims, bands, wcs, mods=mods, callback=write_model) if opt.jpeg: from legacypipe.survey import get_rgb import pylab as plt plt.imsave(opt.jpeg, get_rgb(C.comods, bands), origin='lower') if opt.resid_jpeg: from legacypipe.survey import get_rgb import pylab as plt plt.imsave(opt.resid_jpeg, get_rgb([im-mod for im,mod in zip(C.coimgs, C.comods)], bands), origin='lower')
def top_levels(mp, opt): from map.views import save_jpeg, trymakedirs if opt.kind in ['decaps2', 'decaps2-model', 'decaps2-resid', 'mzls+bass-dr4', 'mzls+bass-dr4-model', 'mzls+bass-dr4-resid', 'decals-dr5', 'decals-dr5-model', 'decals-dr5-resid', 'decals-dr7', 'decals-dr7-model', 'decals-dr7-resid', 'mzls+bass-dr6', 'mzls+bass-dr6-model', 'mzls+bass-dr6-resid', 'des-dr1', 'eboss', 'unwise-neo2', 'unwise-neo3', 'unwise-neo4', 'unwise-cat-model', 'galex', 'sdss2', 'wssa', 'ls-dr56', 'ls-dr67'] or True: import pylab as plt from viewer import settings import fitsio from scipy.ndimage.filters import gaussian_filter from map.views import _unwise_to_rgb tag = opt.kind from map.views import get_layer layer = get_layer(opt.kind) bands = layer.get_bands() print('Layer:', layer) #print('Survey:', layer.survey) #print(' cache_dir:', layer.survey.cache_dir) print('Bands', bands) rgbkwargs = {} if opt.kind in ['unwise-neo2', 'unwise-neo3', 'unwise-neo4', 'unwise-cat-model']: bands = [1, 2] elif opt.kind == 'sdss2': bands = 'gri' elif opt.kind == 'galex': bands = ['n','f'] elif opt.kind == 'wssa': bands = ['x'] #else: # bands = 'grz' ver = tileversions.get(opt.kind, [1])[-1] print('Version', ver) basescale = 5 pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s', '%(zoom)i', '%(x)i', '%(y)i.jpg') patdata = dict(ver=ver) tilesize = 256 tiles = 2**basescale side = tiles * tilesize basepat = 'base-%s-%i-%%s.fits' % (opt.kind, basescale) basefns = [basepat % band for band in bands] if not all([os.path.exists(fn) for fn in basefns]): bases = [np.zeros((side, side), np.float32) for band in bands] args = [] xy = [] if opt.y1 is None: opt.y1 = tiles if opt.x0 is None: opt.x0 = 0 if opt.x1 is None: opt.x1 = tiles for y in range(opt.y0, opt.y1): for x in range(opt.x0, opt.x1): args.append((opt.kind, basescale, x, y, False, True)) xy.append((x,y)) tiles = mp.map(_one_tile, args) for ims,(x,y) in zip(tiles, xy): #for a,(x,y) in zip(args, xy): #print('_one_tile args:', a) #ims = _one_tile(a) #print('-> ', ims) if ims is None: continue for im,base in zip(ims, bases): if im is None: continue base[y*tilesize:(y+1)*tilesize, x*tilesize:(x+1)*tilesize] = im for fn,base in zip(basefns, bases): fitsio.write(fn, base, clobber=True) else: print('Reading', basefns) bases = [fitsio.read(fn) for fn in basefns] for scale in range(basescale, -1, -1): print('Scale', scale) tiles = 2**scale for y in range(tiles): for x in range(tiles): ims = [base[y*tilesize:(y+1)*tilesize, x*tilesize:(x+1)*tilesize] for base in bases] rgb = layer.get_rgb(ims, bands, **rgbkwargs) pp = patdata.copy() pp.update(zoom=scale, x=x, y=y) fn = pat % pp trymakedirs(fn) save_jpeg(fn, rgb) print('Wrote', fn) for i,base in enumerate(bases): if 'vlass' in opt.kind: base = np.maximum(np.maximum(base[::2,::2], base[1::2,::2]), np.maximum(base[1::2,1::2], base[::2,1::2])) else: base = (base[::2,::2] + base[1::2,::2] + base[1::2,1::2] + base[::2,1::2])/4. bases[i] = base elif opt.kind in ['unwise', 'unwise-neo1', 'unwise-w3w4',]: import pylab as plt from viewer import settings from map.views import _unwise_to_rgb import fitsio if opt.kind == 'unwise-w3w4': tag = 'unwise-w3w4' bands = [3,4] bounce = _bounce_map_unwise_w3w4 elif opt.kind == 'unwise-neo1': tag = 'unwise-neo1' bands = [1,2] bounce = _bounce_map_unwise_neo1 else: tag = 'unwise-w1w2' bands = [1,2] bounce = _bounce_map_unwise_w1w2 pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s', '%(zoom)i', '%(x)i', '%(y)i.jpg') ver = 1 patdata = dict(ver=ver) basescale = 4 tilesize = 256 tiles = 2**basescale side = tiles * tilesize basepat = 'base-%s-%i-%%s.fits' % (opt.kind, basescale) basefns = [basepat % band for band in bands] if not all([os.path.exists(fn) for fn in basefns]): bases = [np.zeros((side, side), np.float32) for band in bands] args = [] for y in range(tiles): for x in range(tiles): #print 'Base tile', x, y args.append((req, ver, basescale, x, y)) tiles = mp.map(bounce, args) for ims,arg in zip(tiles,args): x,y = arg[-2:] for im,base in zip(ims, bases): if im is None: continue base[y*tilesize:(y+1)*tilesize, x*tilesize:(x+1)*tilesize] = im for fn,base in zip(basefns, bases): fitsio.write(fn, base, clobber=True) else: print('Reading', basefns) bases = [fitsio.read(fn) for fn in basefns] if False: # Messin' around plt.figure(figsize=(8,8)) plt.subplots_adjust(left=0, right=1, bottom=0, top=1) #for S in [1000, 3000, 10000]: #for Q in [10, 25, 50]: S,Q = 3000,25 im = _unwise_to_rgb([w1base, w2base], S=S, Q=Q) #plt.clf() #plt.imshow(im) plt.imsave('base-S%i-Q%s.png' % (S,Q), im) # Try converting to galactic coords... from astrometry.util.util import anwcs_create_mercator_2 print('Base images:', w1base.shape) zoom = basescale h,w = w1base.shape zoomscale = 2.**zoom * (256./h) print('Zoomscale', zoomscale) wcs = anwcs_create_mercator_2(180., 0., w/2., h/2., zoomscale, w, h, 1) wcs2 = anwcs_create_mercator_2(0., 0., w/2., h/2., zoomscale, w, h, 1) print('WCS:') for x,y in [(1,1), (1,h), (w,1), (w,h), (w/2,1), (w/2,h/2)]: print('x,y', (x,y), '-> RA,Dec', wcs.pixelxy2radec(x,y)[-2:]) ok,ras,nil = wcs2.pixelxy2radec(np.arange(w), np.ones(w)) ok,nil,decs = wcs2.pixelxy2radec(np.ones(h), np.arange(h)) print('RAs', ras.shape) print('Decs', decs.shape) lls = ras bbs = decs ll,bb = np.meshgrid(lls, bbs) print('LL,BB', ll.shape, bb.shape) from astrometry.util.starutil_numpy import lbtoradec ra,dec = lbtoradec(ll,bb) print('RA,Dec', ra.shape, dec.shape) ok,xx,yy = wcs.radec2pixelxy(ra, dec) print('xx,yy', xx.shape, yy.shape) lb1 = w1base[np.clip(np.round(yy-1).astype(int), 0, h-1), np.clip(np.round(xx-1).astype(int), 0, w-1)] lb2 = w2base[np.clip(np.round(yy-1).astype(int), 0, h-1), np.clip(np.round(xx-1).astype(int), 0, w-1)] lbim = _unwise_to_rgb(lb1, lb2, S=S,Q=Q) plt.imsave('lb.png', lbim) sys.exit(0) from scipy.ndimage.filters import gaussian_filter for scale in range(basescale-1, -1, -1): for i,base in enumerate(bases): base = gaussian_filter(base, 1.) base = (base[::2,::2] + base[1::2,::2] + base[1::2,1::2] + base[::2,1::2])/4. bases[i] = base tiles = 2**scale for y in range(tiles): for x in range(tiles): ims = [base[y*tilesize:(y+1)*tilesize, x*tilesize:(x+1)*tilesize] for base in bases] tile = _unwise_to_rgb(ims, bands=bands) pp = patdata.copy() pp.update(zoom=scale, x=x, y=y) fn = pat % pp trymakedirs(fn) save_jpeg(fn, tile) print('Wrote', fn) if opt.kind in ['depth-g', 'depth-r', 'depth-z']: import pylab as plt from viewer import settings from scipy.ndimage.filters import gaussian_filter from map.views import trymakedirs tag = 'decam-' + opt.kind #band = opt.kind[-1] ver = 1 basescale = 5 pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s', '%(zoom)i', '%(x)i', '%(y)i.jpg') patdata = dict(ver=ver) tilesize = 256 tiles = 2**basescale side = tiles * tilesize base = np.zeros((side, side, 3), np.float32) for y in range(tiles): for x in range(tiles): dat = patdata.copy() dat.update(zoom=basescale, x=x, y=y) fn = pat % dat if not os.path.exists(fn): print('Does not exist:', fn) continue img = plt.imread(fn) base[y*tilesize:(y+1)*tilesize, x*tilesize:(x+1)*tilesize,:] = img for scale in range(basescale-1, -1, -1): newbase = [] for i in range(3): b = gaussian_filter(base[:,:,i], 1.) b = (b[ ::2, ::2] + b[1::2, ::2] + b[1::2,1::2] + b[ ::2,1::2])/4. newbase.append(b) base = np.dstack(newbase) tiles = 2**scale for y in range(tiles): for x in range(tiles): img = base[y*tilesize:(y+1)*tilesize, x*tilesize:(x+1)*tilesize, :] pp = patdata.copy() pp.update(zoom=scale, x=x, y=y) fn = pat % pp trymakedirs(fn) plt.imsave(fn, np.clip(np.round(img).astype(np.uint8), 0, 255)) print('Wrote', fn) ### HACK... factor this out... if opt.kind in ['sdss', 'decals-dr2', 'decals-dr2-model', 'decals-dr2-resid', 'decals-dr3', 'decals-dr3-model', 'decals-dr3-resid', ]: import pylab as plt from viewer import settings from legacypipe.survey import get_rgb import fitsio from scipy.ndimage.filters import gaussian_filter tag = opt.kind bouncemap = { 'sdss': _bounce_sdssco, 'decals-dr3': _bounce_decals_dr3, 'decals-dr3-model': _bounce_decals_dr3, 'decals-dr3-resid': _bounce_decals_dr3, } bounce = bouncemap[opt.kind] rgbkwargs = dict(mnmx=(-1,100.), arcsinh=1.) bands = 'grz' if opt.kind in ['decals-dr2', 'decals-dr2-model', 'decals-dr3', 'decals-dr3-model',]: get_rgb = dr2_rgb rgbkwargs = {} elif opt.kind == 'sdssco': rgbkwargs = {} ver = tileversions.get(opt.kind, [1])[-1] print('Version', ver) basescale = 5 pat = os.path.join(settings.DATA_DIR, 'tiles', tag, '%(ver)s', '%(zoom)i', '%(x)i', '%(y)i.jpg') patdata = dict(ver=ver) tilesize = 256 tiles = 2**basescale side = tiles * tilesize basepat = 'base-%s-%i-%%s.fits' % (opt.kind, basescale) basefns = [basepat % band for band in bands] if not all([os.path.exists(fn) for fn in basefns]): bases = [np.zeros((side, side), np.float32) for band in bands] args = [] kwa = dict() if 'model' in opt.kind: kwa.update(model=True, add_gz=True) elif 'resid' in opt.kind: kwa.update(resid=True, model_gz=True) xy = [] if opt.y1 is None: opt.y1 = tiles if opt.x0 is None: opt.x0 = 0 if opt.x1 is None: opt.x1 = tiles for y in range(opt.y0, opt.y1): for x in range(opt.x0, opt.x1): args.append(((req, ver, basescale, x, y),kwa)) xy.append((x,y)) tiles = mp.map(bounce, args) for ims,(x,y) in zip(tiles, xy): if ims is None: continue for im,base in zip(ims, bases): if im is None: continue base[y*tilesize:(y+1)*tilesize, x*tilesize:(x+1)*tilesize] = im for fn,base in zip(basefns, bases): fitsio.write(fn, base, clobber=True) else: print('Reading', basefns) bases = [fitsio.read(fn) for fn in basefns] #for scale in range(basescale-1, -1, -1): for scale in range(basescale, -1, -1): print('Scale', scale) tiles = 2**scale for y in range(tiles): for x in range(tiles): ims = [base[y*tilesize:(y+1)*tilesize, x*tilesize:(x+1)*tilesize] for base in bases] if opt.kind == 'sdss': bands = 'gri' rgb = sdss_rgb(ims, bands) else: rgb = get_rgb(ims, bands, **rgbkwargs) pp = patdata.copy() pp.update(zoom=scale, x=x, y=y) fn = pat % pp trymakedirs(fn) save_jpeg(fn, rgb) print('Wrote', fn) for i,base in enumerate(bases): #base = gaussian_filter(base, 1.) base = (base[::2,::2] + base[1::2,::2] + base[1::2,1::2] + base[::2,1::2])/4. bases[i] = base