def read_sky_model(self, **kwargs): ## HACK -- create the sky model on the fly img = self.read_image() sky = np.median(img) print('Median "sky" model:', sky) sky = ConstantSky(sky) sky.version = '0' sky.plver = '0' return sky
def read_sky_model(self, **kwargs): print('Constant sky model, median of ', self.imgfn) img,hdr = self.read_image(header=True) sky = np.median(img) print('Median "sky" =', sky) sky = ConstantSky(sky) sky.version = '0' sky.plver = '0' return sky
def read_sky_model(self, imghdr=None, **kwargs): ''' The Mosaic CP does a good job of sky subtraction, so just use a constant sky level with value from the header. ''' from tractor.sky import ConstantSky sky = ConstantSky(imghdr['AVSKY']) sky.version = '' phdr = self.read_image_primary_header() sky.plver = phdr.get('PLVER', '').strip() return sky
def read_sky_model(self, imghdr=None, **kwargs): ''' Bok CP does same sky subtraction as Mosaic CP, so just use a constant sky level with value from the header. ''' from tractor.sky import ConstantSky # Frank reocmmends SKYADU phdr = self.read_image_primary_header() sky = ConstantSky(phdr['SKYADU']) sky.version = '' sky.plver = phdr.get('PLVER', '').strip() return sky
def tractor_image(self, subimage): ''' Creates the Tractor image ''' timage = [ Image(data=subimage, invvar=np.ones_like(subimage) / self.noise**2, psf=self.psf_model(), wcs=NullWCS(), photocal=NullPhotoCal(), sky=ConstantSky(self.level)) ] return timage
def get_tractor_image(self, slc=None, radecpoly=None, gaussPsf=False, pixPsf=False, hybridPsf=False, splinesky=False, nanomaggies=True, subsky=True, tiny=10, dq=True, invvar=True, pixels=True, constant_invvar=False): ''' Returns a tractor.Image ("tim") object for this image. Options describing a subimage to return: - *slc*: y,x slice objects - *radecpoly*: numpy array, shape (N,2), RA,Dec polygon describing bounding box to select. Options determining the PSF model to use: - *gaussPsf*: single circular Gaussian PSF based on header FWHM value. - *pixPsf*: pixelized PsfEx model. - *hybridPsf*: combo pixelized PsfEx + Gaussian approx. Options determining the sky model to use: - *splinesky*: median filter chunks of the image, then spline those. Options determining the units of the image: - *nanomaggies*: convert the image to be in units of NanoMaggies; *tim.zpscale* contains the scale value the image was divided by. - *subsky*: instantiate and subtract the initial sky model, leaving a constant zero sky model? ''' get_dq = dq get_invvar = invvar band = self.band imh, imw = self.get_image_shape() wcs = self.get_wcs() x0, y0 = 0, 0 x1 = x0 + imw y1 = y0 + imh # Clip to RA,Dec polygon? if slc is None and radecpoly is not None: from astrometry.util.miscutils import clip_polygon imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)] ok, tx, ty = wcs.radec2pixelxy(radecpoly[:-1, 0], radecpoly[:-1, 1]) tpoly = zip(tx, ty) clip = clip_polygon(imgpoly, tpoly) clip = np.array(clip) if len(clip) == 0: return None x0, y0 = np.floor(clip.min(axis=0)).astype(int) x1, y1 = np.ceil(clip.max(axis=0)).astype(int) slc = slice(y0, y1 + 1), slice(x0, x1 + 1) if y1 - y0 < tiny or x1 - x0 < tiny: print('Skipping tiny subimage') return None # Slice? if slc is not None: sy, sx = slc y0, y1 = sy.start, sy.stop x0, x1 = sx.start, sx.stop # Is part of this image bad? old_extent = (x0, x1, y0, y1) new_extent = self.get_good_image_slice((x0, x1, y0, y1), get_extent=True) if new_extent != old_extent: x0, x1, y0, y1 = new_extent print('Applying good subregion of CCD: slice is', x0, x1, y0, y1) if x0 >= x1 or y0 >= y1: return None slc = slice(y0, y1), slice(x0, x1) # Read image pixels if pixels: print('Reading image slice:', slc) img, imghdr = self.read_image(header=True, slice=slc) self.check_image_header(imghdr) else: img = np.zeros((imh, imw), np.float32) imghdr = self.read_image_header() if slc is not None: img = img[slc] assert (np.all(np.isfinite(img))) # Read inverse-variance (weight) map if get_invvar: invvar = self.read_invvar(slice=slc, clipThresh=0.) else: invvar = np.ones_like(img) assert (np.all(np.isfinite(invvar))) if np.all(invvar == 0.): print('Skipping zero-invvar image') return None # Negative invvars (from, eg, fpack decompression noise) cause havoc assert (np.all(invvar >= 0.)) # Read data-quality (flags) map and zero out the invvars of masked pixels if get_dq: dq = self.read_dq(slice=slc) if dq is not None: invvar[dq != 0] = 0. if np.all(invvar == 0.): print('Skipping zero-invvar image (after DQ masking)') return None # header 'FWHM' is in pixels assert (self.fwhm > 0) psf_fwhm = self.fwhm psf_sigma = psf_fwhm / 2.35 primhdr = self.read_image_primary_header() sky = self.read_sky_model(splinesky=splinesky, slc=slc, primhdr=primhdr, imghdr=imghdr) skysig1 = getattr(sky, 'sig1', None) midsky = 0. if subsky: print('Instantiating and subtracting sky model') from tractor.sky import ConstantSky skymod = np.zeros_like(img) sky.addTo(skymod) img -= skymod midsky = np.median(skymod) zsky = ConstantSky(0.) zsky.version = getattr(sky, 'version', '') zsky.plver = getattr(sky, 'plver', '') del skymod sky = zsky del zsky orig_zpscale = zpscale = NanoMaggies.zeropointToScale(self.ccdzpt) if nanomaggies: # Scale images to Nanomaggies img /= zpscale invvar = invvar * zpscale**2 if not subsky: sky.scale(1. / zpscale) zpscale = 1. # Compute 'sig1', scalar typical per-pixel noise if get_invvar: sig1 = 1. / np.sqrt(np.median(invvar[invvar > 0])) elif skysig1 is not None: sig1 = skysig1 if nanomaggies: # skysig1 is in the native units sig1 /= orig_zpscale else: # Estimate per-pixel noise via Blanton's 5-pixel MAD slice1 = (slice(0, -5, 10), slice(0, -5, 10)) slice2 = (slice(5, None, 10), slice(5, None, 10)) mad = np.median(np.abs(img[slice1] - img[slice2]).ravel()) sig1 = 1.4826 * mad / np.sqrt(2.) print('sig1 estimate:', sig1) invvar *= (1. / sig1**2) assert (np.isfinite(sig1)) if constant_invvar: print('Setting constant invvar', 1. / sig1**2) invvar[invvar > 0] = 1. / sig1**2 if subsky: # Warn if the subtracted sky doesn't seem to work well # (can happen, eg, if sky calibration product is inconsistent with # the data) imgmed = np.median(img[invvar > 0]) if np.abs(imgmed) > sig1: print('WARNING: image median', imgmed, 'is more than 1 sigma', 'away from zero!') # tractor WCS object twcs = self.get_tractor_wcs(wcs, x0, y0, primhdr=primhdr, imghdr=imghdr) if hybridPsf: pixPsf = False psf = self.read_psf_model(x0, y0, gaussPsf=gaussPsf, pixPsf=pixPsf, hybridPsf=hybridPsf, psf_sigma=psf_sigma, cx=(x0 + x1) / 2., cy=(y0 + y1) / 2.) tim = Image(img, invvar=invvar, wcs=twcs, psf=psf, photocal=LinearPhotoCal(zpscale, band=band), sky=sky, name=self.name + ' ' + band) assert (np.all(np.isfinite(tim.getInvError()))) # PSF norm psfnorm = self.psf_norm(tim) # Galaxy-detection norm tim.band = band galnorm = self.galaxy_norm(tim) # CP (DECam) images include DATE-OBS and MJD-OBS, in UTC. import astropy.time mjd_tai = astropy.time.Time(self.mjdobs, format='mjd', scale='utc').tai.mjd tim.time = TAITime(None, mjd=mjd_tai) tim.slice = slc tim.zpscale = orig_zpscale tim.midsky = midsky tim.sig1 = sig1 tim.psf_fwhm = psf_fwhm tim.psf_sigma = psf_sigma tim.propid = self.propid tim.psfnorm = psfnorm tim.galnorm = galnorm tim.sip_wcs = wcs tim.x0, tim.y0 = int(x0), int(y0) tim.imobj = self tim.primhdr = primhdr tim.hdr = imghdr tim.plver = primhdr.get('PLVER', '').strip() tim.skyver = (getattr(sky, 'version', ''), getattr(sky, 'plver', '')) tim.wcsver = (getattr(wcs, 'version', ''), getattr(wcs, 'plver', '')) tim.psfver = (getattr(psf, 'version', ''), getattr(psf, 'plver', '')) if get_dq: tim.dq = dq tim.dq_saturation_bits = self.dq_saturation_bits subh, subw = tim.shape tim.subwcs = tim.sip_wcs.get_subimage(tim.x0, tim.y0, subw, subh) return tim
inverr = np.ones_like(img) / sig1 # PSF... #psfex = PixelizedPsfEx(psffn) psfex = GaussianMixturePSF(1., 0., 0., 4., 4., 0.) #psfex = GaussianMixturePSF(1., 0., 0., 1., 1., 0.) sky = np.median(img) img -= sky tim = Image(data=img, inverr=inverr, wcs=ConstantFitsWcs(wcs), photocal=photocal, psf=psfex, sky=ConstantSky(0.)) ima = dict(interpolation='nearest', origin='lower', cmap='gray', vmin=-2. * sig1, vmax=10. * sig1) plt.clf() plt.hist((img * inverr).ravel(), 100, range=(-5, 5)) plt.xlabel('Image sigma') ps.savefig() plt.clf() plt.imshow(img, **ima) ax = plt.axis()
def stage_fit_on_coadds(survey=None, targetwcs=None, pixscale=None, bands=None, tims=None, brickname=None, version_header=None, coadd_tiers=None, apodize=True, subsky=True, ubercal_sky=False, subsky_radii=None, nsatur=None, fitoncoadds_reweight_ivar=True, plots=False, plots2=False, ps=None, coadd_bw=False, W=None, H=None, brick=None, blobs=None, lanczos=True, ccds=None, write_metrics=True, mp=None, record_event=None, **kwargs): from legacypipe.coadds import make_coadds from legacypipe.bits import DQ_BITS from legacypipe.survey import LegacySurveyWcs from legacypipe.coadds import get_coadd_headers from tractor.image import Image from tractor.basics import LinearPhotoCal from tractor.sky import ConstantSky from tractor.psf import PixelizedPSF from tractor.tractortime import TAITime import astropy.time import fitsio if plots or plots2: import pylab as plt from legacypipe.survey import get_rgb # Custom sky-subtraction for large galaxies. skydict = {} if not subsky: if ubercal_sky: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('fitoncoadds-{}'.format(brickname)) tims, skydict = ubercal_skysub(tims, targetwcs, survey, brickname, bands, mp, subsky_radii=subsky_radii, plots=True, plots2=False, ps=ps, verbose=True) else: print('Skipping sky-subtraction entirely.') # Create coadds and then build custom tims from them. for tim in tims: ie = tim.inverr if np.any(ie < 0): print('Negative inverse error in image {}'.format(tim.name)) CC = [] if coadd_tiers: # Sort by band and sort them into tiers. tiers = [[] for i in range(coadd_tiers)] for b in bands: btims = [] seeing = [] for tim in tims: if tim.band != b: continue btims.append(tim) seeing.append(tim.psf_fwhm * tim.imobj.pixscale) I = np.argsort(seeing) btims = [btims[i] for i in I] seeing = [seeing[i] for i in I] N = min(coadd_tiers, len(btims)) splits = np.round(np.arange(N + 1) * float(len(btims)) / N).astype(int) print('Splitting', len(btims), 'images into', N, 'tiers: splits:', splits) print('Seeing limits:', [seeing[min(s, len(seeing) - 1)] for s in splits]) for s0, s1, tt in zip(splits, splits[1:], tiers): tt.extend(btims[s0:s1]) for itier, tier in enumerate(tiers): print('Producing coadds for tier', (itier + 1)) C = make_coadds( tier, bands, targetwcs, detmaps=True, ngood=True, lanczos=lanczos, allmasks=True, anymasks=True, psf_images=True, nsatur=2, mp=mp, plots=plots2, ps=ps, # note plots2 here! callback=None) if plots: plt.clf() for iband, (band, psf) in enumerate(zip(bands, C.psf_imgs)): plt.subplot(1, len(bands), iband + 1) plt.imshow(psf, interpolation='nearest', origin='lower') plt.title('Coadd PSF image: band %s' % band) plt.suptitle('Tier %i' % (itier + 1)) ps.savefig() # for band,img in zip(bands, C.coimgs): # plt.clf() # plt.imshow(img, plt.clf() plt.imshow(get_rgb(C.coimgs, bands), origin='lower') plt.title('Tier %i' % (itier + 1)) ps.savefig() CC.append(C) else: C = make_coadds( tims, bands, targetwcs, detmaps=True, ngood=True, lanczos=lanczos, allmasks=True, anymasks=True, psf_images=True, mp=mp, plots=plots2, ps=ps, # note plots2 here! callback=None) CC.append(C) cotims = [] for C in CC: if plots2: for band, iv in zip(bands, C.cowimgs): pass # plt.clf() # plt.imshow(np.sqrt(iv), interpolation='nearest', origin='lower') # plt.title('Coadd Inverr: band %s' % band) # ps.savefig() for band, psf in zip(bands, C.psf_imgs): plt.clf() plt.imshow(psf, interpolation='nearest', origin='lower') plt.title('Coadd PSF image: band %s' % band) ps.savefig() for band, img, iv in zip(bands, C.coimgs, C.cowimgs): from scipy.ndimage.filters import gaussian_filter # plt.clf() # plt.hist((img * np.sqrt(iv))[iv>0], bins=50, range=(-5,8), log=True) # plt.title('Coadd pixel values (sigmas): band %s' % band) # ps.savefig() psf_sigma = np.mean([ (tim.psf_sigma * tim.imobj.pixscale / pixscale) for tim in tims if tim.band == band ]) gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) psfnorm = gnorm #np.sqrt(np.sum(psfimg**2)) detim = gaussian_filter(img, psf_sigma) / psfnorm**2 cosig1 = 1. / np.sqrt(np.median(iv[iv > 0])) detsig1 = cosig1 / psfnorm # plt.clf() # plt.subplot(2,1,1) # plt.hist(detim.ravel() / detsig1, bins=50, range=(-5,8), log=True) # plt.title('Coadd detection map values / sig1 (sigmas): band %s' % band) # plt.subplot(2,1,2) # plt.hist(detim.ravel() / detsig1, bins=50, range=(-5,8)) # ps.savefig() # # as in detection.py # detiv = np.zeros_like(detim) + (1. / detsig1**2) # detiv[iv == 0] = 0. # detiv = gaussian_filter(detiv, psf_sigma) # # plt.clf() # plt.hist((detim * np.sqrt(detiv)).ravel(), bins=50, range=(-5,8), log=True) # plt.title('Coadd detection map values / detie (sigmas): band %s' % band) # ps.savefig() for iband, (band, img, iv, allmask, anymask, psfimg) in enumerate( zip(bands, C.coimgs, C.cowimgs, C.allmasks, C.anymasks, C.psf_imgs)): mjd = np.mean( [tim.imobj.mjdobs for tim in tims if tim.band == band]) mjd_tai = astropy.time.Time(mjd, format='mjd', scale='utc').tai.mjd tai = TAITime(None, mjd=mjd_tai) twcs = LegacySurveyWcs(targetwcs, tai) #print('PSF sigmas (in pixels) for band', band, ':', # ['%.2f' % tim.psf_sigma for tim in tims if tim.band == band]) print( 'PSF sigmas in coadd pixels:', ', '.join([ '%.2f' % (tim.psf_sigma * tim.imobj.pixscale / pixscale) for tim in tims if tim.band == band ])) psf_sigma = np.mean([ (tim.psf_sigma * tim.imobj.pixscale / pixscale) for tim in tims if tim.band == band ]) print('Using average PSF sigma', psf_sigma) psf = PixelizedPSF(psfimg) gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) psfnorm = np.sqrt(np.sum(psfimg**2)) print('Gaussian PSF norm', gnorm, 'vs pixelized', psfnorm) # if plots: # from collections import Counter # plt.clf() # plt.imshow(mask, interpolation='nearest', origin='lower') # plt.colorbar() # plt.title('allmask') # ps.savefig() # print('allmask for band', band, ': values:', Counter(mask.ravel())) # Scale invvar to take into account that we have resampled (~double-counted) pixels tim_pixscale = np.mean( [tim.imobj.pixscale for tim in tims if tim.band == band]) cscale = tim_pixscale / pixscale print('average tim pixel scale / coadd scale:', cscale) iv /= cscale**2 if fitoncoadds_reweight_ivar: # We first tried setting the invvars constant per tim -- this # makes things worse, since we *remove* the lowered invvars at # the cores of galaxies. # # Here we're hacking the relative weights -- squaring the # weights but then making the median the same, ie, squaring # the dynamic range or relative weights -- ie, downweighting # the cores even more than they already are from source # Poisson terms. median_iv = np.median(iv[iv > 0]) assert (median_iv > 0) iv = iv * np.sqrt(iv) / np.sqrt(median_iv) assert (np.all(np.isfinite(iv))) assert (np.all(iv >= 0)) cotim = Image(img, invvar=iv, wcs=twcs, psf=psf, photocal=LinearPhotoCal(1., band=band), sky=ConstantSky(0.), name='coadd-' + band) cotim.band = band cotim.subwcs = targetwcs cotim.psf_sigma = psf_sigma cotim.sig1 = 1. / np.sqrt(np.median(iv[iv > 0])) # Often, SATUR masks on galaxies / stars are surrounded by BLEED pixels. Soak these into # the SATUR mask. from scipy.ndimage.morphology import binary_dilation anymask |= np.logical_and(((anymask & DQ_BITS['bleed']) > 0), binary_dilation( ((anymask & DQ_BITS['satur']) > 0), iterations=10)) * DQ_BITS['satur'] # Saturated in any image -> treat as saturated in coadd # (otherwise you get weird systematics in the weighted coadds, and weird source detection!) mask = allmask mask[(anymask & DQ_BITS['satur'] > 0)] |= DQ_BITS['satur'] if coadd_tiers: # nsatur -- reset SATUR bit mask &= ~DQ_BITS['satur'] mask |= DQ_BITS['satur'] * C.satmaps[iband] cotim.dq = mask cotim.dq_saturation_bits = DQ_BITS['satur'] cotim.psfnorm = gnorm cotim.galnorm = 1.0 # bogus! cotim.imobj = Duck() cotim.imobj.fwhm = 2.35 * psf_sigma cotim.imobj.pixscale = pixscale cotim.time = tai cotim.primhdr = fitsio.FITSHDR() get_coadd_headers(cotim.primhdr, tims, band, coadd_headers=skydict) cotims.append(cotim) if plots: plt.clf() bitmap = dict([(v, k) for k, v in DQ_BITS.items()]) k = 1 for i in range(12): bitval = 1 << i if not bitval in bitmap: continue # only 9 bits are actually used plt.subplot(3, 3, k) k += 1 plt.imshow((cotim.dq & bitval) > 0, vmin=0, vmax=1.5, cmap='hot', origin='lower') plt.title(bitmap[bitval]) plt.suptitle('Coadd mask planes %s band' % band) ps.savefig() plt.clf() h, w = cotim.shape rgb = np.zeros((h, w, 3), np.uint8) rgb[:, :, 0] = (cotim.dq & DQ_BITS['satur'] > 0) * 255 rgb[:, :, 1] = (cotim.dq & DQ_BITS['bleed'] > 0) * 255 plt.imshow(rgb, origin='lower') plt.suptitle('Coadd DQ band %s: red = SATUR, green = BLEED' % band) ps.savefig() # Save an image of the coadd PSF # copy version_header before modifying it. hdr = fitsio.FITSHDR() for r in version_header.records(): hdr.add_record(r) hdr.add_record( dict(name='IMTYPE', value='coaddpsf', comment='LegacySurveys image type')) hdr.add_record( dict(name='BAND', value=band, comment='Band of this coadd/PSF')) hdr.add_record( dict(name='PSF_SIG', value=psf_sigma, comment='Average PSF sigma (coadd pixels)')) hdr.add_record( dict(name='PIXSCAL', value=pixscale, comment='Pixel scale of this PSF (arcsec)')) hdr.add_record( dict(name='INPIXSC', value=tim_pixscale, comment='Native image pixscale scale (average, arcsec)')) hdr.add_record( dict(name='MJD', value=mjd, comment='Average MJD for coadd')) hdr.add_record( dict(name='MJD_TAI', value=mjd_tai, comment='Average MJD (in TAI) for coadd')) with survey.write_output('copsf', brick=brickname, band=band) as out: out.fits.write(psfimg, header=hdr) # EVIL return dict(tims=cotims, coadd_headers=skydict)
def ubercal_skysub(tims, targetwcs, survey, brickname, bands, mp, subsky_radii=None, plots=False, plots2=False, ps=None, verbose=False): """With the ubercal option, we (1) read the full-field mosaics ('bandtims') for a given bandpass and put them all on the same 'system' using the overlapping pixels; (2) apply the derived corrections to the in-field 'tims'; (3) build the coadds (per bandpass) from the 'tims'; and (4) subtract the median sky from the mosaic (after aggressively masking objects and reference sources). """ from tractor.sky import ConstantSky from legacypipe.reference import get_reference_sources, get_reference_map from legacypipe.coadds import make_coadds from legacypipe.survey import get_rgb, imsave_jpeg from astropy.stats import sigma_clipped_stats if plots or plots2: import os import matplotlib.pyplot as plt if plots: plt.figure(figsize=(8, 6)) mods = [] for tim in tims: imcopy = tim.getImage().copy() tim.sky.addTo(imcopy, -1) mods.append(imcopy) C = make_coadds(tims, bands, targetwcs, mods=mods, callback=None, mp=mp) imsave_jpeg(os.path.join(survey.output_dir, 'metrics', 'cus', '{}-pipelinesky.jpg'.format(ps.basefn)), get_rgb(C.comods, bands), origin='lower') skydict = {} if subsky_radii is not None: skydict.update( {'NSKYANN': (len(subsky_radii) // 2, 'number of sky annuli')}) for irad, (rin, rout) in enumerate( zip(subsky_radii[0::2], subsky_radii[1::2])): skydict.update({ 'SKYRIN{:02d}'.format(irad): (np.float32(rin), 'inner sky radius {} [arcsec]'.format(irad)) }) skydict.update({ 'SKYROT{:02d}'.format(irad): (np.float32(rout), 'outer sky radius {} [arcsec]'.format(irad)) }) allbands = np.array([tim.band for tim in tims]) # we need the full-field mosaics if doing annular sky-subtraction if subsky_radii is not None: allbandtims = [] else: allbandtims = None for band in sorted(set(allbands)): print('Working on band {}'.format(band)) I = np.where(allbands == band)[0] bandtims = [ tims[ii].imobj.get_tractor_image(gaussPsf=True, pixPsf=False, subsky=False, dq=True, apodize=False) for ii in I ] # Derive the ubercal correction and then apply it. x = coadds_ubercal(bandtims, coaddtims=[tims[ii] for ii in I], plots=plots, plots2=plots2, ps=ps, verbose=True) for ii, corr in zip(I, x): skydict.update({ 'SKCCD{:03d}'.format(ii): (tims[ii].name, 'ubersky CCD {:03d}'.format(ii)) }) skydict.update({ 'SKCOR{:03d}'.format(ii): (corr, 'ubersky corr CCD {:03d}'.format(ii)) }) # Apply the correction and return the tims for jj, (correction, ii) in enumerate(zip(x, I)): tims[ii].data += correction tims[ii].sky = ConstantSky(0.0) # Also correct the full-field mosaics bandtims[jj].data += correction bandtims[jj].sky = ConstantSky(0.0) ## Check-- #for jj, correction in enumerate(x): # fulltims[jj].data += correction #newcorrection = coadds_ubercal(fulltims) #print(newcorrection) if allbandtims is not None: allbandtims = allbandtims + bandtims # Estimate the sky background from an annulus surrounding the object # (assumed to be at the center of the mosaic, targetwcs.crval). if subsky_radii is not None: from astrometry.util.util import Tan # the inner and outer radii / annuli are nested in subsky_radii allrin = subsky_radii[0::2] allrout = subsky_radii[1::2] pixscale = targetwcs.pixel_scale() bigH = float(np.ceil(2 * np.max(allrout) / pixscale)) bigW = bigH # if doing annulur sky-subtraction we need a bigger mosaic bigtargetwcs = Tan(targetwcs.crval[0], targetwcs.crval[1], bigW / 2. + 0.5, bigH / 2. + 0.5, -pixscale / 3600.0, 0., 0., pixscale / 3600.0, float(bigW), float(bigH)) C = make_coadds(allbandtims, bands, bigtargetwcs, callback=None, sbscale=False, mp=mp) _, x0, y0 = bigtargetwcs.radec2pixelxy(bigtargetwcs.crval[0], bigtargetwcs.crval[1]) xcen, ycen = np.round(x0 - 1).astype('int'), np.round(y0 - 1).astype('int') ymask, xmask = np.ogrid[-ycen:bigH - ycen, -xcen:bigW - xcen] refs, _ = get_reference_sources(survey, bigtargetwcs, bigtargetwcs.pixel_scale(), ['r'], tycho_stars=True, gaia_stars=True, large_galaxies=True, star_clusters=True) refmask = get_reference_map(bigtargetwcs, refs) == 0 # True=skypix for coimg, coiv, band in zip(C.coimgs, C.cowimgs, bands): skypix = _build_objmask(coimg, coiv, refmask * (coiv > 0)) for irad, (rin, rout) in enumerate(zip(allrin, allrout)): inmask = (xmask**2 + ymask**2) <= (rin / pixscale)**2 outmask = (xmask**2 + ymask**2) <= (rout / pixscale)**2 skymask = (outmask * 1 - inmask * 1) == 1 # True=skypix # Find and mask objects, then get the sky. skypix_annulus = np.logical_and(skypix, skymask) #import matplotlib.pyplot as plt ; plt.imshow(skypix_annulus, origin='lower') ; plt.savefig('junk3.png') #import pdb ; pdb.set_trace() if np.sum(skypix_annulus) == 0: raise ValueError('No pixels in sky!') _skymean, _skymedian, _skysig = sigma_clipped_stats( coimg, mask=np.logical_not(skypix_annulus), sigma=3.0) skydict.update({ '{}SKYMN{:02d}'.format(band.upper(), irad): (np.float32(_skymean), 'mean {} sky in annulus {}'.format(band, irad)) }) skydict.update({ '{}SKYMD{:02d}'.format(band.upper(), irad): (np.float32(_skymedian), 'median {} sky in annulus {}'.format(band, irad)) }) skydict.update({ '{}SKYSG{:02d}'.format(band.upper(), irad): (np.float32(_skysig), 'sigma {} sky in annulus {}'.format(band, irad)) }) skydict.update({ '{}SKYNP{:02d}'.format(band.upper(), irad): (np.sum(skypix_annulus), 'npix {} sky in annulus {}'.format(band, irad)) }) # the reference annulus is the first one if irad == 0: skymean, skymedian, skysig = _skymean, _skymedian, _skysig skypix_mask = skypix_annulus I = np.where(allbands == band)[0] for ii in I: tims[ii].data -= skymedian #print('Tim', tims[ii], 'after subtracting skymedian: median', np.median(tims[ii].data)) else: # regular mosaic C = make_coadds(tims, bands, targetwcs, callback=None, sbscale=False, mp=mp) refs, _ = get_reference_sources(survey, targetwcs, targetwcs.pixel_scale(), ['r'], tycho_stars=True, gaia_stars=True, large_galaxies=True, star_clusters=True) refmask = get_reference_map(targetwcs, refs) == 0 # True=skypix for coimg, coiv, band in zip(C.coimgs, C.cowimgs, bands): skypix = refmask * (coiv > 0) skypix_mask = _build_objmask(coimg, coiv, skypix) skymean, skymedian, skysig = sigma_clipped_stats( coimg, mask=np.logical_not(skypix_mask), sigma=3.0) skydict.update({ '{}SKYMN00'.format(band.upper(), irad): (np.float32(_skymean), 'mean {} sky'.format(band)) }) skydict.update({ '{}SKYMD00'.format(band.upper(), irad): (np.float32(_skymedian), 'median {} sky'.format(band)) }) skydict.update({ '{}SKYSG00'.format(band.upper(), irad): (np.float32(_skysig), 'sigma {} sky'.format(band)) }) skydict.update({ '{}SKYNP00'.format(band.upper(), irad): (np.sum(skypix_mask), 'npix {} sky'.format(band)) }) I = np.where(allbands == band)[0] for ii in I: tims[ii].data -= skymedian # print('Tim', tims[ii], 'after subtracting skymedian: median', np.median(tims[ii].data)) #print('Band', band, 'Coadd sky:', skymedian) if plots2: plt.clf() plt.hist(coimg.ravel(), bins=50, range=(-3, 3), density=True) plt.axvline(skymedian, color='k') for ii in I: #print('Tim', tims[ii], 'median', np.median(tims[ii].data)) plt.hist((tims[ii].data - skymedian).ravel(), bins=50, range=(-3, 3), histtype='step', density=True) plt.title('Band %s: tim pix & skymedian' % band) ps.savefig() # Produce skymedian-subtracted, masked image for later RGB plot coimg -= skymedian coimg[~skypix_mask] = 0. #coimg[np.logical_not(skymask * (coiv > 0))] = 0. if plots2: plt.clf() plt.imshow(get_rgb(C.coimgs, bands), origin='lower', interpolation='nearest') ps.savefig() for band in bands: for tim in tims: if tim.band != band: continue plt.clf() C = make_coadds([tim], bands, targetwcs, callback=None, sbscale=False, mp=mp) plt.imshow(get_rgb(C.coimgs, bands).sum(axis=2), cmap='gray', interpolation='nearest', origin='lower') plt.title('Band %s: tim %s' % (band, tim.name)) ps.savefig() if plots: import pylab as plt import matplotlib.patches as patches # skysub QA C = make_coadds(tims, bands, targetwcs, callback=None, mp=mp) imsave_jpeg(os.path.join(survey.output_dir, 'metrics', 'cus', '{}-customsky.jpg'.format(ps.basefn)), get_rgb(C.coimgs, bands), origin='lower') # ccdpos QA refs, _ = get_reference_sources(survey, targetwcs, targetwcs.pixel_scale(), ['r'], tycho_stars=False, gaia_stars=False, large_galaxies=True, star_clusters=False) pixscale = targetwcs.pixel_scale() width = targetwcs.get_width() * pixscale / 3600 # [degrees] bb, bbcc = targetwcs.radec_bounds(), targetwcs.radec_center( ) # [degrees] pad = 0.5 * width # [degrees] delta = np.max((np.diff(bb[0:2]), np.diff(bb[2:4]))) / 2 + pad / 2 xlim = bbcc[0] - delta, bbcc[0] + delta ylim = bbcc[1] - delta, bbcc[1] + delta plt.clf() _, allax = plt.subplots(1, 3, figsize=(12, 5), sharey=True, sharex=True) for ax, band in zip(allax, ('g', 'r', 'z')): ax.set_xlabel('RA (deg)') ax.text(0.9, 0.05, band, ha='center', va='bottom', transform=ax.transAxes, fontsize=18) if band == 'g': ax.set_ylabel('Dec (deg)') ax.get_xaxis().get_major_formatter().set_useOffset(False) # individual CCDs these = np.where([tim.band == band for tim in tims])[0] col = plt.cm.Set1(np.linspace(0, 1, len(tims))) for ii, indx in enumerate(these): tim = tims[indx] #wcs = tim.subwcs wcs = tim.imobj.get_wcs() cc = wcs.radec_bounds() ax.add_patch( patches.Rectangle((cc[0], cc[2]), cc[1] - cc[0], cc[3] - cc[2], fill=False, lw=2, edgecolor=col[these[ii]], label='ccd{:02d}'.format(these[ii]))) ax.legend(ncol=2, frameon=False, loc='upper left', fontsize=10) # output mosaic footprint cc = targetwcs.radec_bounds() ax.add_patch( patches.Rectangle((cc[0], cc[2]), cc[1] - cc[0], cc[3] - cc[2], fill=False, lw=2, edgecolor='k')) if subsky_radii is not None: racen, deccen = targetwcs.crval for rad in subsky_radii: ax.add_patch( patches.Circle((racen, deccen), rad / 3600, fill=False, edgecolor='black', lw=2)) else: for gal in refs: ax.add_patch( patches.Circle((gal.ra, gal.dec), gal.radius, fill=False, edgecolor='black', lw=2)) ax.set_ylim(ylim) ax.set_xlim(xlim) ax.invert_xaxis() ax.set_aspect('equal') plt.subplots_adjust(bottom=0.12, wspace=0.05, left=0.12, right=0.97, top=0.95) plt.savefig( os.path.join(survey.output_dir, 'metrics', 'cus', '{}-ccdpos.jpg'.format(ps.basefn))) if plots2: plt.clf() for coimg, band in zip(C.coimgs, bands): plt.hist(coimg.ravel(), bins=50, range=(-0.5, 0.5), histtype='step', label=band) plt.legend() plt.title('After adjustment: coadds (sb scaled)') ps.savefig() return tims, skydict
def get_tractor_image(self, slc=None, radecpoly=None, gaussPsf=False, const2psf=False, pixPsf=False, splinesky=False, nanomaggies=True, subsky=True, tiny=5, dq=True, invvar=True, pixels=True): ''' Returns a tractor.Image ("tim") object for this image. Options describing a subimage to return: - *slc*: y,x slice objects - *radecpoly*: numpy array, shape (N,2), RA,Dec polygon describing bounding box to select. Options determining the PSF model to use: - *gaussPsf*: single circular Gaussian PSF based on header FWHM value. - *const2Psf*: 2-component general Gaussian fit to PsfEx model at image center. - *pixPsf*: pixelized PsfEx model at image center. Options determining the sky model to use: - *splinesky*: median filter chunks of the image, then spline those. Options determining the units of the image: - *nanomaggies*: convert the image to be in units of NanoMaggies; *tim.zpscale* contains the scale value the image was divided by. - *subsky*: instantiate and subtract the initial sky model, leaving a constant zero sky model? ''' from astrometry.util.miscutils import clip_polygon get_dq = dq get_invvar = invvar band = self.band imh,imw = self.get_image_shape() wcs = self.get_wcs() x0,y0 = 0,0 x1 = x0 + imw y1 = y0 + imh if slc is None and radecpoly is not None: imgpoly = [(1,1),(1,imh),(imw,imh),(imw,1)] ok,tx,ty = wcs.radec2pixelxy(radecpoly[:-1,0], radecpoly[:-1,1]) tpoly = zip(tx,ty) clip = clip_polygon(imgpoly, tpoly) clip = np.array(clip) if len(clip) == 0: return None x0,y0 = np.floor(clip.min(axis=0)).astype(int) x1,y1 = np.ceil (clip.max(axis=0)).astype(int) slc = slice(y0,y1+1), slice(x0,x1+1) if y1 - y0 < tiny or x1 - x0 < tiny: print('Skipping tiny subimage') return None if slc is not None: sy,sx = slc y0,y1 = sy.start, sy.stop x0,x1 = sx.start, sx.stop old_extent = (x0,x1,y0,y1) new_extent = self.get_good_image_slice((x0,x1,y0,y1), get_extent=True) if new_extent != old_extent: x0,x1,y0,y1 = new_extent print('Applying good subregion of CCD: slice is', x0,x1,y0,y1) if x0 >= x1 or y0 >= y1: return None slc = slice(y0,y1), slice(x0,x1) if pixels: print('Reading image slice:', slc) img,imghdr = self.read_image(header=True, slice=slc) #print('SATURATE is', imghdr.get('SATURATE', None)) #print('Max value in image is', img.max()) # check consistency... something of a DR1 hangover e = imghdr['EXTNAME'] assert(e.strip() == self.ccdname.strip()) else: img = np.zeros((imh, imw)) imghdr = dict() if slc is not None: img = img[slc] if get_invvar: invvar = self.read_invvar(slice=slc, clipThresh=0.) else: invvar = np.ones_like(img) if get_dq: dq = self.read_dq(slice=slc) invvar[dq != 0] = 0. if np.all(invvar == 0.): print('Skipping zero-invvar image') return None assert(np.all(np.isfinite(img))) assert(np.all(np.isfinite(invvar))) assert(not(np.all(invvar == 0.))) # header 'FWHM' is in pixels # imghdr['FWHM'] psf_fwhm = self.fwhm psf_sigma = psf_fwhm / 2.35 primhdr = self.read_image_primary_header() sky = self.read_sky_model(splinesky=splinesky, slc=slc) midsky = 0. if subsky: print('Instantiating and subtracting sky model...') from tractor.sky import ConstantSky skymod = np.zeros_like(img) sky.addTo(skymod) img -= skymod midsky = np.median(skymod) zsky = ConstantSky(0.) zsky.version = sky.version zsky.plver = sky.plver del skymod del sky sky = zsky del zsky magzp = self.decals.get_zeropoint_for(self) orig_zpscale = zpscale = NanoMaggies.zeropointToScale(magzp) if nanomaggies: # Scale images to Nanomaggies img /= zpscale invvar *= zpscale**2 if not subsky: sky.scale(1./zpscale) zpscale = 1. assert(np.sum(invvar > 0) > 0) if get_invvar: sig1 = 1./np.sqrt(np.median(invvar[invvar > 0])) else: # Estimate from the image? # # Estimate per-pixel noise via Blanton's 5-pixel MAD slice1 = (slice(0,-5,10),slice(0,-5,10)) slice2 = (slice(5,None,10),slice(5,None,10)) mad = np.median(np.abs(img[slice1] - img[slice2]).ravel()) sig1 = 1.4826 * mad / np.sqrt(2.) print('sig1 estimate:', sig1) invvar *= (1. / sig1**2) assert(np.all(np.isfinite(img))) assert(np.all(np.isfinite(invvar))) assert(np.isfinite(sig1)) if subsky: ## imgmed = np.median(img[invvar>0]) if np.abs(imgmed) > sig1: print('WARNING: image median', imgmed, 'is more than 1 sigma away from zero!') # Boom! #assert(False) twcs = ConstantFitsWcs(wcs) if x0 or y0: twcs.setX0Y0(x0,y0) psf = self.read_psf_model(x0, y0, gaussPsf=gaussPsf, pixPsf=pixPsf, const2psf=const2psf, psf_sigma=psf_sigma, cx=(x0+x1)/2., cy=(y0+y1)/2.) tim = Image(img, invvar=invvar, wcs=twcs, psf=psf, photocal=LinearPhotoCal(zpscale, band=band), sky=sky, name=self.name + ' ' + band) assert(np.all(np.isfinite(tim.getInvError()))) # PSF norm psfnorm = self.psf_norm(tim) print('PSF norm', psfnorm, 'vs Gaussian', 1./(2. * np.sqrt(np.pi) * psf_sigma)) # Galaxy-detection norm tim.band = band galnorm = self.galaxy_norm(tim) print('Galaxy norm:', galnorm) # CP (DECam) images include DATE-OBS and MJD-OBS, in UTC. import astropy.time #mjd_utc = mjd=primhdr.get('MJD-OBS', 0) mjd_tai = astropy.time.Time(primhdr['DATE-OBS']).tai.mjd tim.slice = slc tim.time = TAITime(None, mjd=mjd_tai) tim.zr = [-3. * sig1, 10. * sig1] tim.zpscale = orig_zpscale tim.midsky = midsky tim.sig1 = sig1 tim.psf_fwhm = psf_fwhm tim.psf_sigma = psf_sigma tim.propid = self.propid tim.psfnorm = psfnorm tim.galnorm = galnorm tim.sip_wcs = wcs tim.x0,tim.y0 = int(x0),int(y0) tim.imobj = self tim.primhdr = primhdr tim.hdr = imghdr tim.plver = primhdr['PLVER'].strip() tim.skyver = (sky.version, sky.plver) tim.wcsver = (wcs.version, wcs.plver) tim.psfver = (psf.version, psf.plver) if get_dq: tim.dq = dq tim.dq_bits = CP_DQ_BITS tim.saturation = imghdr.get('SATURATE', None) tim.satval = tim.saturation or 0. if subsky: tim.satval -= midsky if nanomaggies: tim.satval /= orig_zpscale subh,subw = tim.shape tim.subwcs = tim.sip_wcs.get_subimage(tim.x0, tim.y0, subw, subh) mn,mx = tim.zr tim.ima = dict(interpolation='nearest', origin='lower', cmap='gray', vmin=mn, vmax=mx) return tim
def get_tractor_image(self, slc=None, radecpoly=None, gaussPsf=False, const2psf=False, pixPsf=False, splinesky=False, nanomaggies=True, subsky=True, tiny=5, dq=True, invvar=True, pixels=True): ''' Returns a tractor.Image ("tim") object for this image. Options describing a subimage to return: - *slc*: y,x slice objects - *radecpoly*: numpy array, shape (N,2), RA,Dec polygon describing bounding box to select. Options determining the PSF model to use: - *gaussPsf*: single circular Gaussian PSF based on header FWHM value. - *const2Psf*: 2-component general Gaussian fit to PsfEx model at image center. - *pixPsf*: pixelized PsfEx model at image center. Options determining the sky model to use: - *splinesky*: median filter chunks of the image, then spline those. Options determining the units of the image: - *nanomaggies*: convert the image to be in units of NanoMaggies; *tim.zpscale* contains the scale value the image was divided by. - *subsky*: instantiate and subtract the initial sky model, leaving a constant zero sky model? ''' from astrometry.util.miscutils import clip_polygon get_dq = dq get_invvar = invvar band = self.band imh,imw = self.get_image_shape() wcs = self.get_wcs() x0,y0 = 0,0 x1 = x0 + imw y1 = y0 + imh #if don't comment out tim = NoneType b/c clips all pixels out #if slc is None and radecpoly is not None: # imgpoly = [(1,1),(1,imh),(imw,imh),(imw,1)] # ok,tx,ty = wcs.radec2pixelxy(radecpoly[:-1,0], radecpoly[:-1,1]) # tpoly = zip(tx,ty) # clip = clip_polygon(imgpoly, tpoly) # clip = np.array(clip) # if len(clip) == 0: # return None # x0,y0 = np.floor(clip.min(axis=0)).astype(int) # x1,y1 = np.ceil (clip.max(axis=0)).astype(int) # slc = slice(y0,y1+1), slice(x0,x1+1) # if y1 - y0 < tiny or x1 - x0 < tiny: # print('Skipping tiny subimage') # return None #if slc is not None: # sy,sx = slc # y0,y1 = sy.start, sy.stop # x0,x1 = sx.start, sx.stop #old_extent = (x0,x1,y0,y1) #new_extent = self.get_good_image_slice((x0,x1,y0,y1), get_extent=True) #if new_extent != old_extent: # x0,x1,y0,y1 = new_extent # print('Applying good subregion of CCD: slice is', x0,x1,y0,y1) # if x0 >= x1 or y0 >= y1: # return None # slc = slice(y0,y1), slice(x0,x1) if pixels: print('Reading image slice:', slc) img,imghdr = self.read_image(header=True, slice=slc) #print('SATURATE is', imghdr.get('SATURATE', None)) #print('Max value in image is', img.max()) # check consistency... something of a DR1 hangover #e = imghdr['EXTNAME'] #assert(e.strip() == self.ccdname.strip()) else: img = np.zeros((imh, imw)) imghdr = dict() if slc is not None: img = img[slc] if get_invvar: invvar = self.read_invvar(slice=slc, clipThresh=0.) else: invvar = np.ones_like(img) if get_dq: dq = self.read_dq(slice=slc) invvar[dq != 0] = 0. if np.all(invvar == 0.): print('Skipping zero-invvar image') return None assert(np.all(np.isfinite(img))) assert(np.all(np.isfinite(invvar))) assert(not(np.all(invvar == 0.))) # header 'FWHM' is in pixels # imghdr['FWHM'] psf_fwhm = self.fwhm psf_sigma = psf_fwhm / 2.35 primhdr = self.read_image_primary_header() sky = self.read_sky_model(splinesky=splinesky, slc=slc) midsky = 0. if subsky: print('Instantiating and subtracting sky model...') from tractor.sky import ConstantSky skymod = np.zeros_like(img) sky.addTo(skymod) img -= skymod midsky = np.median(skymod) zsky = ConstantSky(0.) zsky.version = sky.version zsky.plver = sky.plver del skymod del sky sky = zsky del zsky magzp = self.survey.get_zeropoint_for(self) orig_zpscale = zpscale = NanoMaggies.zeropointToScale(magzp) if nanomaggies: # Scale images to Nanomaggies img /= zpscale invvar *= zpscale**2 if not subsky: sky.scale(1./zpscale) zpscale = 1. assert(np.sum(invvar > 0) > 0) if get_invvar: sig1 = 1./np.sqrt(np.median(invvar[invvar > 0])) else: # Estimate from the image? # # Estimate per-pixel noise via Blanton's 5-pixel MAD slice1 = (slice(0,-5,10),slice(0,-5,10)) slice2 = (slice(5,None,10),slice(5,None,10)) mad = np.median(np.abs(img[slice1] - img[slice2]).ravel()) sig1 = 1.4826 * mad / np.sqrt(2.) print('sig1 estimate:', sig1) invvar *= (1. / sig1**2) assert(np.all(np.isfinite(img))) assert(np.all(np.isfinite(invvar))) assert(np.isfinite(sig1)) if subsky: ## imgmed = np.median(img[invvar>0]) if np.abs(imgmed) > sig1: print('WARNING: image median', imgmed, 'is more than 1 sigma away from zero!') # Boom! assert(False) twcs = ConstantFitsWcs(wcs) if x0 or y0: twcs.setX0Y0(x0,y0) #print('gaussPsf:', gaussPsf, 'pixPsf:', pixPsf, 'const2psf:', const2psf) psf = self.read_psf_model(x0, y0, gaussPsf=gaussPsf, pixPsf=pixPsf, psf_sigma=psf_sigma, cx=(x0+x1)/2., cy=(y0+y1)/2.) tim = Image(img, invvar=invvar, wcs=twcs, psf=psf, photocal=LinearPhotoCal(zpscale, band=band), sky=sky, name=self.name + ' ' + band) assert(np.all(np.isfinite(tim.getInvError()))) # PSF norm psfnorm = self.psf_norm(tim) print('PSF norm', psfnorm, 'vs Gaussian', 1./(2. * np.sqrt(np.pi) * psf_sigma)) # Galaxy-detection norm tim.band = band galnorm = self.galaxy_norm(tim) print('Galaxy norm:', galnorm) # CP (DECam) images include DATE-OBS and MJD-OBS, in UTC. import astropy.time #mjd_utc = mjd=primhdr.get('MJD-OBS', 0) mjd_tai = astropy.time.Time(primhdr['DATE-OBS']).tai.mjd tim.slice = slc tim.time = TAITime(None, mjd=mjd_tai) tim.zpscale = orig_zpscale tim.midsky = midsky tim.sig1 = sig1 tim.psf_fwhm = psf_fwhm tim.psf_sigma = psf_sigma tim.propid = self.propid tim.psfnorm = psfnorm tim.galnorm = galnorm tim.sip_wcs = wcs tim.x0,tim.y0 = int(x0),int(y0) tim.imobj = self tim.primhdr = primhdr tim.hdr = imghdr tim.plver = str(primhdr['PTFVERSN']).strip() tim.skyver = (sky.version, sky.plver) tim.wcsver = ('-1','-1') #wcs.version, wcs.plver) tim.psfver = (psf.version, psf.plver) if get_dq: tim.dq = dq tim.dq_saturation_bits = 0 tim.saturation = imghdr.get('SATURATE', None) tim.satval = tim.saturation or 0. if subsky: tim.satval -= midsky if nanomaggies: tim.satval /= orig_zpscale subh,subw = tim.shape tim.subwcs = tim.sip_wcs.get_subimage(tim.x0, tim.y0, subw, subh) return tim