def _check_sdss_files(sdss, run, camcol, field, bandname, filetypes, retrieve=True, tryopen=False): from astrometry.sdss import band_index bandnum = band_index(bandname) for filetype in filetypes: fn = sdss.getPath(filetype, run, camcol, field, bandname) print('Looking for file', fn) exists = os.path.exists(fn) retrieveKwargs = {} if exists and tryopen: import pyfits # This doesn't catch *all* types of errors you can imagine... try: T = pyfits.open(fn) except: print('Failed to open file', fn, 'as FITS file: maybe corrupt.') exists = False retrieveKwargs.update(skipExisting=False) if (not exists) and retrieve: print('Retrieving', fn) res = sdss.retrieve(filetype, run, camcol, field, bandnum, **retrieveKwargs) if res is False: raise RuntimeError('No such file on SDSS DAS: %s, ' % filetype + 'rcfb %i/%i/%i/%s' % (run, camcol, field, bandname)) elif not exists: raise os.OSError('no such file: "%s"' % fn)
def __init__(self, tsfield, bandname): from astrometry.sdss import band_index self.bandname = bandname self.band = band_index(bandname) #self.tsfield = tsfield band = self.band self.exptime = tsfield.exptime self.aa = tsfield.aa[band] self.kk = tsfield.kk[band] self.airmass = tsfield.airmass[band] super(SdssMagsPhotoCal, self).__init__()
def __init__(self, tsfield, bandname): from astrometry.sdss import band_index self.bandname = bandname self.band = band_index(bandname) #self.tsfield = tsfield band = self.band self.exptime = tsfield.exptime self.aa = tsfield.aa[band] self.kk = tsfield.kk[band] self.airmass = tsfield.airmass[band] super(SdssMagsPhotoCal,self).__init__()
def sdss_coadd(targetwcs, bands): from astrometry.sdss import DR9, band_index from astrometry.sdss import AsTransWrapper #sdss = DR9(basedir=photoobjdir) #sdss.useLocalTree() sdss = DR9(basedir='tmp') sdss.saveUnzippedFiles('tmp') #wfn = sdss.filenames.get('window_flist', None) wfn = os.path.join(os.environ.get('PHOTO_RESOLVE', ''), 'window_flist.fits') from astrometry.sdss.fields import radec_to_sdss_rcf ra, dec = targetwcs.radec_center() rad = targetwcs.radius() rad = rad + np.hypot(10., 14.) / 2. / 60. print('Searching for run,camcol,fields with radius', rad, 'deg') RCF = radec_to_sdss_rcf(ra, dec, radius=rad * 60., tablefn=wfn) print('Found %i fields possibly in range' % len(RCF)) H, W = targetwcs.shape sdsscoimgs = [np.zeros((H, W), np.float32) for band in bands] sdsscons = [np.zeros((H, W), np.float32) for band in bands] for run, camcol, field, r, d in RCF: for iband, band in enumerate(bands): bandnum = band_index(band) sdss.retrieve('frame', run, camcol, field, band) frame = sdss.readFrame(run, camcol, field, bandnum) print('Got frame', frame) h, w = frame.getImageShape() simg = frame.getImage() wcs = AsTransWrapper(frame.astrans, w, h, 0.5, 0.5) try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, wcs) except OverlapError: continue sdsscoimgs[iband][Yo, Xo] += simg[Yi, Xi] sdsscons[iband][Yo, Xo] += 1 for co, n in zip(sdsscoimgs, sdsscons): co /= np.maximum(1e-6, n) return sdsscoimgs, sdsscons
def sdss_coadd(targetwcs, bands): from astrometry.sdss import DR9, band_index from astrometry.sdss import AsTransWrapper #sdss = DR9(basedir=photoobjdir) #sdss.useLocalTree() sdss = DR9(basedir='tmp') sdss.saveUnzippedFiles('tmp') #wfn = sdss.filenames.get('window_flist', None) wfn = os.path.join(os.environ.get('PHOTO_RESOLVE',''), 'window_flist.fits') from astrometry.sdss.fields import radec_to_sdss_rcf ra,dec = targetwcs.radec_center() rad = targetwcs.radius() rad = rad + np.hypot(10.,14.)/2./60. print 'Searching for run,camcol,fields with radius', rad, 'deg' RCF = radec_to_sdss_rcf(ra, dec, radius=rad*60., tablefn=wfn) print 'Found %i fields possibly in range' % len(RCF) H,W = targetwcs.shape sdsscoimgs = [np.zeros((H,W),np.float32) for band in bands] sdsscons = [np.zeros((H,W),np.float32) for band in bands] for run,camcol,field,r,d in RCF: for iband,band in enumerate(bands): bandnum = band_index(band) sdss.retrieve('frame', run, camcol, field, band) frame = sdss.readFrame(run, camcol, field, bandnum) print 'Got frame', frame h,w = frame.getImageShape() simg = frame.getImage() wcs = AsTransWrapper(frame.astrans, w, h, 0.5, 0.5) try: Yo,Xo,Yi,Xi,nil = resample_with_wcs(targetwcs, wcs, [], 3) except OverlapError: continue sdsscoimgs[iband][Yo,Xo] += simg[Yi,Xi] sdsscons [iband][Yo,Xo] += 1 for co,n in zip(sdsscoimgs, sdsscons): co /= np.maximum(1e-6, n) return sdsscoimgs, sdsscons
def _check_sdss_files(sdss, run, camcol, field, bandname, filetypes, retrieve=True, tryopen=False): from astrometry.sdss import band_index bandnum = band_index(bandname) for filetype in filetypes: fn = sdss.getPath(filetype, run, camcol, field, bandname) print('Looking for file', fn) exists = os.path.exists(fn) retrieveKwargs = {} if exists and tryopen: # This doesn't catch *all* types of errors you can imagine... cmd = 'fitsverify -q %s' % fn try: print('Running:', cmd) rtn = os.system(cmd) print('rtn:', rtn) if rtn != 0: exists = False except: print('Failed to fitsverify', fn, ': maybe corrupt.') exists = False if not exists: retrieveKwargs.update(skipExisting=False) if (not exists) and retrieve: print('Retrieving', fn) res = sdss.retrieve(filetype, run, camcol, field, bandnum, **retrieveKwargs) if res is False: raise RuntimeError('No such file on SDSS DAS: %s, ' % filetype + 'rcfb %i/%i/%i/%s' % (run, camcol, field, bandname)) elif not exists: raise OSError('no such file: "%s", or failed to verify' % fn)
def get_tractor_image_dr7(run, camcol, field, bandname, sdssobj=None, release='DR7', retrieve=True, curl=False, psf='kl-gm', useMags=False, roi=None, roiradecsize=None, roiradecbox=None, nanomaggies=False, savepsfimg=None, zrange=[-3, 10]): ''' Creates a tractor.Image given an SDSS field identifier. If not None, roi = (x0, x1, y0, y1) defines a region-of-interest in the image, in zero-indexed pixel coordinates. x1,y1 are NON-inclusive; roi=(0,100,0,100) will yield a 100 x 100 image. psf can be: "dg" for double-Gaussian "kl-gm" for SDSS KL-decomposition approximated as a Gaussian mixture "roiradecsize" = (ra, dec, half-size in pixels) indicates that you want to grab a ROI around the given RA,Dec. Returns: (tractor.Image, dict) dict contains useful details like: 'sky' 'skysig' ''' from astrometry.sdss import DR7, band_index if sdssobj is None: # Ugly if release != 'DR7': raise RuntimeError('We only support DR7 currently') sdss = DR7(curl=curl) else: sdss = sdssobj valid_psf = ['dg', 'kl-gm'] if psf not in valid_psf: raise RuntimeError('PSF must be in ' + str(valid_psf)) # FIXME rerun = 0 bandnum = band_index(bandname) _check_sdss_files(sdss, run, camcol, field, bandname, ['fpC', 'tsField', 'psField', 'fpM'], retrieve=retrieve) fpC = sdss.readFpC(run, camcol, field, bandname) hdr = fpC.getHeader() fpC = fpC.getImage() fpC = fpC.astype(np.float32) - sdss.softbias image = fpC (H, W) = image.shape info = dict() tai = hdr.get('TAI') stripe = hdr.get('STRIPE') strip = hdr.get('STRIP') obj = hdr.get('OBJECT') info.update(tai=tai, stripe=stripe, strip=strip, object=obj, hdr=hdr) tsf = sdss.readTsField(run, camcol, field, rerun) astrans = tsf.getAsTrans(bandnum) wcs = SdssWcs(astrans) #print('Created SDSS Wcs:', wcs) X = interpret_roi(wcs, (H, W), roi=roi, roiradecsize=roiradecsize, roiradecbox=roiradecbox) if X is None: return None, None roi, hasroi = X info.update(roi=roi) x0, x1, y0, y1 = roi # Mysterious half-pixel shift. asTrans pixel coordinates? wcs.setX0Y0(x0 + 0.5, y0 + 0.5) if nanomaggies: zp = tsf.get_zeropoint(bandnum) photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp), band=bandname) elif useMags: photocal = SdssMagsPhotoCal(tsf, bandname) else: photocal = SdssFluxPhotoCal() psfield = sdss.readPsField(run, camcol, field) sky = psfield.getSky(bandnum) skysig = sqrt(sky) skyobj = ConstantSky(sky) zr = sky + np.array(zrange) * skysig info.update(sky=sky, skysig=skysig, zr=zr) fpM = sdss.readFpM(run, camcol, field, bandname) gain = psfield.getGain(bandnum) darkvar = psfield.getDarkVariance(bandnum) skyerr = psfield.getSkyErr(bandnum) invvar = sdss.getInvvar(fpC, fpM, gain, darkvar, sky, skyerr) dgpsf = psfield.getDoubleGaussian(bandnum, normalize=True) info.update(dgpsf=dgpsf) if roi is not None: roislice = (slice(y0, y1), slice(x0, x1)) image = image[roislice].copy() invvar = invvar[roislice].copy() if psf == 'kl-gm': from emfit import em_fit_2d from fitpsf import em_init_params # Create Gaussian mixture model PSF approximation. H, W = image.shape klpsf = psfield.getPsfAtPoints(bandnum, x0 + W / 2, y0 + H / 2) S = klpsf.shape[0] # number of Gaussian components K = 3 w, mu, sig = em_init_params(K, None, None, None) II = klpsf.copy() II /= II.sum() # HIDEOUS HACK II = np.maximum(II, 0) #print('Multi-Gaussian PSF fit...') xm, ym = -(S / 2), -(S / 2) if savepsfimg is not None: plt.clf() plt.imshow(II, interpolation='nearest', origin='lower') plt.title('PSF image to fit with EM') plt.savefig(savepsfimg) res = em_fit_2d(II, xm, ym, w, mu, sig) print('em_fit_2d result:', res) if res == 0: # print('w,mu,sig', w,mu,sig) mypsf = GaussianMixturePSF(w, mu, sig) mypsf.computeRadius() else: # Failed! Return 'dg' model instead? print('PSF model fit', psf, 'failed! Returning DG model instead') psf = 'dg' if psf == 'dg': print('Creating double-Gaussian PSF approximation') (a, s1, b, s2) = dgpsf mypsf = NCircularGaussianPSF([s1, s2], [a, b]) timg = Image(data=image, invvar=invvar, psf=mypsf, wcs=wcs, sky=skyobj, photocal=photocal, name=('SDSS (r/c/f/b=%i/%i/%i/%s)' % (run, camcol, field, bandname))) timg.zr = zr return timg, info
def _get_tractor_image_dr8(run, camcol, field, bandname, sdss=None, roi=None, psf='kl-gm', roiradecsize=None, roiradecbox=None, savepsfimg=None, curl=False, nanomaggies=False, zrange=[-3, 10], invvarIgnoresSourceFlux=False, invvarAtCenter=False, invvarAtCenterImage=False, retrieve=True, imargs={}): from astrometry.sdss import band_index origpsf = psf if psf.startswith('bright-'): psf = psf[7:] brightpsf = True print('Setting bright PSF handling') else: brightpsf = False valid_psf = ['dg', 'kl-gm', 'kl-pix'] if psf not in valid_psf: raise RuntimeError('PSF must be in ' + str(valid_psf)) if sdss is None: from astrometry.sdss import DR8 sdss = DR8(curl=curl) bandnum = band_index(bandname) if retrieve: for ft in ['psField', 'fpM']: fn = sdss.retrieve(ft, run, camcol, field, bandname) fn = sdss.retrieve('frame', run, camcol, field, bandname) else: fn = sdss.getPath('frame', run, camcol, field, bandname) # http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/frames/RERUN/RUN/CAMCOL/frame.html frame = sdss.readFrame(run, camcol, field, bandname, filename=fn) H, W = frame.getImageShape() info = dict() hdr = frame.getHeader() tai = hdr.get('TAI') stripe = hdr.get('STRIPE') strip = hdr.get('STRIP') obj = hdr.get('OBJECT') info.update(tai=tai, stripe=stripe, strip=strip, object=obj, hdr=hdr) astrans = frame.getAsTrans() wcs = SdssWcs(astrans) #print('Created SDSS Wcs:', wcs) #print('(x,y) = 1,1 -> RA,Dec', wcs.pixelToPosition(1,1)) X = interpret_roi(wcs, (H, W), roi=roi, roiradecsize=roiradecsize, roiradecbox=roiradecbox) if X is None: return None, None roi, hasroi = X info.update(roi=roi) x0, x1, y0, y1 = roi # Half-pixel shift for asTrans pixel coordinates. wcs.setX0Y0(x0 + 0.5, y0 + 0.5) if nanomaggies: photocal = LinearPhotoCal(1., band=bandname) else: photocal = MagsPhotoCal(bandname, 22.5) sky = 0. skyobj = ConstantSky(sky) calibvec = frame.getCalibVec() invvarAtCenter = invvarAtCenter or invvarAtCenterImage psfield = sdss.readPsField(run, camcol, field) iva = dict(ignoreSourceFlux=invvarIgnoresSourceFlux) if invvarAtCenter: if hasroi: iva.update(constantSkyAt=(int((x0 + x1) / 2.), int((y0 + y1) / 2.))) else: iva.update(constantSkyAt=(int(W / 2.), int(H / 2.))) invvar = frame.getInvvar(psfield, bandnum, **iva) invvar = invvar.astype(np.float32) if not invvarAtCenter: assert (invvar.shape == (H, W)) # Could get this from photoField instead # http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/RERUN/RUN/photoField.html gain = psfield.getGain(bandnum) darkvar = psfield.getDarkVariance(bandnum) meansky = np.mean(frame.sky) meancalib = np.mean(calibvec) skysig = sqrt((meansky / gain) + darkvar) * meancalib info.update(sky=sky, skysig=skysig) zr = np.array(zrange) * skysig + sky info.update(zr=zr) # http://data.sdss3.org/datamodel/files/PHOTO_REDUX/RERUN/RUN/objcs/CAMCOL/fpM.html fpM = sdss.readFpM(run, camcol, field, bandname) if not hasroi: image = frame.getImage() else: roislice = (slice(y0, y1), slice(x0, x1)) image = frame.getImageSlice(roislice).astype(np.float32) if invvarAtCenterImage: invvar = invvar + np.zeros(image.shape, np.float32) elif invvarAtCenter: pass else: invvar = invvar[roislice].copy() H, W = image.shape if (not invvarAtCenter) or invvarAtCenterImage: for plane in ['INTERP', 'SATUR', 'CR', 'GHOST']: fpM.setMaskedPixels(plane, invvar, 0, roi=roi) dgpsf = psfield.getDoubleGaussian(bandnum, normalize=True) info.update(dgpsf=dgpsf) if psf == 'kl-pix': # Pixelized KL-PSF klpsf = psfield.getPsfAtPoints(bandnum, x0 + W / 2, y0 + H / 2) # Trim symmetric zeros sh, sw = klpsf.shape while True: if (np.all(klpsf[0, :] == 0.) and np.all(klpsf[:, 0] == 0.) and np.all(klpsf[-1, :] == 0.) and np.all(klpsf[:, -1] == 0.)): klpsf = klpsf[1:-1, 1:-1] else: break mypsf = PixelizedPSF(klpsf) elif psf == 'kl-gm': from tractor.emfit import em_fit_2d from tractor.fitpsf import em_init_params # Create Gaussian mixture model PSF approximation. klpsf = psfield.getPsfAtPoints(bandnum, x0 + W / 2, y0 + H / 2) S = klpsf.shape[0] # number of Gaussian components K = 3 w, mu, sig = em_init_params(K, None, None, None) II = klpsf.copy() II /= II.sum() # HIDEOUS HACK II = np.maximum(II, 0) #print('Multi-Gaussian PSF fit...') xm, ym = -(S // 2), -(S // 2) if savepsfimg is not None: plt.clf() plt.imshow(II, interpolation='nearest', origin='lower') plt.title('PSF image to fit with EM') plt.savefig(savepsfimg) res = em_fit_2d(II, xm, ym, w, mu, sig) #print('em_fit_2d result:', res) if res == 0: # print('w,mu,sig', w,mu,sig) mypsf = GaussianMixturePSF(w, mu, sig) mypsf.computeRadius() else: # Failed! Return 'dg' model instead? print('PSF model fit', psf, 'failed! Returning DG model instead') psf = 'dg' if psf == 'dg': print('Creating double-Gaussian PSF approximation') (a, s1, b, s2) = dgpsf mypsf = NCircularGaussianPSF([s1, s2], [a, b]) if brightpsf: print('Wrapping PSF in SdssBrightPSF') (a1, s1, a2, s2, a3, sigmap, beta) = psfield.getPowerLaw(bandnum) mypsf = SdssBrightPSF(mypsf, a1, s1, a2, s2, a3, sigmap, beta) print('PSF:', mypsf) timg = Image(data=image, invvar=invvar, psf=mypsf, wcs=wcs, sky=skyobj, photocal=photocal, name=('SDSS (r/c/f/b=%i/%i/%i/%s)' % (run, camcol, field, bandname)), time=TAITime(tai), **imargs) timg.zr = zr return timg, info
def _get_sources(run, camcol, field, bandname='r', sdss=None, release='DR7', objs=None, retrieve=True, checkFiles=True, curl=False, roi=None, radecroi=None, radecrad=None, bands=None, badmag=25, nanomaggies=False, getobjs=False, getsourceobjs=False, getobjinds=False, extrabands=None, fixedComposites=False, forcePointSources=False, useObjcType=False, objCuts=True, classmap={}, ellipse=GalaxyShape, cutToPrimary=False): ''' If set, radecrad = (ra,dec,rad) returns sources within "rad" degrees of the given RA,Dec (in degrees) center. WARNING, this method alters the "objs" argument, if given. Consider calling objs.copy() before calling. -"bandname" is the SDSS band used to cut on position, select star/gal/exp/dev, and set galaxy shapes. -"bands" are the bands to include in the returned Source objects; they will be initialized from the SDSS bands. -"extrabands" are also included in the returned Source objects; they will be initialized to the SDSS flux for either the first of "bands", if given, or "bandname". ''' from astrometry.sdss import (DR7, DR8, DR9, band_names, band_index, photo_flags1_map) # brightPointSourceThreshold=0.): if sdss is None: dr = dict(DR7=DR7, DR8=DR8, DR9=DR9)[release] sdss = dr(curl=curl) drnum = sdss.getDRNumber() isdr7 = (drnum == 7) if bands is None: bands = band_names() bandnum = band_index(bandname) bandnums = np.array([band_index(b) for b in bands]) bandnames = bands if extrabands is None: extrabands = [] if objs is None: from astrometry.util.fits import fits_table if isdr7: # FIXME rerun = 0 if checkFiles: _check_sdss_files(sdss, run, camcol, field, bandnum, ['tsObj', 'tsField'], retrieve=retrieve, tryopen=True) tsf = sdss.readTsField(run, camcol, field, rerun) objfn = sdss.getPath('tsObj', run, camcol, field, bandname, rerun=rerun) else: if checkFiles: _check_sdss_files(sdss, run, camcol, field, bandnum, ['photoObj'], retrieve=retrieve, tryopen=True) objfn = sdss.getPath('photoObj', run, camcol, field) objs = fits_table(objfn) if objs is None: print('No sources in SDSS file', objfn) return [] objs.index = np.arange(len(objs)) if getobjs: allobjs = objs.copy() if roi is not None: x0, x1, y0, y1 = roi # FIXME -- we keep only the sources whose centers are within # the ROI box. Should instead do some ellipse-overlaps # geometry. x = objs.colc[:, bandnum] y = objs.rowc[:, bandnum] objs.cut((x >= x0) * (x < x1) * (y >= y0) * (y < y1)) if radecroi is not None: r0, r1, d0, d1 = radecroi objs.cut((objs.ra >= r0) * (objs.ra <= r1) * (objs.dec >= d0) * (objs.dec <= d1)) if radecrad is not None: from astrometry.libkd.spherematch import match_radec (ra, dec, rad) = radecrad I, J, d = match_radec(ra, dec, objs.ra, objs.dec, rad) objs.cut(J) del I del d if objCuts: # Only deblended children; # No BRIGHT sources bright = photo_flags1_map.get('BRIGHT') objs.cut((objs.nchild == 0) * ((objs.objc_flags & bright) == 0)) if cutToPrimary: objs.cut((objs.resolve_status & 256) > 0) if len(objs) == 0: sources = [] if not (getobjs or getobjinds or getsourceobjs): return sources rtn = [sources] if getobjs: rtn.append(None) if getobjinds: rtn.append(None) if getsourceobjs: rtn.append(None) return rtn if isdr7: objs.rename('phi_dev', 'phi_dev_deg') objs.rename('phi_exp', 'phi_exp_deg') objs.rename('r_dev', 'theta_dev') objs.rename('r_exp', 'theta_exp') # SDSS and Tractor have different opinions on which way this rotation goes objs.phi_dev_deg *= -1. objs.phi_exp_deg *= -1. # MAGIC -- minimum size of galaxy. objs.theta_dev = np.maximum(objs.theta_dev, 1. / 30.) objs.theta_exp = np.maximum(objs.theta_exp, 1. / 30.) if forcePointSources: Lstar = np.ones(len(objs), float) Lgal = np.zeros(len(objs), float) Ldev = Lexp = Lgal else: if useObjcType: objs.cut(np.logical_or(objs.objc_type == 6, objs.objc_type == 3)) Lstar = (objs.objc_type == 6) Lgal = (objs.objc_type == 3) else: Lstar = (objs.prob_psf[:, bandnum] == 1) * 1.0 Lgal = (objs.prob_psf[:, bandnum] == 0) * 1.0 if isdr7: fracdev = objs.fracpsf[:, bandnum] else: fracdev = objs.fracdev[:, bandnum] Ldev = Lgal * fracdev Lexp = Lgal * (1. - fracdev) if isdr7: from .sdss_dr7 import _dr7_getBrightness if nanomaggies: raise RuntimeError('Nanomaggies not supported for DR7 (yet)') def lup2bright(lups): counts = [ tsf.luptitude_to_counts(lup, j) for j, lup in enumerate(lups) ] counts = np.array(counts) bright = _dr7_getBrightness(counts, tsf, bandnames, extrabands) return bright flux2bright = lup2bright starflux = objs.psfcounts compflux = objs.counts_model devflux = objs.counts_dev expflux = objs.counts_exp def comp2bright(lups, Ldev, Lexp): counts = [ tsf.luptitude_to_counts(lup, j) for j, lup in enumerate(lups) ] counts = np.array(counts) dcounts = counts * Ldev ecounts = counts * Lexp dbright = _dr7_getBrightness(dcounts, tsf, bands, extrabands) ebright = _dr7_getBrightness(ecounts, tsf, bands, extrabands) return dbright, ebright else: def nmgy2bright(flux): if len(bandnums): flux = flux[bandnums] else: flux = flux[np.array([bandnum])] bb = bandnames + extrabands if nanomaggies: if len(extrabands): if len(bandnums) == 0: # Only "extrabands", no SDSS bands. flux = np.zeros(len(extrabands)) + flux[0] else: flux = np.append(flux, np.zeros(len(extrabands))) bright = NanoMaggies(order=bb, **dict(zip(bb, flux))) else: I = (flux > 0) mag = np.zeros_like(flux) + badmag mag[I] = sdss.nmgy_to_mag(flux[I]) if len(extrabands): mag = np.append(mag, np.zeros(len(extrabands)) + badmag) bright = Mags(order=bb, **dict(zip(bb, mag))) return bright def comp2bright(flux, Ldev, Lexp): dflux = flux * Ldev eflux = flux * Lexp dbright = nmgy2bright(dflux) ebright = nmgy2bright(eflux) return dbright, ebright flux2bright = nmgy2bright starflux = objs.psfflux compflux = objs.cmodelflux devflux = objs.devflux expflux = objs.expflux sources = [] nstars, ndev, nexp, ncomp = 0, 0, 0, 0 isources = [] ptsrcclass = classmap.get(PointSource, PointSource) for i in range(len(objs)): if Lstar[i]: pos = RaDecPos(objs.ra[i], objs.dec[i]) flux = starflux[i, :] bright = flux2bright(flux) # This should work, I just don't feel like testing it now... # if brightPointSourceThreshold: # ps = SdssPointSource(pos, bright, thresh=brightPointSourceThreshold) # else: # ps = PointSource(pos, bright) sources.append(ptsrcclass(pos, bright)) nstars += 1 isources.append(i) continue hasdev = (Ldev[i] > 0) hasexp = (Lexp[i] > 0) iscomp = (hasdev and hasexp) pos = RaDecPos(objs.ra[i], objs.dec[i]) if iscomp: flux = compflux[i, :] elif hasdev: flux = devflux[i, :] elif hasexp: flux = expflux[i, :] else: print( 'Skipping object with Lstar = %g, Ldev = %g, Lexp = %g (fracdev=%g)' % (Lstar[i], Ldev[i], Lexp[i], fracdev[i])) continue isources.append(i) if iscomp: if fixedComposites: bright = flux2bright(flux) fdev = (Ldev[i] / (Ldev[i] + Lexp[i])) else: dbright, ebright = comp2bright(flux, Ldev[i], Lexp[i]) else: bright = flux2bright(flux) if hasdev: re = objs.theta_dev[i, bandnum] ab = objs.ab_dev[i, bandnum] phi = objs.phi_dev_deg[i, bandnum] dshape = ellipse(re, ab, phi) if hasexp: re = objs.theta_exp[i, bandnum] ab = objs.ab_exp[i, bandnum] phi = objs.phi_exp_deg[i, bandnum] eshape = ellipse(re, ab, phi) if iscomp: if fixedComposites: gal = FixedCompositeGalaxy(pos, bright, fdev, eshape, dshape) else: gal = CompositeGalaxy(pos, ebright, eshape, dbright, dshape) ncomp += 1 elif hasdev: gal = DevGalaxy(pos, bright, dshape) ndev += 1 elif hasexp: gal = ExpGalaxy(pos, bright, eshape) nexp += 1 sources.append(gal) print( 'Created', ndev, 'deV,', nexp, 'exp,', ncomp, 'composite', ) print('(total %i) galaxies and %i stars' % (ndev + nexp + ncomp, nstars)) if not (getobjs or getobjinds or getsourceobjs): return sources if nstars + ndev + nexp + ncomp < len(objs): objs = objs[np.array(isources)] rtn = [sources] if getobjs: rtn.append(allobjs) if getobjinds: rtn.append(objs.index if len(objs) else np.array([])) if getsourceobjs: rtn.append(objs) return rtn
def _get_tractor_image_dr8(run, camcol, field, bandname, sdss=None, roi=None, psf='kl-gm', roiradecsize=None, roiradecbox=None, savepsfimg=None, curl=False, nanomaggies=False, zrange=[-3,10], invvarIgnoresSourceFlux=False, invvarAtCenter=False, invvarAtCenterImage=False, imargs={}): # retry_retrieve=True, ''' Creates a tractor.Image given an SDSS field identifier. If not None, roi = (x0, x1, y0, y1) defines a region-of-interest in the image, in zero-indexed pixel coordinates. x1,y1 are NON-inclusive; roi=(0,100,0,100) will yield a 100 x 100 image. psf can be: "dg" for double-Gaussian "kl-gm" for SDSS KL-decomposition approximated as a Gaussian mixture "bright-*", "*" one of the above PSFs, with special handling at the bright end. "roiradecsize" = (ra, dec, half-size in pixels) indicates that you want to grab a ROI around the given RA,Dec. "roiradecbox" = (ra0, ra1, dec0, dec1) indicates that you want to grab a ROI containing the given RA,Dec ranges. "invvarAtCentr" -- get a scalar constant inverse-variance "invvarAtCenterImage" -- get a scalar constant inverse-variance but still make an image out of it. Returns: (tractor.Image, dict) dict contains useful details like: 'sky' 'skysig' ''' from astrometry.sdss import band_index origpsf = psf if psf.startswith('bright-'): psf = psf[7:] brightpsf = True print('Setting bright PSF handling') else: brightpsf = False valid_psf = ['dg', 'kl-gm', 'kl-pix'] if psf not in valid_psf: raise RuntimeError('PSF must be in ' + str(valid_psf)) if sdss is None: from astrometry.sdss import DR8 sdss = DR8(curl=curl) bandnum = band_index(bandname) for ft in ['psField', 'fpM']: fn = sdss.retrieve(ft, run, camcol, field, bandname) fn = sdss.retrieve('frame', run, camcol, field, bandname) # http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/frames/RERUN/RUN/CAMCOL/frame.html frame = sdss.readFrame(run, camcol, field, bandname, filename=fn) #image = frame.getImage().astype(np.float32) #(H,W) = image.shape H,W = frame.getImageShape() info = dict() hdr = frame.getHeader() tai = hdr.get('TAI') stripe = hdr.get('STRIPE') strip = hdr.get('STRIP') obj = hdr.get('OBJECT') info.update(tai=tai, stripe=stripe, strip=strip, object=obj, hdr=hdr) astrans = frame.getAsTrans() wcs = SdssWcs(astrans) #print('Created SDSS Wcs:', wcs) #print('(x,y) = 1,1 -> RA,Dec', wcs.pixelToPosition(1,1)) X = interpret_roi(wcs, (H,W), roi=roi, roiradecsize=roiradecsize, roiradecbox=roiradecbox) if X is None: return None,None roi,hasroi = X info.update(roi=roi) x0,x1,y0,y1 = roi # Mysterious half-pixel shift. asTrans pixel coordinates? wcs.setX0Y0(x0 + 0.5, y0 + 0.5) if nanomaggies: photocal = LinearPhotoCal(1., band=bandname) else: photocal = SdssNanomaggiesPhotoCal(bandname) sky = 0. skyobj = ConstantSky(sky) calibvec = frame.getCalibVec() invvarAtCenter = invvarAtCenter or invvarAtCenterImage psfield = sdss.readPsField(run, camcol, field) iva = dict(ignoreSourceFlux=invvarIgnoresSourceFlux) if invvarAtCenter: if hasroi: iva.update(constantSkyAt=((x0+x1)/2., (y0+y1)/2.)) else: iva.update(constantSkyAt=(W/2., H/2.)) invvar = frame.getInvvar(psfield, bandnum, **iva) invvar = invvar.astype(np.float32) if not invvarAtCenter: assert(invvar.shape == (H,W)) # Could get this from photoField instead # http://data.sdss3.org/datamodel/files/BOSS_PHOTOOBJ/RERUN/RUN/photoField.html gain = psfield.getGain(bandnum) darkvar = psfield.getDarkVariance(bandnum) meansky = np.mean(frame.sky) meancalib = np.mean(calibvec) skysig = sqrt((meansky / gain) + darkvar) * meancalib info.update(sky=sky, skysig=skysig) zr = np.array(zrange)*skysig + sky info.update(zr=zr) # http://data.sdss3.org/datamodel/files/PHOTO_REDUX/RERUN/RUN/objcs/CAMCOL/fpM.html fpM = sdss.readFpM(run, camcol, field, bandname) if not hasroi: image = frame.getImage() else: roislice = (slice(y0,y1), slice(x0,x1)) image = frame.getImageSlice(roislice).astype(np.float32) if invvarAtCenterImage: invvar = invvar + np.zeros(image.shape, np.float32) elif invvarAtCenter: pass else: invvar = invvar[roislice].copy() H,W = image.shape if (not invvarAtCenter) or invvarAtCenterImage: for plane in [ 'INTERP', 'SATUR', 'CR', 'GHOST' ]: fpM.setMaskedPixels(plane, invvar, 0, roi=roi) dgpsf = psfield.getDoubleGaussian(bandnum, normalize=True) info.update(dgpsf=dgpsf) if psf == 'kl-pix': # Pixelized KL-PSF klpsf = psfield.getPsfAtPoints(bandnum, x0+W/2, y0+H/2) # Trim symmetric zeros sh,sw = klpsf.shape while True: if (np.all(klpsf[0,:] == 0.) and np.all(klpsf[:,0] == 0.) and np.all(klpsf[-1,:] == 0.) and np.all(klpsf[:,-1] == 0.)): klpsf = klpsf[1:-1, 1:-1] else: break mypsf = PixelizedPSF(klpsf) elif psf == 'kl-gm': from emfit import em_fit_2d from fitpsf import em_init_params # Create Gaussian mixture model PSF approximation. klpsf = psfield.getPsfAtPoints(bandnum, x0+W/2, y0+H/2) S = klpsf.shape[0] # number of Gaussian components K = 3 w,mu,sig = em_init_params(K, None, None, None) II = klpsf.copy() II /= II.sum() # HIDEOUS HACK II = np.maximum(II, 0) #print('Multi-Gaussian PSF fit...') xm,ym = -(S/2), -(S/2) if savepsfimg is not None: plt.clf() plt.imshow(II, interpolation='nearest', origin='lower') plt.title('PSF image to fit with EM') plt.savefig(savepsfimg) res = em_fit_2d(II, xm, ym, w, mu, sig) #print('em_fit_2d result:', res) if res == 0: # print('w,mu,sig', w,mu,sig) mypsf = GaussianMixturePSF(w, mu, sig) mypsf.computeRadius() else: # Failed! Return 'dg' model instead? print('PSF model fit', psf, 'failed! Returning DG model instead') psf = 'dg' if psf == 'dg': print('Creating double-Gaussian PSF approximation') (a,s1, b,s2) = dgpsf mypsf = NCircularGaussianPSF([s1, s2], [a, b]) if brightpsf: print('Wrapping PSF in SdssBrightPSF') (a1,s1, a2,s2, a3,sigmap,beta) = psfield.getPowerLaw(bandnum) mypsf = SdssBrightPSF(mypsf, a1,s1,a2,s2,a3,sigmap,beta) print('PSF:', mypsf) timg = Image(data=image, invvar=invvar, psf=mypsf, wcs=wcs, sky=skyobj, photocal=photocal, name=('SDSS (r/c/f/b=%i/%i/%i/%s)' % (run, camcol, field, bandname)), time=TAITime(tai), **imargs) timg.zr = zr return timg,info
def get_tractor_image(run, camcol, field, bandname, sdssobj=None, release='DR7', retrieve=True, curl=False, psf='kl-gm', useMags=False, roi=None, roiradecsize=None, roiradecbox=None, nanomaggies=False, savepsfimg=None, zrange=[-3,10]): ''' Creates a tractor.Image given an SDSS field identifier. If not None, roi = (x0, x1, y0, y1) defines a region-of-interest in the image, in zero-indexed pixel coordinates. x1,y1 are NON-inclusive; roi=(0,100,0,100) will yield a 100 x 100 image. psf can be: "dg" for double-Gaussian "kl-gm" for SDSS KL-decomposition approximated as a Gaussian mixture "roiradecsize" = (ra, dec, half-size in pixels) indicates that you want to grab a ROI around the given RA,Dec. Returns: (tractor.Image, dict) dict contains useful details like: 'sky' 'skysig' ''' from astrometry.sdss import DR7, band_index if sdssobj is None: # Ugly if release != 'DR7': raise RuntimeError('We only support DR7 currently') sdss = DR7(curl=curl) else: sdss = sdssobj valid_psf = ['dg', 'kl-gm'] if psf not in valid_psf: raise RuntimeError('PSF must be in ' + str(valid_psf)) # FIXME rerun = 0 bandnum = band_index(bandname) _check_sdss_files(sdss, run, camcol, field, bandname, ['fpC', 'tsField', 'psField', 'fpM'], retrieve=retrieve) fpC = sdss.readFpC(run, camcol, field, bandname) hdr = fpC.getHeader() fpC = fpC.getImage() fpC = fpC.astype(np.float32) - sdss.softbias image = fpC (H,W) = image.shape info = dict() tai = hdr.get('TAI') stripe = hdr.get('STRIPE') strip = hdr.get('STRIP') obj = hdr.get('OBJECT') info.update(tai=tai, stripe=stripe, strip=strip, object=obj, hdr=hdr) tsf = sdss.readTsField(run, camcol, field, rerun) astrans = tsf.getAsTrans(bandnum) wcs = SdssWcs(astrans) #print('Created SDSS Wcs:', wcs) X = interpret_roi(wcs, (H,W), roi=roi, roiradecsize=roiradecsize, roiradecbox=roiradecbox) if X is None: return None,None roi,hasroi = X info.update(roi=roi) x0,x1,y0,y1 = roi # Mysterious half-pixel shift. asTrans pixel coordinates? wcs.setX0Y0(x0 + 0.5, y0 + 0.5) if nanomaggies: zp = tsf.get_zeropoint(bandnum) photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp), band=bandname) elif useMags: photocal = SdssMagsPhotoCal(tsf, bandname) else: photocal = SdssFluxPhotoCal() psfield = sdss.readPsField(run, camcol, field) sky = psfield.getSky(bandnum) skysig = sqrt(sky) skyobj = ConstantSky(sky) zr = sky + np.array(zrange) * skysig info.update(sky=sky, skysig=skysig, zr=zr) fpM = sdss.readFpM(run, camcol, field, bandname) gain = psfield.getGain(bandnum) darkvar = psfield.getDarkVariance(bandnum) skyerr = psfield.getSkyErr(bandnum) invvar = sdss.getInvvar(fpC, fpM, gain, darkvar, sky, skyerr) dgpsf = psfield.getDoubleGaussian(bandnum, normalize=True) info.update(dgpsf=dgpsf) if roi is not None: roislice = (slice(y0,y1), slice(x0,x1)) image = image[roislice].copy() invvar = invvar[roislice].copy() if psf == 'kl-gm': from emfit import em_fit_2d from fitpsf import em_init_params # Create Gaussian mixture model PSF approximation. H,W = image.shape klpsf = psfield.getPsfAtPoints(bandnum, x0+W/2, y0+H/2) S = klpsf.shape[0] # number of Gaussian components K = 3 w,mu,sig = em_init_params(K, None, None, None) II = klpsf.copy() II /= II.sum() # HIDEOUS HACK II = np.maximum(II, 0) #print('Multi-Gaussian PSF fit...') xm,ym = -(S/2), -(S/2) if savepsfimg is not None: plt.clf() plt.imshow(II, interpolation='nearest', origin='lower') plt.title('PSF image to fit with EM') plt.savefig(savepsfimg) res = em_fit_2d(II, xm, ym, w, mu, sig) print('em_fit_2d result:', res) if res == 0: # print('w,mu,sig', w,mu,sig) mypsf = GaussianMixturePSF(w, mu, sig) mypsf.computeRadius() else: # Failed! Return 'dg' model instead? print('PSF model fit', psf, 'failed! Returning DG model instead') psf = 'dg' if psf == 'dg': print('Creating double-Gaussian PSF approximation') (a,s1, b,s2) = dgpsf mypsf = NCircularGaussianPSF([s1, s2], [a, b]) timg = Image(data=image, invvar=invvar, psf=mypsf, wcs=wcs, sky=skyobj, photocal=photocal, name=('SDSS (r/c/f/b=%i/%i/%i/%s)' % (run, camcol, field, bandname))) timg.zr = zr return timg,info
def _get_sources(run, camcol, field, bandname='r', sdss=None, release='DR7', objs=None, retrieve=True, curl=False, roi=None, radecroi=None, radecrad=None, bands=None, badmag=25, nanomaggies=False, getobjs=False, getsourceobjs=False, getobjinds=False, extrabands=None, fixedComposites=False, forcePointSources=False, useObjcType=False, objCuts=True, classmap={}, ellipse=GalaxyShape, cutToPrimary=False): ''' If set, radecrad = (ra,dec,rad) returns sources within "rad" degrees of the given RA,Dec (in degrees) center. WARNING, this method alters the "objs" argument, if given. Consider calling objs.copy() before calling. -"bandname" is the SDSS band used to cut on position, select star/gal/exp/dev, and set galaxy shapes. -"bands" are the bands to include in the returned Source objects; they will be initialized from the SDSS bands. -"extrabands" are also included in the returned Source objects; they will be initialized to the SDSS flux for either the first of "bands", if given, or "bandname". ''' from astrometry.sdss import (DR7, DR8, DR9, band_names, band_index, photo_flags1_map) # brightPointSourceThreshold=0.): if sdss is None: dr = dict(DR7=DR7, DR8=DR8, DR9=DR9)[release] sdss = dr(curl=curl) drnum = sdss.getDRNumber() isdr7 = (drnum == 7) if bands is None: bands = band_names() bandnum = band_index(bandname) bandnums = np.array([band_index(b) for b in bands]) bandnames = bands if extrabands is None: extrabands = [] if objs is None: from astrometry.util.fits import fits_table if isdr7: # FIXME rerun = 0 _check_sdss_files(sdss, run, camcol, field, bandnum, ['tsObj', 'tsField'], retrieve=retrieve, tryopen=True) tsf = sdss.readTsField(run, camcol, field, rerun) objfn = sdss.getPath('tsObj', run, camcol, field, bandname, rerun=rerun) else: _check_sdss_files(sdss, run, camcol, field, bandnum, ['photoObj'], tryopen=True, retrieve=retrieve) objfn = sdss.getPath('photoObj', run, camcol, field) objs = fits_table(objfn) if objs is None: print('No sources in SDSS file', objfn) return [] objs.index = np.arange(len(objs)) if getobjs: allobjs = objs.copy() if roi is not None: x0,x1,y0,y1 = roi # FIXME -- we keep only the sources whose centers are within # the ROI box. Should instead do some ellipse-overlaps # geometry. x = objs.colc[:,bandnum] y = objs.rowc[:,bandnum] objs.cut((x >= x0) * (x < x1) * (y >= y0) * (y < y1)) if radecroi is not None: r0,r1,d0,d1 = radecroi objs.cut((objs.ra >= r0) * (objs.ra <= r1) * (objs.dec >= d0) * (objs.dec <= d1)) if radecrad is not None: from astrometry.libkd.spherematch import match_radec (ra,dec,rad) = radecrad I,J,d = match_radec(ra, dec, objs.ra, objs.dec, rad) objs.cut(J) del I del d if objCuts: # Only deblended children; # No BRIGHT sources bright = photo_flags1_map.get('BRIGHT') objs.cut((objs.nchild == 0) * ((objs.objc_flags & bright) == 0)) if cutToPrimary: objs.cut((objs.resolve_status & 256) > 0) if isdr7: objs.rename('phi_dev', 'phi_dev_deg') objs.rename('phi_exp', 'phi_exp_deg') objs.rename('r_dev', 'theta_dev') objs.rename('r_exp', 'theta_exp') # SDSS and Tractor have different opinions on which way this rotation goes objs.phi_dev_deg *= -1. objs.phi_exp_deg *= -1. # MAGIC -- minimum size of galaxy. objs.theta_dev = np.maximum(objs.theta_dev, 1./30.) objs.theta_exp = np.maximum(objs.theta_exp, 1./30.) if forcePointSources: Lstar = np.ones(len(objs), float) Lgal = np.zeros(len(objs), float) Ldev = Lexp = Lgal else: if useObjcType: objs.cut(np.logical_or(objs.objc_type == 6, objs.objc_type == 3)) Lstar = (objs.objc_type == 6) Lgal = (objs.objc_type == 3) else: Lstar = (objs.prob_psf[:,bandnum] == 1) * 1.0 Lgal = (objs.prob_psf[:,bandnum] == 0) if isdr7: fracdev = objs.fracpsf[:,bandnum] else: fracdev = objs.fracdev[:,bandnum] Ldev = Lgal * fracdev Lexp = Lgal * (1. - fracdev) if isdr7: if nanomaggies: raise RuntimeError('Nanomaggies not supported for DR7 (yet)') def lup2bright(lups): counts = [tsf.luptitude_to_counts(lup,j) for j,lup in enumerate(lups)] counts = np.array(counts) bright = _getBrightness(counts, tsf, bandnames, extrabands) return bright flux2bright = lup2bright starflux = objs.psfcounts compflux = objs.counts_model devflux = objs.counts_dev expflux = objs.counts_exp def comp2bright(lups, Ldev, Lexp): counts = [tsf.luptitude_to_counts(lup,j) for j,lup in enumerate(lups)] counts = np.array(counts) dcounts = counts * Ldev ecounts = counts * Lexp dbright = _getBrightness(dcounts, tsf, bands, extrabands) ebright = _getBrightness(ecounts, tsf, bands, extrabands) return dbright, ebright else: def nmgy2bright(flux): if len(bandnums): flux = flux[bandnums] else: flux = flux[np.array([bandnum])] bb = bandnames + extrabands if nanomaggies: if len(extrabands): if len(bandnums) == 0: # Only "extrabands", no SDSS bands. flux = np.zeros(len(extrabands)) + flux[0] else: flux = np.append(flux, np.zeros(len(extrabands))) bright = NanoMaggies(order=bb, **dict(zip(bb, flux))) else: I = (flux > 0) mag = np.zeros_like(flux) + badmag mag[I] = sdss.nmgy_to_mag(flux[I]) if len(extrabands): mag = np.append(mag, np.zeros(len(extrabands)) + badmag) bright = Mags(order=bb, **dict(zip(bb,mag))) return bright def comp2bright(flux, Ldev, Lexp): dflux = flux * Ldev eflux = flux * Lexp dbright = nmgy2bright(dflux) ebright = nmgy2bright(eflux) return dbright, ebright flux2bright = nmgy2bright starflux = objs.psfflux compflux = objs.cmodelflux devflux = objs.devflux expflux = objs.expflux sources = [] nstars, ndev, nexp, ncomp = 0, 0, 0, 0 isources = [] ptsrcclass = classmap.get(PointSource, PointSource) for i in range(len(objs)): if Lstar[i]: pos = RaDecPos(objs.ra[i], objs.dec[i]) flux = starflux[i,:] bright = flux2bright(flux) # This should work, I just don't feel like testing it now... # if brightPointSourceThreshold: # ps = SdssPointSource(pos, bright, thresh=brightPointSourceThreshold) # else: # ps = PointSource(pos, bright) sources.append(ptsrcclass(pos, bright)) nstars += 1 isources.append(i) continue hasdev = (Ldev[i] > 0) hasexp = (Lexp[i] > 0) iscomp = (hasdev and hasexp) pos = RaDecPos(objs.ra[i], objs.dec[i]) if iscomp: flux = compflux[i,:] elif hasdev: flux = devflux[i,:] elif hasexp: flux = expflux[i,:] else: print('Skipping object with Lstar = %g, Ldev = %g, Lexp = %g (fracdev=%g)' % (Lstar[i], Ldev[i], Lexp[i], fracdev[i])) continue isources.append(i) if iscomp: if fixedComposites: bright = flux2bright(flux) fdev = (Ldev[i] / (Ldev[i] + Lexp[i])) else: dbright,ebright = comp2bright(flux, Ldev[i], Lexp[i]) else: bright = flux2bright(flux) if hasdev: re = objs.theta_dev [i,bandnum] ab = objs.ab_dev [i,bandnum] phi = objs.phi_dev_deg[i,bandnum] dshape = ellipse(re, ab, phi) if hasexp: re = objs.theta_exp [i,bandnum] ab = objs.ab_exp [i,bandnum] phi = objs.phi_exp_deg[i,bandnum] eshape = ellipse(re, ab, phi) if iscomp: if fixedComposites: gal = FixedCompositeGalaxy(pos, bright, fdev, eshape, dshape) else: gal = CompositeGalaxy(pos, ebright, eshape, dbright, dshape) ncomp += 1 elif hasdev: gal = DevGalaxy(pos, bright, dshape) ndev += 1 elif hasexp: gal = ExpGalaxy(pos, bright, eshape) nexp += 1 sources.append(gal) print('Created', ndev, 'deV,', nexp, 'exp,', ncomp, 'composite',) print('(total %i) galaxies and %i stars' % (ndev+nexp+ncomp, nstars)) if not (getobjs or getobjinds or getsourceobjs): return sources if nstars + ndev + nexp + ncomp < len(objs): objs = objs[np.array(isources)] rtn = [sources] if getobjs: rtn.append(allobjs) if getobjinds: rtn.append(objs.index if len(objs) else np.array([])) if getsourceobjs: rtn.append(objs) return rtn