def main(): """Main program. """ import argparse parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--force', action='store_true', help='Run calib processes even if files already exist?') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--expnum', type=int, help='Cut to a single exposure') parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name') parser.add_argument('--no-astrom', dest='astrom', action='store_false', help='Do not compute astrometric calibs') parser.add_argument('--no-psf', dest='psfex', action='store_false', help='Do not compute PsfEx calibs') parser.add_argument('--no-sky', dest='sky', action='store_false', help='Do not compute sky models') parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor') parser.add_argument('--splinesky', action='store_true', help='Spline sky, not constant') parser.add_argument('args',nargs=argparse.REMAINDER) opt = parser.parse_args() D = Decals() if opt.ccds is not None: T = fits_table(opt.ccds) print('Read', len(T), 'from', opt.ccds) else: T = D.get_ccds() #print len(T), 'CCDs' if len(opt.args) == 0: if opt.expnum is not None: T.cut(T.expnum == opt.expnum) print('Cut to', len(T), 'with expnum =', opt.expnum) if opt.extname is not None: T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname])) print('Cut to', len(T), 'with extname =', opt.extname) opt.args = range(len(T)) for a in opt.args: i = int(a) t = T[i] im = D.get_image_object(t) print('Running', im.calname) kwargs = dict(pvastrom=opt.astrom, psfex=opt.psfex, sky=opt.sky) if opt.force: kwargs.update(force=True) if opt.run_se: kwargs.update(se=True) if opt.splinesky: kwargs.update(splinesky=True) run_calibs((im, kwargs)) return 0
def main(): brickname = '2428p117' objtype = 'STAR' lobjtype = objtype.lower() decals_sim_dir = '/Users/ioannis/decals_sim_dir/star/2428p117' zoom = [1800,2400,1800,2400] #zoom = [1800,2000,1800,2000] #zoom = None decals = Decals() brickinfo = decals.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brickinfo) W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale() if zoom is not None: # See also runbrick.stage_tims() #(x0,x1,y0,y1) = args.zoom (x0,x1,y0,y1) = zoom W = x1-x0 H = y1-y0 targetwcs = brickwcs.get_subimage(x0, y0, W, H) bounds = brickwcs.radec_bounds() ra_range = bounds[1]-bounds[0] dec_range = bounds[3]-bounds[2] radec_center = brickwcs.radec_center() print(radec_center, ra_range, dec_range) corners = np.array([brickwcs.pixelxy2radec(x,y) for x,y in [(1,1),(W,1),(W,H),(1,H),(1,1)]]) # Identify the CCDs in the region of interest. ccdinfo = decals.ccds_touching_wcs(brickwcs) ccdinfo.about() log.info('Got {} CCDs'.format(len(ccdinfo))) # Generate the catalog of simulated sources here! simcat = fits_table('simcat-{}-{}-00.fits'.format(brickname,lobjtype)) #simcat = fits.getdata('simcat-{}-{}-00.fits'.format(brickname,lobjtype)) simcat.about() blobxy = zip(simcat.x,simcat.y) simdecals = SimDecals(sim_sources=simcat) # For testing! #sim = SimImage(simdecals,ccdinfo[0]) #tim = sim.get_tractor_image(const2psf=True) run_brick(brickname, decals=simdecals, outdir=decals_sim_dir, threads=8, zoom=zoom, wise=False, sdssInit=False, forceAll=True, writePickles=False, blobxy=blobxy, pixPsf=False, stages=['tims','srcs','fitblobs', 'fitblobs_finish','coadds', 'writecat'])
def __init__(self,expnum=None,ccdname=None,ccdwcs=None): """Initialize the class with either the exposure number *and* CCD name, or directly with the WCS of the CCD of interest. """ self.ps1dir = os.getenv('PS1CAT_DIR') self.nside = 32 if ccdwcs is None: from legacypipe.common import Decals, DecamImage decals = Decals() ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0] im = DecamImage(decals,ccd) self.ccdwcs = im.get_wcs() else: self.ccdwcs = ccdwcs
def __init__(self,expnum=None,ccdname=None,ccdwcs=None): """Initialize the class with either the exposure number *and* CCD name, or directly with the WCS of the CCD of interest. """ self.ps1dir = os.getenv('PS1CAT_DIR') if self.ps1dir is None: raise ValueError('You must have the PS1CAT_DIR environment variable set to point to Pan-STARRS1 catalogs') self.nside = 32 if ccdwcs is None: from legacypipe.common import Decals decals = Decals() ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0] im = decals.get_image_object(ccd) self.ccdwcs = im.get_wcs() else: self.ccdwcs = ccdwcs
def __init__(self, expnum=None, ccdname=None, ccdwcs=None): """Initialize the class with either the exposure number *and* CCD name, or directly with the WCS of the CCD of interest. """ self.ps1dir = os.getenv('PS1CAT_DIR') if self.ps1dir is None: raise ValueError( 'You must have the PS1CAT_DIR environment variable set to point to Pan-STARRS1 catalogs' ) self.nside = 32 if ccdwcs is None: from legacypipe.common import Decals decals = Decals() ccd = decals.find_ccds(expnum=expnum, ccdname=ccdname)[0] im = decals.get_image_object(ccd) self.ccdwcs = im.get_wcs() else: self.ccdwcs = ccdwcs
def psf_residuals(expnum, ccdname, stampsize=35, nstar=30, magrange=(13, 17), verbose=0, splinesky=False): # Set the debugging level. if verbose == 0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) pngprefix = 'qapsf-{}-{}'.format(expnum, ccdname) # Gather all the info we need about this CCD. decals = Decals() ccd = decals.find_ccds(expnum=expnum, ccdname=ccdname)[0] band = ccd.filter ps1band = dict(g=0, r=1, i=2, z=3, Y=4) print('Band {}'.format(band)) #scales = dict(g=0.0066, r=0.01, z=0.025) #vmin, vmax = np.arcsinh(-1), np.arcsinh(100) #print(scales[band]) im = decals.get_image_object(ccd) iminfo = im.get_image_info() H, W = iminfo['dims'] wcs = im.get_wcs() # Choose a uniformly selected subset of PS1 stars on this CCD. ps1 = ps1cat(ccdwcs=wcs) cat = ps1.get_stars(band=band, magrange=magrange) rand = np.random.RandomState(seed=expnum * ccd.ccdnum) these = rand.choice(len(cat) - 1, nstar, replace=False) #these = rand.random_integers(0,len(cat)-1,nstar) cat = cat[these] cat = cat[np.argsort(cat.median[:, ps1band[band]])] # sort by magnitude #print(cat.nmag_ok) get_tim_kwargs = dict(const2psf=True, splinesky=splinesky) # Make a QAplot of the positions of all the stars. tim = im.get_tractor_image(**get_tim_kwargs) img = tim.getImage() #img = tim.getImage()/scales[band] fig = plt.figure(figsize=(5, 10)) ax = fig.gca() ax.get_xaxis().get_major_formatter().set_useOffset(False) #ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest', # origin='lower',vmin=vmax,vmax=vmax) ax.imshow(img, **tim.ima) ax.axis('off') ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.format( band, expnum, ccdname, ccd.airmass, ccd.seeing)) for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ax.text(xpos, ypos, '{:2d}'.format(istar + 1), color='red', horizontalalignment='left') circ = plt.Circle((xpos, ypos), radius=30, color='g', fill=False, lw=1) ax.add_patch(circ) #radec = wcs.radec_bounds() #ax.scatter(cat.ra,cat.dec) #ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998]) #ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015]) #ax.set_xlabel('$RA\ (deg)$',fontsize=18) #ax.set_ylabel('$Dec\ (deg)$',fontsize=18) fig.savefig(pngprefix + '-ccd.png', bbox_inches='tight') # Initialize the many-stamp QAplot ncols = 3 nrows = np.ceil(nstar / ncols).astype('int') inchperstamp = 2.0 fig = plt.figure(figsize=(inchperstamp * 3 * ncols, inchperstamp * nrows)) irow = 0 icol = 0 for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) mag = ps1star.median[ps1band[band]] # r-band ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ix, iy = int(xpos), int(ypos) # create a little tractor Image object around the star slc = (slice(max(iy - stampsize, 0), min(iy + stampsize + 1, H)), slice(max(ix - stampsize, 0), min(ix + stampsize + 1, W))) # The PSF model 'const2Psf' is the one used in DR1: a 2-component # Gaussian fit to PsfEx instantiated in the image center. tim = im.get_tractor_image(slc=slc, **get_tim_kwargs) stamp = tim.getImage() ivarstamp = tim.getInvvar() # Initialize a tractor PointSource from PS1 measurements flux = NanoMaggies.magToNanomaggies(mag) star = PointSource(RaDecPos(ra, dec), NanoMaggies(**{band: flux})) # Fit just the source RA,Dec,flux. tractor = Tractor([tim], [star]) tractor.freezeParam('images') print('2-component MOG:', tim.psf) tractor.printThawedParams() for step in range(50): dlnp, X, alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_mog = tractor.getModelImage(0) chi2_mog = -2.0 * tractor.getLogLikelihood() mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0] # Now change the PSF model to a pixelized PSF model from PsfEx instantiated # at this place in the image. psf = PixelizedPsfEx(im.psffn) tim.psf = psf.constantPsfAt(xpos, ypos) #print('PSF model:', tim.psf) #tractor.printThawedParams() for step in range(50): dlnp, X, alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_psfex = tractor.getModelImage(0) chi2_psfex = -2.0 * tractor.getLogLikelihood() mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0] #mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95]) sig = np.std((stamp - model_psfex)[ivarstamp > 0]) mn, mx = [-2.0 * sig, 5 * sig] # Generate a QAplot. if (istar > 0) and (istar % (ncols) == 0): irow = irow + 1 icol = 3 * istar - 3 * ncols * irow #print(istar, irow, icol, icol+1, icol+2) ax1 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol), aspect='equal') ax1.axis('off') #ax1.imshow(stamp, **tim.ima) ax1.imshow(stamp, cmap='gray', interpolation='nearest', origin='lower', vmin=mn, vmax=mx) ax1.text(0.1, 0.9, '{:2d}'.format(istar + 1), color='white', horizontalalignment='left', verticalalignment='top', transform=ax1.transAxes) ax2 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 1), aspect='equal') ax2.axis('off') #ax2.imshow(stamp-model_mog, **tim.ima) ax2.imshow(stamp - model_mog, cmap='gray', interpolation='nearest', origin='lower', vmin=mn, vmax=mx) ax2.text(0.1, 0.9, 'MoG', color='white', horizontalalignment='left', verticalalignment='top', transform=ax2.transAxes) ax2.text(0.08, 0.08, '{:.3f}'.format(mag_mog), color='white', horizontalalignment='left', verticalalignment='bottom', transform=ax2.transAxes) #ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14) #ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex)) ax3 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 2), aspect='equal') ax3.axis('off') #ax3.imshow(stamp-model_psfex, **tim.ima) ax3.imshow(stamp - model_psfex, cmap='gray', interpolation='nearest', origin='lower', vmin=mn, vmax=mx) ax3.text(0.1, 0.9, 'PSFEx', color='white', horizontalalignment='left', verticalalignment='top', transform=ax3.transAxes) ax3.text(0.08, 0.08, '{:.3f}'.format(mag_psfex), color='white', horizontalalignment='left', verticalalignment='bottom', transform=ax3.transAxes) if istar == (nstar - 1): break fig.savefig(pngprefix + '-stargrid.png', bbox_inches='tight')
from glob import glob from legacypipe.common import Decals, wcs_for_brick from map.utils import tiles_touching_wcs from map.views import map_decals_dr2 class duck(object): pass req = duck() req.META = dict(HTTP_IF_MODIFIED_SINCE=None) if __name__ == '__main__': decals = Decals() fns = glob('dr2p/coadd/*/*/*-image.jpg') fns.sort() bricknames = [] for fn in fns: print 'File', fn (h,t) = os.path.split(fn) (h,t) = os.path.split(h) print 'Brick:', t brickname = t bricknames.append(brickname) for brickname in bricknames: scaledfns = glob('data/scaled/decals-dr2/*/%s/image-%s-*.fits' % (brickname[:3], brickname))
def main(): decals = Decals() ccds = decals.get_ccds() print(len(ccds), 'CCDs') expnums = np.unique(ccds.expnum) print(len(expnums), 'unique exposures') for expnum in expnums: expnumstr = '%08i' % expnum skyoutfn = os.path.join('splinesky', expnumstr[:5], 'decam-%s.fits' % expnumstr) psfoutfn = os.path.join('psfex', expnumstr[:5], 'decam-%s.fits' % expnumstr) if os.path.exists(skyoutfn) and os.path.exists(psfoutfn): print('Exposure', expnum, 'is done already') continue C = ccds[ccds.expnum == expnum] print(len(C), 'CCDs in expnum', expnum) psfex = [] psfhdrvals = [] splinesky = [] skyhdrvals = [] for ccd in C: im = decals.get_image_object(ccd) fn = im.splineskyfn if os.path.exists(fn): T = fits_table(fn) splinesky.append(T) # print(fn) # T.about() hdr = fitsio.read_header(fn) skyhdrvals.append( [hdr[k] for k in ['SKY', 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname]) else: print('File not found:', fn) fn = im.psffn if os.path.exists(fn): T = fits_table(fn) hdr = fitsio.read_header(fn, ext=1) keys = [ 'LOADED', 'ACCEPTED', 'CHI2', 'POLNAXIS', 'POLNGRP', 'PSF_FWHM', 'PSF_SAMP', 'PSFNAXIS', 'PSFAXIS1', 'PSFAXIS2', 'PSFAXIS3', ] if hdr['POLNAXIS'] == 0: # No polynomials. Fake it. T.polgrp1 = np.array([0]) T.polgrp2 = np.array([0]) T.polname1 = np.array(['fake']) T.polname2 = np.array(['fake']) T.polzero1 = np.array([0]) T.polzero2 = np.array([0]) T.polscal1 = np.array([1]) T.polscal2 = np.array([1]) T.poldeg1 = np.array([0]) T.poldeg2 = np.array([0]) else: keys.extend([ 'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1', 'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2', 'POLDEG1' ]) for k in keys: T.set(k.lower(), np.array([hdr[k]])) psfex.append(T) #print(fn) #T.about() hdr = fitsio.read_header(fn) psfhdrvals.append( [hdr.get(k, '') for k in ['LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname]) else: print('File not found:', fn) if len(psfex): padded = pad_arrays([p.psf_mask[0] for p in psfex]) cols = psfex[0].columns() cols.remove('psf_mask') T = merge_tables(psfex, columns=cols) T.psf_mask = np.concatenate([[p] for p in padded]) T.legpipev = np.array([h[0] for h in psfhdrvals]) T.plver = np.array([h[1] for h in psfhdrvals]) T.expnum = np.array([h[2] for h in psfhdrvals]) T.ccdname = np.array([h[3] for h in psfhdrvals]) fn = psfoutfn trymakedirs(fn, dir=True) T.writeto(fn) print('Wrote', fn) if len(splinesky): T = fits_table() T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky]) T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky]) padded = pad_arrays([t.gridvals[0] for t in splinesky]) T.gridvals = np.concatenate([[p] for p in padded]) padded = pad_arrays([t.xgrid[0] for t in splinesky]) T.xgrid = np.concatenate([[p] for p in padded]) padded = pad_arrays([t.xgrid[0] for t in splinesky]) T.ygrid = np.concatenate([[p] for p in padded]) cols = splinesky[0].columns() print('Columns:', cols) for c in ['gridvals', 'xgrid', 'ygrid']: cols.remove(c) T.add_columns_from(merge_tables(splinesky, columns=cols)) T.skyclass = np.array([h[0] for h in skyhdrvals]) T.legpipev = np.array([h[1] for h in skyhdrvals]) T.plver = np.array([h[2] for h in skyhdrvals]) T.expnum = np.array([h[3] for h in skyhdrvals]) T.ccdname = np.array([h[4] for h in skyhdrvals]) fn = skyoutfn trymakedirs(fn, dir=True) T.writeto(fn) print('Wrote', fn)
def psf_residuals(expnum,ccdname,stampsize=35,nstar=30, magrange=(13,17),verbose=0): # Set the debugging level. if verbose==0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout) pngprefix = 'qapsf-{}-{}'.format(expnum,ccdname) # Gather all the info we need about this CCD. decals = Decals() ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0] band = ccd.filter ps1band = dict(g=0,r=1,i=2,z=3,Y=4) print('Band {}'.format(band)) #scales = dict(g=0.0066, r=0.01, z=0.025) #vmin, vmax = np.arcsinh(-1), np.arcsinh(100) #print(scales[band]) im = DecamImage(decals,ccd) iminfo = im.get_image_info() H,W = iminfo['dims'] wcs = im.get_wcs() # Choose a uniformly selected subset of PS1 stars on this CCD. ps1 = ps1cat(ccdwcs=wcs) cat = ps1.get_stars(band=band,magrange=magrange) rand = np.random.RandomState(seed=expnum*ccd.ccdnum) these = rand.choice(len(cat)-1,nstar,replace=False) #these = rand.random_integers(0,len(cat)-1,nstar) cat = cat[these] cat = cat[np.argsort(cat.median[:,ps1band[band]])] # sort by magnitude #print(cat.nmag_ok) # Make a QAplot of the positions of all the stars. tim = im.get_tractor_image(const2psf=True) img = tim.getImage() #img = tim.getImage()/scales[band] fig = plt.figure(figsize=(5,10)) ax = fig.gca() ax.get_xaxis().get_major_formatter().set_useOffset(False) #ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest', # origin='lower',vmin=vmax,vmax=vmax) ax.imshow(img, **tim.ima) ax.axis('off') ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'. format(band,expnum,ccdname,ccd.airmass,ccd.seeing)) for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ax.text(xpos,ypos,'{:2d}'.format(istar+1),color='red', horizontalalignment='left') circ = plt.Circle((xpos,ypos),radius=30,color='g',fill=False,lw=1) ax.add_patch(circ) #radec = wcs.radec_bounds() #ax.scatter(cat.ra,cat.dec) #ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998]) #ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015]) #ax.set_xlabel('$RA\ (deg)$',fontsize=18) #ax.set_ylabel('$Dec\ (deg)$',fontsize=18) fig.savefig(pngprefix+'-ccd.png',bbox_inches='tight') # Initialize the many-stamp QAplot ncols = 3 nrows = np.ceil(nstar/ncols).astype('int') inchperstamp = 2.0 fig = plt.figure(figsize=(inchperstamp*3*ncols,inchperstamp*nrows)) irow = 0 icol = 0 for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) mag = ps1star.median[ps1band[band]] # r-band ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ix,iy = int(xpos), int(ypos) # create a little tractor Image object around the star slc = (slice(max(iy-stampsize, 0), min(iy+stampsize+1, H)), slice(max(ix-stampsize, 0), min(ix+stampsize+1, W))) # The PSF model 'const2Psf' is the one used in DR1: a 2-component # Gaussian fit to PsfEx instantiated in the image center. tim = im.get_tractor_image(slc=slc, const2psf=True) stamp = tim.getImage() ivarstamp = tim.getInvvar() # Initialize a tractor PointSource from PS1 measurements flux = NanoMaggies.magToNanomaggies(mag) star = PointSource(RaDecPos(ra,dec), NanoMaggies(**{band: flux})) # Fit just the source RA,Dec,flux. tractor = Tractor([tim], [star]) tractor.freezeParam('images') print('2-component MOG:', tim.psf) tractor.printThawedParams() for step in range(50): dlnp,X,alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_mog = tractor.getModelImage(0) chi2_mog = -2.0*tractor.getLogLikelihood() mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0] # Now change the PSF model to a pixelized PSF model from PsfEx instantiated # at this place in the image. psf = PixelizedPsfEx(im.psffn) tim.psf = psf.constantPsfAt(xpos, ypos) #print('PSF model:', tim.psf) #tractor.printThawedParams() for step in range(50): dlnp,X,alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_psfex = tractor.getModelImage(0) chi2_psfex = -2.0*tractor.getLogLikelihood() mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0] #mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95]) sig = np.std((stamp-model_psfex)[ivarstamp>0]) mn, mx = [-2.0*sig,5*sig] # Generate a QAplot. if (istar>0) and (istar%(ncols)==0): irow = irow+1 icol = 3*istar - 3*ncols*irow #print(istar, irow, icol, icol+1, icol+2) ax1 = plt.subplot2grid((nrows,3*ncols), (irow,icol), aspect='equal') ax1.axis('off') #ax1.imshow(stamp, **tim.ima) ax1.imshow(stamp,cmap='gray',interpolation='nearest', origin='lower',vmin=mn,vmax=mx) ax1.text(0.1,0.9,'{:2d}'.format(istar+1),color='white', horizontalalignment='left',verticalalignment='top', transform=ax1.transAxes) ax2 = plt.subplot2grid((nrows,3*ncols), (irow,icol+1), aspect='equal') ax2.axis('off') #ax2.imshow(stamp-model_mog, **tim.ima) ax2.imshow(stamp-model_mog,cmap='gray',interpolation='nearest', origin='lower',vmin=mn,vmax=mx) ax2.text(0.1,0.9,'MoG',color='white', horizontalalignment='left',verticalalignment='top', transform=ax2.transAxes) ax2.text(0.08,0.08,'{:.3f}'.format(mag_mog),color='white', horizontalalignment='left',verticalalignment='bottom', transform=ax2.transAxes) #ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14) #ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex)) ax3 = plt.subplot2grid((nrows,3*ncols), (irow,icol+2), aspect='equal') ax3.axis('off') #ax3.imshow(stamp-model_psfex, **tim.ima) ax3.imshow(stamp-model_psfex,cmap='gray',interpolation='nearest', origin='lower',vmin=mn,vmax=mx) ax3.text(0.1,0.9,'PSFEx',color='white', horizontalalignment='left',verticalalignment='top', transform=ax3.transAxes) ax3.text(0.08,0.08,'{:.3f}'.format(mag_psfex),color='white', horizontalalignment='left',verticalalignment='bottom', transform=ax3.transAxes) if istar==(nstar-1): break fig.savefig(pngprefix+'-stargrid.png',bbox_inches='tight')
def main(outfn='ccds-annotated.fits', ccds=None): decals = Decals() if ccds is None: ccds = decals.get_ccds() # File from the "observing" svn repo: # https://desi.lbl.gov/svn/decam/code/observing/trunk tiles = fits_table('decam-tiles_obstatus.fits') #ccds.cut(np.arange(100)) #print("HACK!") #ccds.cut(np.array([name in ['N15', 'N16', 'N21', 'N9'] # for name in ccds.ccdname]) * # ccds.expnum == 229683) I = decals.photometric_ccds(ccds) ccds.photometric = np.zeros(len(ccds), bool) ccds.photometric[I] = True I = decals.apply_blacklist(ccds) ccds.blacklist_ok = np.zeros(len(ccds), bool) ccds.blacklist_ok[I] = True ccds.good_region = np.empty((len(ccds), 4), np.int16) ccds.good_region[:,:] = -1 ccds.ra0 = np.zeros(len(ccds), np.float64) ccds.dec0 = np.zeros(len(ccds), np.float64) ccds.ra1 = np.zeros(len(ccds), np.float64) ccds.dec1 = np.zeros(len(ccds), np.float64) ccds.ra2 = np.zeros(len(ccds), np.float64) ccds.dec2 = np.zeros(len(ccds), np.float64) ccds.ra3 = np.zeros(len(ccds), np.float64) ccds.dec3 = np.zeros(len(ccds), np.float64) ccds.dra = np.zeros(len(ccds), np.float32) ccds.ddec = np.zeros(len(ccds), np.float32) ccds.ra_center = np.zeros(len(ccds), np.float64) ccds.dec_center = np.zeros(len(ccds), np.float64) ccds.sig1 = np.zeros(len(ccds), np.float32) ccds.meansky = np.zeros(len(ccds), np.float32) ccds.stdsky = np.zeros(len(ccds), np.float32) ccds.maxsky = np.zeros(len(ccds), np.float32) ccds.minsky = np.zeros(len(ccds), np.float32) ccds.pixscale_mean = np.zeros(len(ccds), np.float32) ccds.pixscale_std = np.zeros(len(ccds), np.float32) ccds.pixscale_max = np.zeros(len(ccds), np.float32) ccds.pixscale_min = np.zeros(len(ccds), np.float32) ccds.psfnorm_mean = np.zeros(len(ccds), np.float32) ccds.psfnorm_std = np.zeros(len(ccds), np.float32) ccds.galnorm_mean = np.zeros(len(ccds), np.float32) ccds.galnorm_std = np.zeros(len(ccds), np.float32) gaussgalnorm = np.zeros(len(ccds), np.float32) # 2nd moments ccds.psf_mx2 = np.zeros(len(ccds), np.float32) ccds.psf_my2 = np.zeros(len(ccds), np.float32) ccds.psf_mxy = np.zeros(len(ccds), np.float32) # ccds.psf_a = np.zeros(len(ccds), np.float32) ccds.psf_b = np.zeros(len(ccds), np.float32) ccds.psf_theta = np.zeros(len(ccds), np.float32) ccds.psf_ell = np.zeros(len(ccds), np.float32) ccds.humidity = np.zeros(len(ccds), np.float32) ccds.outtemp = np.zeros(len(ccds), np.float32) ccds.tileid = np.zeros(len(ccds), np.int32) ccds.tilepass = np.zeros(len(ccds), np.uint8) ccds.tileebv = np.zeros(len(ccds), np.float32) plvers = [] for iccd,ccd in enumerate(ccds): im = decals.get_image_object(ccd) print('Reading CCD %i of %i:' % (iccd+1, len(ccds)), im) X = im.get_good_image_subregion() for i,x in enumerate(X): if x is not None: ccds.good_region[iccd,i] = x W,H = ccd.width, ccd.height psf = None wcs = None sky = None try: tim = im.get_tractor_image(pixPsf=True, splinesky=True, subsky=False, pixels=False) except: import traceback traceback.print_exc() plvers.append('') continue if tim is None: plvers.append('') continue psf = tim.psf wcs = tim.wcs.wcs sky = tim.sky hdr = tim.primhdr # print('Got PSF', psf) # print('Got sky', type(sky)) # print('Got WCS', wcs) ccds.humidity[iccd] = hdr.get('HUMIDITY') ccds.outtemp[iccd] = hdr.get('OUTTEMP') ccds.sig1[iccd] = tim.sig1 plvers.append(tim.plver) obj = hdr.get('OBJECT') # parse 'DECaLS_15150_r' words = obj.split('_') tile = None if len(words) == 3 and words[0] == 'DECaLS': try: tileid = int(words[1]) tile = tiles[tileid - 1] if tile.tileid != tileid: I = np.flatnonzero(tile.tileid == tileid) tile = tiles[I[0]] except: pass if tile is not None: ccds.tileid [iccd] = tile.tileid ccds.tilepass[iccd] = tile.get('pass') ccds.tileebv [iccd] = tile.ebv_med # Instantiate PSF on a grid S = 32 xx = np.linspace(1+S, W-S, 5) yy = np.linspace(1+S, H-S, 5) xx,yy = np.meshgrid(xx, yy) psfnorms = [] galnorms = [] for x,y in zip(xx.ravel(), yy.ravel()): p = im.psf_norm(tim, x=x, y=y) g = im.galaxy_norm(tim, x=x, y=y) psfnorms.append(p) galnorms.append(g) ccds.psfnorm_mean[iccd] = np.mean(psfnorms) ccds.psfnorm_std [iccd] = np.std (psfnorms) ccds.galnorm_mean[iccd] = np.mean(galnorms) ccds.galnorm_std [iccd] = np.std (galnorms) # PSF in center of field cx,cy = (W+1)/2., (H+1)/2. p = psf.getPointSourcePatch(cx, cy).patch ph,pw = p.shape px,py = np.meshgrid(np.arange(pw), np.arange(ph)) psum = np.sum(p) # print('psum', psum) p /= psum # centroids cenx = np.sum(p * px) ceny = np.sum(p * py) # print('cenx,ceny', cenx,ceny) # second moments x2 = np.sum(p * (px - cenx)**2) y2 = np.sum(p * (py - ceny)**2) xy = np.sum(p * (px - cenx)*(py - ceny)) # semi-major/minor axes and position angle theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.) theta = np.abs(theta) * np.sign(xy) s = np.sqrt(((x2 - y2)/2.)**2 + xy**2) a = np.sqrt((x2 + y2) / 2. + s) b = np.sqrt((x2 + y2) / 2. - s) ell = 1. - b/a # print('PSF second moments', x2, y2, xy) # print('PSF position angle', theta) # print('PSF semi-axes', a, b) # print('PSF ellipticity', ell) ccds.psf_mx2[iccd] = x2 ccds.psf_my2[iccd] = y2 ccds.psf_mxy[iccd] = xy ccds.psf_a[iccd] = a ccds.psf_b[iccd] = b ccds.psf_theta[iccd] = theta ccds.psf_ell [iccd] = ell # Galaxy norm using Gaussian approximation of PSF. realpsf = tim.psf tim.psf = im.read_psf_model(0, 0, gaussPsf=True, psf_sigma=tim.psf_sigma) gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy) tim.psf = realpsf # Sky mod = np.zeros((ccd.height, ccd.width), np.float32) sky.addTo(mod) ccds.meansky[iccd] = np.mean(mod) ccds.stdsky[iccd] = np.std(mod) ccds.maxsky[iccd] = mod.max() ccds.minsky[iccd] = mod.min() # WCS ccds.ra0[iccd],ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1) ccds.ra1[iccd],ccds.dec1[iccd] = wcs.pixelxy2radec(1, H) ccds.ra2[iccd],ccds.dec2[iccd] = wcs.pixelxy2radec(W, H) ccds.ra3[iccd],ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1) midx, midy = (W+1)/2., (H+1)/2. rc,dc = wcs.pixelxy2radec(midx, midy) ra,dec = wcs.pixelxy2radec([1,W,midx,midx], [midy,midy,1,H]) ccds.dra [iccd] = max(degrees_between(ra, dc+np.zeros_like(ra), rc, dc)) ccds.ddec[iccd] = max(degrees_between(rc+np.zeros_like(dec), dec, rc, dc)) ccds.ra_center [iccd] = rc ccds.dec_center[iccd] = dc # Compute scale change across the chip # how many pixels to step step = 10 xx = np.linspace(1+step, W-step, 5) yy = np.linspace(1+step, H-step, 5) xx,yy = np.meshgrid(xx, yy) pixscale = [] for x,y in zip(xx.ravel(), yy.ravel()): sx = [x-step, x-step, x+step, x+step, x-step] sy = [y-step, y+step, y+step, y-step, y-step] sr,sd = wcs.pixelxy2radec(sx, sy) rc,dc = wcs.pixelxy2radec(x, y) # project around a tiny little TAN WCS at (x,y), with 1" pixels locwcs = Tan(rc, dc, 0., 0., 1./3600, 0., 0., 1./3600, 1., 1.) ok,lx,ly = locwcs.radec2pixelxy(sr, sd) #print('local x,y:', lx, ly) A = polygon_area((lx, ly)) pixscale.append(np.sqrt(A / (2*step)**2)) # print('Pixel scales:', pixscale) ccds.pixscale_mean[iccd] = np.mean(pixscale) ccds.pixscale_min[iccd] = min(pixscale) ccds.pixscale_max[iccd] = max(pixscale) ccds.pixscale_std[iccd] = np.std(pixscale) ccds.plver = np.array(plvers) sfd = tractor.sfd.SFDMap() allbands = 'ugrizY' filts = ['%s %s' % ('DES', f) for f in allbands] wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4'] ebv,ext = sfd.extinction(filts + wisebands, ccds.ra_center, ccds.dec_center, get_ebv=True) ext = ext.astype(np.float32) ccds.ebv = ebv.astype(np.float32) ccds.decam_extinction = ext[:,:len(allbands)] ccds.wise_extinction = ext[:,len(allbands):] # Depth detsig1 = ccds.sig1 / ccds.psfnorm_mean depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.psfdepth = -2.5 * (np.log10(depth) - 9) detsig1 = ccds.sig1 / ccds.galnorm_mean depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.galdepth = -2.5 * (np.log10(depth) - 9) # Depth using Gaussian FWHM. psf_sigma = ccds.fwhm / 2.35 gnorm = 1./(2. * np.sqrt(np.pi) * psf_sigma) detsig1 = ccds.sig1 / gnorm depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9) # Gaussian galaxy depth detsig1 = ccds.sig1 / gaussgalnorm depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9) ccds.writeto(outfn)
main(outfn=outfn, ccds=ccds) except: import traceback traceback.print_exc() if __name__ == '__main__': TT = [fits_table('ccds-annotated/ccds-annotated-%03i.fits' % i) for i in range(515)] T = merge_tables(TT) T.writeto('ccds-annotated.fits') import sys sys.exit() #sys.exit(main()) decals = Decals() ccds = decals.get_ccds() from astrometry.util.multiproc import * #mp = multiproc(8) mp = multiproc(4) N = 1000 args = [] i = 0 while len(ccds): c = ccds[:N] ccds = ccds[N:] args.append((i, c)) i += 1 print('Split CCDs file into', len(args), 'pieces') print('sizes:', [len(c) for i,c in args]) mp.map(_bounce_main, args)
def main(decals=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0, x1, y0, y1) = opt.zoom zoomslice = (slice(y0, y1), slice(x0, x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for decals.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if decals is None: decals = Decals() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from decals-ccds.fits table T = decals.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut( np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert (len(T) == 1) im = decals.get_image_object(T[0]) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True) print('Got tim:', tim) if opt.catfn in ['DR1', 'DR2']: if opt.catalog_path is None: opt.catalog_path = opt.catfn.lower() margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, decals=decals) for b in bricks: # there is some overlap with this brick... read the catalog. fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3], 'tractor-%s.fits' % b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec) W, H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) * (yy >= -margin) * (yy <= (H + margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') TT.append(T) T = merge_tables(TT) T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE) # print('Got cat:', cat) print('Forced photom...') opti = None if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)) ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x - 1).astype(np.float32) F.y = (y - 1).astype(np.float32) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack( [tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap ap = 1. / (np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap if opt.forced: kwa = {} if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa) if opt.plots: (data, mod, ie, chi, roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([ src.getBrightness().getFlux(tim.band) for src in cat ]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2.astype(np.float32) program_name = sys.argv[0] version_hdr = get_version_header(program_name, decals.decals_dir) # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) fname = os.path.join(d2, d1, fname) print('Trimmed filename to', fname) #version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file')) version_hdr.add_record( dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file')) version_hdr.add_record( dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext')) version_hdr.add_record( dict(name='CAMERA', value='DECam', comment='Dark Energy Camera')) version_hdr.add_record( dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num')) version_hdr.add_record( dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name')) version_hdr.add_record( dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record( dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image')) keys = [ 'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME' ] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = { 'mjd': 'sec', 'exptime': 'sec', 'flux': 'nanomaggy', 'flux_ivar': '1/nanomaggy^2' } columns = F.get_columns() for i, col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) print('Finished forced phot:', Time() - t0) return 0
def main(): import optparse parser = optparse.OptionParser() parser.add_option('--zoom', '-z', type=int, action='append', default=[], help='Add zoom level; default 13') parser.add_option('--threads', type=int, default=1, help='Number of threads') parser.add_option('--y0', type=int, default=0, help='Start row') parser.add_option('--y1', type=int, default=None, help='End row (non-inclusive)') parser.add_option('--x0', type=int, default=None) parser.add_option('--x1', type=int, default=None) parser.add_option('-x', type=int) parser.add_option('-y', type=int) parser.add_option('--mindec', type=float, default=None, help='Minimum Dec to run') parser.add_option('--maxdec', type=float, default=None, help='Maximum Dec to run') parser.add_option('--minra', type=float, default=0., help='Minimum RA to run') parser.add_option('--maxra', type=float, default=360., help='Maximum RA to run') parser.add_option('--near', action='store_true', help='Only run tiles near bricks') parser.add_option('--near-ccds', action='store_true', help='Only run tiles near CCDs') parser.add_option('--queue', action='store_true', default=False, help='Print qdo commands') parser.add_option('--all', action='store_true', help='Render all tiles') parser.add_option('--ignore', action='store_true', help='Ignore cached tile files', default=False) parser.add_option('--top', action='store_true', help='Top levels of the pyramid') parser.add_option('--kind', default='image') parser.add_option('--scale', action='store_true', help='Scale images?') parser.add_option('--coadd', action='store_true', help='Create SDSS coadd images?') parser.add_option('--grass', action='store_true', help='progress plot') opt,args = parser.parse_args() if len(opt.zoom) == 0: opt.zoom = [13] mp = multiproc(opt.threads) if opt.kind == 'sdss': if opt.maxdec is None: opt.maxdec = 90 if opt.mindec is None: opt.mindec = -25 elif opt.kind in ['halpha', 'unwise-neo1']: if opt.maxdec is None: opt.maxdec = 90 if opt.mindec is None: opt.mindec = -90 else: if opt.maxdec is None: opt.maxdec = 40 if opt.mindec is None: opt.mindec = -20 if opt.top: top_levels(mp, opt) sys.exit(0) from legacypipe.common import Decals decals = Decals() if opt.near: if opt.kind == 'sdss': B = fits_table(os.path.join(settings.DATA_DIR, 'bricks-sdssco.fits')) else: B = decals.get_bricks() print len(B), 'bricks' if opt.scale: opt.near_ccds = True if opt.near_ccds: if opt.kind == 'sdss': C = fits_table(os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'), columns=['rerun','ra','dec', 'run', 'camcol', 'field', 'score']) C.cut(C.rerun == '301') C.cut(C.score >= 0.6) #C.delete_column('rerun') # SDSS field size radius = 1.01 * np.hypot(10., 14.)/2. / 60. ccdsize = radius print len(C), 'SDSS fields' else: C = decals.get_ccds() print len(C), 'CCDs' ccdsize = 0.2 if opt.x is not None: opt.x0 = opt.x opt.x1 = opt.x + 1 if opt.y is not None: opt.y0 = opt.y opt.y1 = opt.y + 1 if opt.coadd and opt.kind == 'sdss': from legacypipe.common import wcs_for_brick from map.views import trymakedirs B = decals.get_bricks() print len(B), 'bricks' B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec)) print len(B), 'in Dec range' B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra)) print len(B), 'in RA range' if opt.queue: # ~ square-degree tiles # RA slices rr = np.arange(opt.minra , opt.maxra +1) dd = np.arange(opt.mindec, opt.maxdec+1) for rlo,rhi in zip(rr, rr[1:]): for dlo,dhi in zip(dd, dd[1:]): print 'time python render-tiles.py --kind sdss --coadd --minra %f --maxra %f --mindec %f --maxdec %f' % (rlo, rhi, dlo, dhi) sys.exit(0) if opt.grass: basedir = settings.DATA_DIR codir = os.path.join(basedir, 'coadd', 'sdssco') rr,dd = [],[] exist = [] for i,b in enumerate(B): print 'Brick', b.brickname, fn = os.path.join(codir, b.brickname[:3], 'sdssco-%s-%s.fits' % (b.brickname, 'r')) print '-->', fn, if not os.path.exists(fn): print continue print 'found' rr.append(b.ra) dd.append(b.dec) exist.append(i) exist = np.array(exist) B.cut(exist) B.writeto('bricks-sdssco-exist.fits') import pylab as plt plt.clf() plt.plot(rr, dd, 'k.') plt.title('SDSS coadd tiles') plt.savefig('sdss.png') sys.exit(0) basedir = settings.DATA_DIR codir = os.path.join(basedir, 'coadd', 'sdssco') for b in B: print 'Brick', b.brickname wcs = wcs_for_brick(b, W=2400, H=2400, pixscale=0.396) bands = 'gri' dirnm = os.path.join(codir, b.brickname[:3]) fns = [os.path.join(dirnm, 'sdssco-%s-%s.fits' % (b.brickname, band)) for band in bands] hdr = fitsio.FITSHDR() hdr['SURVEY'] = 'SDSS' wcs.add_to_header(hdr) if all([os.path.exists(fn) for fn in fns]): print 'Already exist' continue ims = map_sdss(req, 1, 0, 0, 0, get_images=True, wcs=wcs, ignoreCached=True, forcescale=0) if ims is None: print 'No overlap' continue trymakedirs(os.path.join(dirnm, 'xxx')) for fn,band,im in zip(fns,bands, ims): fitsio.write(fn, im, header=hdr, clobber=True) print 'Wrote', fn # Also write scaled versions dirnm = os.path.join(basedir, 'scaled', 'sdssco') scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(brickname).3s', 'sdssco-%(brickname)s-%(band)s.fits') for im,band in zip(ims,bands): scalekwargs = dict(band=band, brick=b.brickid, brickname=b.brickname) imwcs = wcs for scale in range(1, 7): print 'Writing scale level', scale im,imwcs,sfn = get_scaled(scalepat, scalekwargs, scale, None, wcs=imwcs, img=im, return_data=True) sys.exit(0) if opt.scale: if opt.kind == 'sdss': C.cut((C.dec >= opt.mindec) * (C.dec < opt.maxdec)) print len(C), 'in Dec range' C.cut((C.ra >= opt.minra) * (C.ra < opt.maxra)) print len(C), 'in RA range' from astrometry.sdss import AsTransWrapper, DR9 sdss = DR9(basedir=settings.SDSS_DIR) sdss.saveUnzippedFiles(settings.SDSS_DIR) sdss.setFitsioReadBZ2() if settings.SDSS_PHOTOOBJS: sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS, resolve=settings.SDSS_RESOLVE) basedir = settings.DATA_DIR scaledir = 'sdss' dirnm = os.path.join(basedir, 'scaled', scaledir) for im in C: from map.views import _read_sip_wcs #if im.rerun != '301': # continue for band in 'gri': scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i', 'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits') tmpsuff = '.tmp%08i' % np.random.randint(100000000) basefn = sdss.retrieve('frame', im.run, im.camcol, field=im.field, band=band, rerun=im.rerun, tempsuffix=tmpsuff) fnargs = dict(band=band, rerun=im.rerun, run=im.run, camcol=im.camcol, field=im.field) scaled = 1 fn = get_scaled(scalepat, fnargs, scaled, basefn, read_base_wcs=read_astrans, read_wcs=_read_sip_wcs) print 'get_scaled:', fn return 0 else: assert(False) for zoom in opt.zoom: N = 2**zoom if opt.y1 is None: y1 = N else: y1 = opt.y1 if opt.x0 is None: opt.x0 = 0 x1 = opt.x1 if x1 is None: x1 = N # Find grid of Ra,Dec tile centers and select the ones near DECaLS bricks. rr,dd = [],[] yy = np.arange(opt.y0, y1) xx = np.arange(opt.x0, x1) if opt.grass: import pylab as plt tileexists = np.zeros((len(yy),len(xx)), bool) basedir = settings.DATA_DIR ver = tileversions[opt.kind][-1] tiledir = os.path.join(basedir, 'tiles', opt.kind, '%i'%ver, '%i'%zoom) for dirpath,dirnames,filenames in os.walk(tiledir): # change walk order dirnames.sort() if len(filenames) == 0: continue print 'Dirpath', dirpath #print 'Dirnames', dirnames #print 'Filenames', filenames # check for symlinks if False: fns = [] for fn in filenames: fullfn = os.path.join(tiledir, dirpath, fn) if os.path.isfile(fullfn) and not os.path.islink(fullfn): fns.append(fn) print len(fns), 'of', len(filenames), 'are files (not symlinks)' filenames = fns x = os.path.basename(dirpath) x = int(x) #print 'x', x yy = [int(fn.replace('.jpg','')) for fn in filenames] #print 'yy', yy print len(yy), 'tiles' for y in yy: tileexists[y - opt.y0, x - opt.x0] = True plt.clf() plt.imshow(tileexists, interpolation='nearest', origin='upper', vmin=0, vmax=1, cmap='gray') fn = 'exist-%s-z%02i' % (opt.kind, zoom) plt.savefig(fn+'.png') fitsio.write(fn+'.fits', tileexists, clobber=True) print 'Wrote', fn+'.png and', fn+'.fits' continue if not opt.all: for y in yy: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, y) r,d = wcs.get_center() dd.append(d) for x in xx: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, x, 0) r,d = wcs.get_center() rr.append(r) dd = np.array(dd) rr = np.array(rr) if len(dd) > 1: tilesize = max(np.abs(np.diff(dd))) print 'Tile size:', tilesize else: if opt.near_ccds or opt.near: try: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0+1) r2,d2 = wcs.get_center() except: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0-1) r2,d2 = wcs.get_center() tilesize = np.abs(dd[0] - d2) print 'Tile size:', tilesize else: tilesize = 180. I = np.flatnonzero((dd >= opt.mindec) * (dd <= opt.maxdec)) print 'Keeping', len(I), 'Dec points between', opt.mindec, 'and', opt.maxdec dd = dd[I] yy = yy[I] I = np.flatnonzero((rr >= opt.minra) * (rr <= opt.maxra)) print 'Keeping', len(I), 'RA points between', opt.minra, 'and', opt.maxra rr = rr[I] xx = xx[I] print len(rr), 'RA points x', len(dd), 'Dec points' print 'x tile range:', xx.min(), xx.max(), 'y tile range:', yy.min(), yy.max() for iy,y in enumerate(yy): print print 'Y row', y if opt.near: d = dd[iy] I,J,dist = match_radec(rr, d+np.zeros_like(rr), B.ra, B.dec, 0.25 + tilesize, nearest=True) if len(I) == 0: print 'No matches to bricks' continue keep = np.zeros(len(rr), bool) keep[I] = True print 'Keeping', sum(keep), 'tiles in row', y, 'Dec', d x = xx[keep] elif opt.near_ccds: d = dd[iy] print 'RA range of tiles:', rr.min(), rr.max() print 'Dec of tile row:', d I,J,dist = match_radec(rr, d+np.zeros_like(rr), C.ra, C.dec, ccdsize + tilesize, nearest=True) if len(I) == 0: print 'No matches to CCDs' continue keep = np.zeros(len(rr), bool) keep[I] = True print 'Keeping', sum(keep), 'tiles in row', y, 'Dec', d x = xx[keep] else: x = xx if opt.queue: cmd = 'python -u render-tiles.py --zoom %i --y0 %i --y1 %i --kind %s' % (zoom, y, y+1, opt.kind) if opt.near_ccds: cmd += ' --near-ccds' if opt.all: cmd += ' --all' if opt.ignore: cmd += ' --ignore' print cmd continue # if opt.grass: # for xi in x: # basedir = settings.DATA_DIR # ver = tileversions[opt.kind][-1] # tilefn = os.path.join(basedir, 'tiles', opt.kind, # '%i/%i/%i/%i.jpg' % (ver, zoom, xi, y)) # print 'Checking for', tilefn # if os.path.exists(tilefn): # print 'EXISTS' # tileexists[yi-opt.y0, xi-opt.x0] # continue args = [] for xi in x: args.append((opt.kind,zoom,xi,y, opt.ignore)) print 'Rendering', len(args), 'tiles in row y =', y mp.map(_bounce_one_tile, args, chunksize=min(100, max(1, len(args)/opt.threads))) print 'Rendered', len(args), 'tiles'
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).") parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') opt = parser.parse_args() decals = Decals() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = decals.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = decals.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo,dhi = -15, 40 rlo,rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict(test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo,dhi = -90,90 rlo,rhi = 0, 360 elif opt.region == 'badsky': # A handful of exposures in DR2 with inconsistent sky estimates. #C = decals.get_ccds() #log(len(C), 'CCDs') #C.cut((C.expnum >= 257400) * (C.expnum < 257500)) T.cut(np.array([e in [257460, 257461, 257462, 257463, 257464, 257465, 257466, 257467, 257469, 257472, 257483, 257496] for e in T.expnum])) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25)/2. I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'badsky2': # UGH, missed this one in original 'badsky' definition. T.cut(T.expnum == 257466) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25)/2. I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo,rhi = 240,245 dlo,dhi = 5, 12 elif opt.region == 'edr-south': rlo,rhi = 240,245 dlo,dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo,rhi = 149.75, 150.75 dlo,dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo,rhi = 240,250 dlo,dhi = 10,15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo,rhi = 320., 360. dlo,dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo,rhi = 240,360 dlo,dhi = 20,40 elif opt.region == 'north': rlo,rhi = 120,240 dlo,dhi = 20,40 elif opt.region == 'northeast': rlo,rhi = 0,120 dlo,dhi = 20,40 elif opt.region == 'southwest': rlo,rhi = 240,360 dlo,dhi = -20,0 elif opt.region == 'south': rlo,rhi = 120,240 dlo,dhi = -20,0 elif opt.region == 'southeast': rlo,rhi = 0,120 dlo,dhi = -20,0 elif opt.region == 'midwest': rlo,rhi = 240,360 dlo,dhi = 0,20 elif opt.region == 'middle': rlo,rhi = 120,240 dlo,dhi = 0,20 elif opt.region == 'mideast': rlo,rhi = 0,120 dlo,dhi = 0,20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo,rhi = 250,260 dlo,dhi = 30,35 elif opt.region == 'virgo': rlo,rhi = 185,190 dlo,dhi = 10, 15 elif opt.region == 'virgo2': rlo,rhi = 182,192 dlo,dhi = 8, 18 elif opt.region == 'lsb': rlo,rhi = 147.2, 147.8 dlo,dhi = -0.4, 0.4 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, 0.25) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('decals-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing B.cut(np.argsort(-B.dec)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue print(b.brickname) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): expstr = '%08i' % T.expnum[i] #outdir = os.path.join('forced', expstr[:5], expstr) #trymakedirs(outdir) outfn = os.path.join('forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(decals.decals_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] f.write('python legacypipe/forced-photom-decam.py %s %i DR1 %s\n' % (imgfn, T.cpimage_hdu[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if opt.command: s = '; '.join(batch) else: s = ' '.join(batch) f.write(s + '\n') for j,i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j+1, 'of', len(allI)) im = decals.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j+1, 'of', len(allI)) im = decals.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = ('python legacypipe/run-calib.py --expnum %i --ccdname %s' % (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] if not opt.nper: f.write(s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, opt.command) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, opt.command) f.close() log('Wrote', opt.out) return 0
def main(): """Main routine which parses the optional inputs.""" parser = argparse.ArgumentParser(formatter_class=argparse. ArgumentDefaultsHelpFormatter, description='DECaLS simulations.') parser.add_argument('-n', '--nobj', type=long, default=None, metavar='', help='number of objects to simulate (required input)') parser.add_argument('-c', '--chunksize', type=long, default=500, metavar='', help='divide NOBJ into CHUNKSIZE chunks') parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='', help='simulate objects in this brick') parser.add_argument('-o', '--objtype', type=str, default='STAR', metavar='', help='object type (STAR, ELG, LRG, QSO, LSB)') parser.add_argument('-t', '--threads', type=int, default=8, metavar='', help='number of threads to use when calling The Tractor') parser.add_argument('-s', '--seed', type=long, default=None, metavar='', help='random number seed') parser.add_argument('-z', '--zoom', nargs=4, type=int, metavar='', help='see runbrick.py; (default is 0 3600 0 3600)') parser.add_argument('--rmag-range', nargs=2, type=float, default=(18,25), metavar='', help='r-band magnitude range') parser.add_argument('--test', action='store_true', help='run the test code and write out test images') parser.add_argument('-v', '--verbose', action='store_true', help='toggle on verbose output') args = parser.parse_args() if args.nobj is None: parser.print_help() sys.exit(1) # Set the debugging level if args.verbose: lvl = logging.DEBUG else: lvl = logging.INFO logging.basicConfig(format='%(message)s',level=lvl,stream=sys.stdout) log = logging.getLogger('__name__') brickname = args.brick objtype = args.objtype.upper() lobjtype = objtype.lower() if objtype=='LRG': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) elif objtype=='LSB': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) elif objtype=='QSO': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) # Determine how many chunks we need nobj = args.nobj chunksize = args.chunksize nchunk = long(np.ceil(nobj/chunksize)) log.info('Object type = {}'.format(objtype)) log.info('Number of objects = {}'.format(nobj)) log.info('Chunksize = {}'.format(chunksize)) log.info('Number of chunks = {}'.format(nchunk)) # Optionally zoom into a portion of the brick decals = Decals() brickinfo = decals.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brickinfo) W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale() print(W, H, pixscale) log.info('Brick = {}'.format(brickname)) if args.zoom is not None: # See also runbrick.stage_tims() (x0,x1,y0,y1) = args.zoom W = x1-x0 H = y1-y0 brickwcs = brickwcs.get_subimage(x0, y0, W, H) log.info('Zoom (pixel boundaries) = {}'.format(args.zoom)) targetrd = np.array([brickwcs.pixelxy2radec(x,y) for x,y in [(1,1),(W,1),(W,H),(1,H),(1,1)]]) radec_center = brickwcs.radec_center() log.info('RA, Dec center = {}'.format(radec_center)) log.info('Brick = {}'.format(brickname)) if args.seed is not None: log.info('Random seed = {}'.format(args.seed)) # Pack the input parameters into a meta-data table. metacat = Table() metacat['brickname'] = Column([brickname],dtype='S10') metacat['objtype'] = Column([objtype],dtype='S10') if args.seed is not None: metacat['seed'] = Column([args.seed],dtype='i4') metacat['nobj'] = Column([args.nobj],dtype='i4') metacat['chunksize'] = Column([args.chunksize],dtype='i2') metacat['nchunk'] = Column([nchunk],dtype='i2') #metacat['RA'] = Column([ra_range],dtype='f8') #metacat['DEC'] = Column([dec_range],dtype='f8') if args.zoom is None: metacat['zoom'] = Column([[0,3600,0,3600]],dtype='i4') else: metacat['zoom'] = Column([args.zoom],dtype='i4') metacat['rmag_range'] = Column([args.rmag_range],dtype='f4') print('Hack!') decals_sim_dir = './' metafile = os.path.join(decals_sim_dir,'metacat-'+brickname+'-'+lobjtype+'.fits') log.info('Writing {}'.format(metafile)) if os.path.isfile(metafile): os.remove(metafile) metacat.write(metafile) # Work in chunks. for ichunk in range(nchunk): log.info('Working on chunk {:02d}/{:02d}'.format(ichunk+1,nchunk)) chunksuffix = '{:02d}'.format(ichunk) # There's probably a smarter way to do this nobjchunk = np.min((nobj,chunksize)) if (nchunk>1) and (ichunk==(nchunk-1)): nobjchunk = np.max((nobjchunk,nobj%((nchunk-1)*chunksize))) # Build and write out the simulated object catalog. simcat = build_simcat(nobjchunk,brickname,brickwcs,metacat) simcatfile = os.path.join(decals_sim_dir,'simcat-'+brickname+'-'+ lobjtype+'-'+chunksuffix+'.fits') log.info('Writing {}'.format(simcatfile)) if os.path.isfile(simcatfile): os.remove(simcatfile) simcat.write(simcatfile) # # Use Tractor to just process the blobs containing the simulated sources. if args.test: # Test code simdecals = SimDecals(metacat=metacat, simcat=simcat, test=True) ccdinfo = decals.ccds_touching_wcs(brickwcs) sim = SimImage(simdecals, ccdinfo[19]) tim = sim.get_tractor_image(const2psf=True, radecpoly=targetrd) #for ii, ccd in enumerate(ccdinfo): # log.info('Working on CCD {}'.format(ii)) # sim = SimImage(simdecals,ccd) # tim = sim.get_tractor_image(const2psf=True, radecpoly=targetrd, test=args.test) else: simdecals = SimDecals(metacat=metacat,simcat=simcat) blobxy = zip(simcat['x'],simcat['y']) run_brick(brickname, decals=simdecals, outdir=decals_sim_dir, threads=8, zoom=args.zoom, wise=False, sdssInit=False, forceAll=True, writePickles=False, do_calibs=False, write_metrics=False, pixPsf=False, blobxy=blobxy, early_coadds=False, stages=['writecat'])
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser( description= "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)." ) parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') opt = parser.parse_args() decals = Decals() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = decals.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = decals.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo, dhi = -15, 40 rlo, rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict( test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo, dhi = -90, 90 rlo, rhi = 0, 360 elif opt.region == 'badsky': # A handful of exposures in DR2 with inconsistent sky estimates. #C = decals.get_ccds() #log(len(C), 'CCDs') #C.cut((C.expnum >= 257400) * (C.expnum < 257500)) T.cut( np.array([ e in [ 257460, 257461, 257462, 257463, 257464, 257465, 257466, 257467, 257469, 257472, 257483, 257496 ] for e in T.expnum ])) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25) / 2. I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'badsky2': # UGH, missed this one in original 'badsky' definition. T.cut(T.expnum == 257466) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25) / 2. I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo, rhi = 240, 245 dlo, dhi = 5, 12 elif opt.region == 'edr-south': rlo, rhi = 240, 245 dlo, dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo, rhi = 149.75, 150.75 dlo, dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo, rhi = 240, 250 dlo, dhi = 10, 15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo, rhi = 320., 360. dlo, dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo, rhi = 240, 360 dlo, dhi = 20, 40 elif opt.region == 'north': rlo, rhi = 120, 240 dlo, dhi = 20, 40 elif opt.region == 'northeast': rlo, rhi = 0, 120 dlo, dhi = 20, 40 elif opt.region == 'southwest': rlo, rhi = 240, 360 dlo, dhi = -20, 0 elif opt.region == 'south': rlo, rhi = 120, 240 dlo, dhi = -20, 0 elif opt.region == 'southeast': rlo, rhi = 0, 120 dlo, dhi = -20, 0 elif opt.region == 'midwest': rlo, rhi = 240, 360 dlo, dhi = 0, 20 elif opt.region == 'middle': rlo, rhi = 120, 240 dlo, dhi = 0, 20 elif opt.region == 'mideast': rlo, rhi = 0, 120 dlo, dhi = 0, 20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo, rhi = 250, 260 dlo, dhi = 30, 35 elif opt.region == 'virgo': rlo, rhi = 185, 190 dlo, dhi = 10, 15 elif opt.region == 'virgo2': rlo, rhi = 182, 192 dlo, dhi = 8, 18 elif opt.region == 'lsb': rlo, rhi = 147.2, 147.8 dlo, dhi = -0.4, 0.4 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut( np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, 0.25) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('decals-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing B.cut(np.argsort(-B.dec)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % ( b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue print(b.brickname) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): expstr = '%08i' % T.expnum[i] #outdir = os.path.join('forced', expstr[:5], expstr) #trymakedirs(outdir) outfn = os.path.join( 'forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(decals.decals_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] f.write('python legacypipe/forced-photom-decam.py %s %i DR1 %s\n' % (imgfn, T.cpimage_hdu[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write( 'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if opt.command: s = '; '.join(batch) else: s = ' '.join(batch) f.write(s + '\n') for j, i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j + 1, 'of', len(allI)) im = decals.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j + 1, 'of', len(allI)) im = decals.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = ('python legacypipe/run-calib.py --expnum %i --ccdname %s' % (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] if not opt.nper: f.write(s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, opt.command) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, opt.command) f.close() log('Wrote', opt.out) return 0
def main(): """Main routine which parses the optional inputs.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='DECaLS simulations.') parser.add_argument('-n', '--nobj', type=long, default=None, metavar='', help='number of objects to simulate (required input)') parser.add_argument('-c', '--chunksize', type=long, default=500, metavar='', help='divide NOBJ into CHUNKSIZE chunks') parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='', help='simulate objects in this brick') parser.add_argument('-o', '--objtype', type=str, default='STAR', metavar='', help='object type (STAR, ELG, LRG, QSO, LSB)') parser.add_argument( '-t', '--threads', type=int, default=8, metavar='', help='number of threads to use when calling The Tractor') parser.add_argument('-s', '--seed', type=long, default=None, metavar='', help='random number seed') parser.add_argument('-z', '--zoom', nargs=4, type=int, metavar='', help='see runbrick.py; (default is 0 3600 0 3600)') parser.add_argument('--rmag-range', nargs=2, type=float, default=(18, 26), metavar='', help='r-band magnitude range') parser.add_argument('-v', '--verbose', action='store_true', help='toggle on verbose output') args = parser.parse_args() if args.nobj is None: parser.print_help() sys.exit(1) # Set the debugging level if args.verbose: lvl = logging.DEBUG else: lvl = logging.INFO logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout) log = logging.getLogger('__name__') brickname = args.brick objtype = args.objtype.upper() lobjtype = objtype.lower() if objtype == 'LRG': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) elif objtype == 'LSB': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) elif objtype == 'QSO': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) # Determine how many chunks we need nobj = args.nobj chunksize = args.chunksize nchunk = long(np.ceil(nobj / chunksize)) log.info('Object type = {}'.format(objtype)) log.info('Number of objects = {}'.format(nobj)) log.info('Chunksize = {}'.format(chunksize)) log.info('Number of chunks = {}'.format(nchunk)) # Optionally zoom into a portion of the brick decals = Decals() brickinfo = decals.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brickinfo) W, H, pixscale = brickwcs.get_width(), brickwcs.get_height( ), brickwcs.pixel_scale() print(W, H, pixscale) log.info('Brick = {}'.format(brickname)) if args.zoom is not None: # See also runbrick.stage_tims() (x0, x1, y0, y1) = args.zoom W = x1 - x0 H = y1 - y0 brickwcs = brickwcs.get_subimage(x0, y0, W, H) log.info('Zoom (pixel boundaries) = {}'.format(args.zoom)) targetrd = np.array([ brickwcs.pixelxy2radec(x, y) for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)] ]) radec_center = brickwcs.radec_center() log.info('RA, Dec center = {}'.format(radec_center)) log.info('Brick = {}'.format(brickname)) # Pack the input parameters into a meta-data table. metacat = Table() metacat['brickname'] = Column([brickname], dtype='S10') metacat['objtype'] = Column([objtype], dtype='S10') metacat['nobj'] = Column([args.nobj], dtype='i4') metacat['chunksize'] = Column([args.chunksize], dtype='i2') metacat['nchunk'] = Column([nchunk], dtype='i2') #metacat['RA'] = Column([ra_range],dtype='f8') #metacat['DEC'] = Column([dec_range],dtype='f8') if args.zoom is None: metacat['zoom'] = Column([[0, 3600, 0, 3600]], dtype='i4') else: metacat['zoom'] = Column([args.zoom], dtype='i4') metacat['rmag_range'] = Column([args.rmag_range], dtype='f4') # Deal with the random seed if args.seed is not None: log.info('Random seed = {}'.format(args.seed)) metacat['seed'] = Column([args.seed], dtype='i4') rand = np.random.RandomState(args.seed) seeds = rand.random_integers(0, 2**18, nchunk) if 'DECALS_SIM_DIR' in os.environ: decals_sim_dir = os.getenv('DECALS_SIM_DIR') else: decals_sim_dir = '.' if not os.path.exists(os.path.join(decals_sim_dir, brickname)): os.makedirs(os.path.join(decals_sim_dir, brickname)) metafile = os.path.join(decals_sim_dir, brickname, 'metacat-' + brickname + '-' + lobjtype + '.fits') log.info('Writing {}'.format(metafile)) if os.path.isfile(metafile): os.remove(metafile) metacat.write(metafile) # Work in chunks. for ichunk in range(nchunk): log.info('Working on chunk {:02d}/{:02d}'.format(ichunk + 1, nchunk)) chunksuffix = '{:02d}'.format(ichunk) # There's probably a smarter way to do this nobjchunk = np.min((nobj, chunksize)) if (nchunk > 1) and (ichunk == (nchunk - 1)): nobjchunk = np.max((nobjchunk, nobj % ((nchunk - 1) * chunksize))) # Build and write out the simulated object catalog. simcat = build_simcat(nobjchunk, brickname, brickwcs, metacat, seeds[ichunk]) simcatfile = os.path.join( decals_sim_dir, brickname, 'simcat-' + brickname + '-' + lobjtype + '-' + chunksuffix + '.fits') log.info('Writing {}'.format(simcatfile)) if os.path.isfile(simcatfile): os.remove(simcatfile) simcat.write(simcatfile) # # Use Tractor to just process the blobs containing the simulated sources. simdecals = SimDecals(metacat=metacat, simcat=simcat) blobxy = zip(simcat['x'], simcat['y']) run_brick(brickname, decals=simdecals, outdir=os.path.join(decals_sim_dir, brickname), threads=args.threads, zoom=args.zoom, wise=False, sdssInit=False, forceAll=True, writePickles=False, do_calibs=True, write_metrics=False, pixPsf=True, blobxy=blobxy, early_coadds=False, stages=['writecat'], splinesky=True) log.info('Cleaning up...') shutil.move( os.path.join(decals_sim_dir, brickname, 'tractor', brickname[:3], 'tractor-' + brickname + '.fits'), os.path.join( decals_sim_dir, brickname, 'tractor-' + brickname + '-' + lobjtype + '-' + chunksuffix + '.fits')) shutil.move( os.path.join(decals_sim_dir, brickname, 'coadd', brickname[:3], brickname, 'decals-' + brickname + '-image.jpg'), os.path.join( decals_sim_dir, brickname, 'qa-' + brickname + '-' + lobjtype + '-image-' + chunksuffix + '.jpg')) shutil.move( os.path.join(decals_sim_dir, brickname, 'coadd', brickname[:3], brickname, 'decals-' + brickname + '-resid.jpg'), os.path.join( decals_sim_dir, brickname, 'qa-' + brickname + '-' + lobjtype + '-resid-' + chunksuffix + '.jpg')) shutil.rmtree(os.path.join(decals_sim_dir, brickname, 'coadd')) shutil.rmtree(os.path.join(decals_sim_dir, brickname, 'tractor')) #shutil.rmtree(os.path.join(decals_sim_dir,brickname,'images')) #shutil.rmtree(os.path.join(decals_sim_dir,brickname,'metrics')) # Write a log file log.info('All done!')
def main(outfn='ccds-annotated.fits', ccds=None): decals = Decals() if ccds is None: ccds = decals.get_ccds() # File from the "observing" svn repo: # https://desi.lbl.gov/svn/decam/code/observing/trunk tiles = fits_table('decam-tiles_obstatus.fits') #ccds.cut(np.arange(100)) #print("HACK!") #ccds.cut(np.array([name in ['N15', 'N16', 'N21', 'N9'] # for name in ccds.ccdname]) * # ccds.expnum == 229683) I = decals.photometric_ccds(ccds) ccds.photometric = np.zeros(len(ccds), bool) ccds.photometric[I] = True I = decals.apply_blacklist(ccds) ccds.blacklist_ok = np.zeros(len(ccds), bool) ccds.blacklist_ok[I] = True ccds.good_region = np.empty((len(ccds), 4), np.int16) ccds.good_region[:, :] = -1 ccds.ra0 = np.zeros(len(ccds), np.float64) ccds.dec0 = np.zeros(len(ccds), np.float64) ccds.ra1 = np.zeros(len(ccds), np.float64) ccds.dec1 = np.zeros(len(ccds), np.float64) ccds.ra2 = np.zeros(len(ccds), np.float64) ccds.dec2 = np.zeros(len(ccds), np.float64) ccds.ra3 = np.zeros(len(ccds), np.float64) ccds.dec3 = np.zeros(len(ccds), np.float64) ccds.dra = np.zeros(len(ccds), np.float32) ccds.ddec = np.zeros(len(ccds), np.float32) ccds.ra_center = np.zeros(len(ccds), np.float64) ccds.dec_center = np.zeros(len(ccds), np.float64) ccds.sig1 = np.zeros(len(ccds), np.float32) ccds.meansky = np.zeros(len(ccds), np.float32) ccds.stdsky = np.zeros(len(ccds), np.float32) ccds.maxsky = np.zeros(len(ccds), np.float32) ccds.minsky = np.zeros(len(ccds), np.float32) ccds.pixscale_mean = np.zeros(len(ccds), np.float32) ccds.pixscale_std = np.zeros(len(ccds), np.float32) ccds.pixscale_max = np.zeros(len(ccds), np.float32) ccds.pixscale_min = np.zeros(len(ccds), np.float32) ccds.psfnorm_mean = np.zeros(len(ccds), np.float32) ccds.psfnorm_std = np.zeros(len(ccds), np.float32) ccds.galnorm_mean = np.zeros(len(ccds), np.float32) ccds.galnorm_std = np.zeros(len(ccds), np.float32) gaussgalnorm = np.zeros(len(ccds), np.float32) # 2nd moments ccds.psf_mx2 = np.zeros(len(ccds), np.float32) ccds.psf_my2 = np.zeros(len(ccds), np.float32) ccds.psf_mxy = np.zeros(len(ccds), np.float32) # ccds.psf_a = np.zeros(len(ccds), np.float32) ccds.psf_b = np.zeros(len(ccds), np.float32) ccds.psf_theta = np.zeros(len(ccds), np.float32) ccds.psf_ell = np.zeros(len(ccds), np.float32) ccds.humidity = np.zeros(len(ccds), np.float32) ccds.outtemp = np.zeros(len(ccds), np.float32) ccds.tileid = np.zeros(len(ccds), np.int32) ccds.tilepass = np.zeros(len(ccds), np.uint8) ccds.tileebv = np.zeros(len(ccds), np.float32) plvers = [] for iccd, ccd in enumerate(ccds): im = decals.get_image_object(ccd) print('Reading CCD %i of %i:' % (iccd + 1, len(ccds)), im) X = im.get_good_image_subregion() for i, x in enumerate(X): if x is not None: ccds.good_region[iccd, i] = x W, H = ccd.width, ccd.height psf = None wcs = None sky = None try: tim = im.get_tractor_image(pixPsf=True, splinesky=True, subsky=False, pixels=False) except: import traceback traceback.print_exc() plvers.append('') continue if tim is None: plvers.append('') continue psf = tim.psf wcs = tim.wcs.wcs sky = tim.sky hdr = tim.primhdr # print('Got PSF', psf) # print('Got sky', type(sky)) # print('Got WCS', wcs) ccds.humidity[iccd] = hdr.get('HUMIDITY') ccds.outtemp[iccd] = hdr.get('OUTTEMP') ccds.sig1[iccd] = tim.sig1 plvers.append(tim.plver) obj = hdr.get('OBJECT') # parse 'DECaLS_15150_r' words = obj.split('_') tile = None if len(words) == 3 and words[0] == 'DECaLS': try: tileid = int(words[1]) tile = tiles[tileid - 1] if tile.tileid != tileid: I = np.flatnonzero(tile.tileid == tileid) tile = tiles[I[0]] except: pass if tile is not None: ccds.tileid[iccd] = tile.tileid ccds.tilepass[iccd] = tile.get('pass') ccds.tileebv[iccd] = tile.ebv_med # Instantiate PSF on a grid S = 32 xx = np.linspace(1 + S, W - S, 5) yy = np.linspace(1 + S, H - S, 5) xx, yy = np.meshgrid(xx, yy) psfnorms = [] galnorms = [] for x, y in zip(xx.ravel(), yy.ravel()): p = im.psf_norm(tim, x=x, y=y) g = im.galaxy_norm(tim, x=x, y=y) psfnorms.append(p) galnorms.append(g) ccds.psfnorm_mean[iccd] = np.mean(psfnorms) ccds.psfnorm_std[iccd] = np.std(psfnorms) ccds.galnorm_mean[iccd] = np.mean(galnorms) ccds.galnorm_std[iccd] = np.std(galnorms) # PSF in center of field cx, cy = (W + 1) / 2., (H + 1) / 2. p = psf.getPointSourcePatch(cx, cy).patch ph, pw = p.shape px, py = np.meshgrid(np.arange(pw), np.arange(ph)) psum = np.sum(p) # print('psum', psum) p /= psum # centroids cenx = np.sum(p * px) ceny = np.sum(p * py) # print('cenx,ceny', cenx,ceny) # second moments x2 = np.sum(p * (px - cenx)**2) y2 = np.sum(p * (py - ceny)**2) xy = np.sum(p * (px - cenx) * (py - ceny)) # semi-major/minor axes and position angle theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.) theta = np.abs(theta) * np.sign(xy) s = np.sqrt(((x2 - y2) / 2.)**2 + xy**2) a = np.sqrt((x2 + y2) / 2. + s) b = np.sqrt((x2 + y2) / 2. - s) ell = 1. - b / a # print('PSF second moments', x2, y2, xy) # print('PSF position angle', theta) # print('PSF semi-axes', a, b) # print('PSF ellipticity', ell) ccds.psf_mx2[iccd] = x2 ccds.psf_my2[iccd] = y2 ccds.psf_mxy[iccd] = xy ccds.psf_a[iccd] = a ccds.psf_b[iccd] = b ccds.psf_theta[iccd] = theta ccds.psf_ell[iccd] = ell # Galaxy norm using Gaussian approximation of PSF. realpsf = tim.psf tim.psf = im.read_psf_model(0, 0, gaussPsf=True, psf_sigma=tim.psf_sigma) gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy) tim.psf = realpsf # Sky mod = np.zeros((ccd.height, ccd.width), np.float32) sky.addTo(mod) ccds.meansky[iccd] = np.mean(mod) ccds.stdsky[iccd] = np.std(mod) ccds.maxsky[iccd] = mod.max() ccds.minsky[iccd] = mod.min() # WCS ccds.ra0[iccd], ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1) ccds.ra1[iccd], ccds.dec1[iccd] = wcs.pixelxy2radec(1, H) ccds.ra2[iccd], ccds.dec2[iccd] = wcs.pixelxy2radec(W, H) ccds.ra3[iccd], ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1) midx, midy = (W + 1) / 2., (H + 1) / 2. rc, dc = wcs.pixelxy2radec(midx, midy) ra, dec = wcs.pixelxy2radec([1, W, midx, midx], [midy, midy, 1, H]) ccds.dra[iccd] = max( degrees_between(ra, dc + np.zeros_like(ra), rc, dc)) ccds.ddec[iccd] = max( degrees_between(rc + np.zeros_like(dec), dec, rc, dc)) ccds.ra_center[iccd] = rc ccds.dec_center[iccd] = dc # Compute scale change across the chip # how many pixels to step step = 10 xx = np.linspace(1 + step, W - step, 5) yy = np.linspace(1 + step, H - step, 5) xx, yy = np.meshgrid(xx, yy) pixscale = [] for x, y in zip(xx.ravel(), yy.ravel()): sx = [x - step, x - step, x + step, x + step, x - step] sy = [y - step, y + step, y + step, y - step, y - step] sr, sd = wcs.pixelxy2radec(sx, sy) rc, dc = wcs.pixelxy2radec(x, y) # project around a tiny little TAN WCS at (x,y), with 1" pixels locwcs = Tan(rc, dc, 0., 0., 1. / 3600, 0., 0., 1. / 3600, 1., 1.) ok, lx, ly = locwcs.radec2pixelxy(sr, sd) #print('local x,y:', lx, ly) A = polygon_area((lx, ly)) pixscale.append(np.sqrt(A / (2 * step)**2)) # print('Pixel scales:', pixscale) ccds.pixscale_mean[iccd] = np.mean(pixscale) ccds.pixscale_min[iccd] = min(pixscale) ccds.pixscale_max[iccd] = max(pixscale) ccds.pixscale_std[iccd] = np.std(pixscale) ccds.plver = np.array(plvers) sfd = tractor.sfd.SFDMap() allbands = 'ugrizY' filts = ['%s %s' % ('DES', f) for f in allbands] wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4'] ebv, ext = sfd.extinction(filts + wisebands, ccds.ra_center, ccds.dec_center, get_ebv=True) ext = ext.astype(np.float32) ccds.ebv = ebv.astype(np.float32) ccds.decam_extinction = ext[:, :len(allbands)] ccds.wise_extinction = ext[:, len(allbands):] # Depth detsig1 = ccds.sig1 / ccds.psfnorm_mean depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.psfdepth = -2.5 * (np.log10(depth) - 9) detsig1 = ccds.sig1 / ccds.galnorm_mean depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.galdepth = -2.5 * (np.log10(depth) - 9) # Depth using Gaussian FWHM. psf_sigma = ccds.fwhm / 2.35 gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) detsig1 = ccds.sig1 / gnorm depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9) # Gaussian galaxy depth detsig1 = ccds.sig1 / gaussgalnorm depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9) ccds.writeto(outfn)
if __name__ == '__main__': TT = [ fits_table('ccds-annotated/ccds-annotated-%03i.fits' % i) for i in range(515) ] T = merge_tables(TT) T.writeto('ccds-annotated.fits') import sys sys.exit() #sys.exit(main()) decals = Decals() ccds = decals.get_ccds() from astrometry.util.multiproc import * #mp = multiproc(8) mp = multiproc(4) N = 1000 args = [] i = 0 while len(ccds): c = ccds[:N] ccds = ccds[N:] args.append((i, c)) i += 1 print('Split CCDs file into', len(args), 'pieces') print('sizes:', [len(c) for i, c in args]) mp.map(_bounce_main, args)
from glob import glob from legacypipe.common import Decals, wcs_for_brick from map.utils import tiles_touching_wcs from map.views import map_decals_dr2 class duck(object): pass req = duck() req.META = dict(HTTP_IF_MODIFIED_SINCE=None) if __name__ == '__main__': decals = Decals() fns = glob('dr2p/coadd/*/*/*-image.jpg') fns.sort() bricknames = [] for fn in fns: print 'File', fn (h, t) = os.path.split(fn) (h, t) = os.path.split(h) print 'Brick:', t brickname = t bricknames.append(brickname) for brickname in bricknames: scaledfns = glob('data/scaled/decals-dr2/*/%s/image-%s-*.fits' % (brickname[:3], brickname))
def main(): """Main routine which parses the optional inputs.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='DECaLS simulations.') parser.add_argument('-n', '--nobj', type=long, default=None, metavar='', help='number of objects to simulate (required input)') parser.add_argument('-c', '--chunksize', type=long, default=500, metavar='', help='divide NOBJ into CHUNKSIZE chunks') parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='', help='simulate objects in this brick') parser.add_argument('-o', '--objtype', type=str, default='STAR', metavar='', help='object type (STAR, ELG, LRG, QSO, LSB)') parser.add_argument( '-t', '--threads', type=int, default=8, metavar='', help='number of threads to use when calling The Tractor') parser.add_argument('-s', '--seed', type=long, default=None, metavar='', help='random number seed') parser.add_argument('-z', '--zoom', nargs=4, type=int, metavar='', help='see runbrick.py; (default is 0 3600 0 3600)') parser.add_argument('--rmag-range', nargs=2, type=float, default=(18, 25), metavar='', help='r-band magnitude range') parser.add_argument('--no-qaplots', action='store_true', help='do not generate QAplots') parser.add_argument('-v', '--verbose', dest='verbose', action='count', default=0, help='toggle on verbose output') args = parser.parse_args() if args.nobj is None: parser.print_help() sys.exit(1) # Set the debugging level if args.verbose: lvl = logging.DEBUG else: lvl = logging.INFO logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout) log = logging.getLogger('__name__') brickname = args.brick objtype = args.objtype.upper() lobjtype = objtype.lower() if objtype == 'LRG': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) elif objtype == 'LSB': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) elif objtype == 'QSO': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) # Determine how many chunks we need nobj = args.nobj chunksize = args.chunksize nchunk = long(np.ceil(nobj / chunksize)) log.info('Object type = {}'.format(objtype)) log.info('Number of objects = {}'.format(nobj)) log.info('Chunksize = {}'.format(chunksize)) log.info('Number of chunks = {}'.format(nchunk)) # Optionally zoom into a portion of the brick decals = Decals() brickinfo = decals.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brickinfo) W, H, pixscale = brickwcs.get_width(), brickwcs.get_height( ), brickwcs.pixel_scale() log.info('Brick = {}'.format(brickname)) if args.zoom is not None: # See also runbrick.stage_tims() (x0, x1, y0, y1) = args.zoom W = x1 - x0 H = y1 - y0 brickwcs = brickwcs.get_subimage(x0, y0, W, H) log.info('Zoom (pixel boundaries) = {}'.format(args.zoom)) radec_center = brickwcs.radec_center() log.info('RA, Dec center = {}'.format(radec_center)) log.info('Brick = {}'.format(brickname)) if args.seed is not None: log.info('Random seed = {}'.format(args.seed)) # Pack the input parameters into a meta-data table. meta = Table() meta['brickname'] = Column([brickname], dtype='S10') meta['objtype'] = Column([objtype], dtype='S10') if args.seed is not None: meta['seed'] = Column([args.seed], dtype='i4') meta['nobj'] = Column([args.nobj], dtype='i4') meta['chunksize'] = Column([args.chunksize], dtype='i2') meta['nchunk'] = Column([nchunk], dtype='i2') #meta['RA'] = Column([ra_range],dtype='f8') #meta['DEC'] = Column([dec_range],dtype='f8') if args.zoom is None: meta['zoom'] = Column([[0, 3600, 0, 3600]], dtype='i4') else: meta['zoom'] = Column([args.zoom], dtype='i4') meta['rmag_range'] = Column([args.rmag_range], dtype='f4') print('Hack!') decals_sim_dir = './' metafile = os.path.join(decals_sim_dir, 'metacat-' + brickname + '-' + lobjtype + '.fits') log.info('Writing {}'.format(metafile)) if os.path.isfile(metafile): os.remove(metafile) meta.write(metafile) # Work in chunks. for ichunk in range(nchunk): log.info('Working on chunk {:02d}/{:02d}'.format(ichunk + 1, nchunk)) chunksuffix = '{:02d}'.format(ichunk) # There's probably a smarter way to do this if ((ichunk + 1) * chunksize) > nobj: nobjchunk = (ichunk + 1) * chunksize - nobj else: nobjchunk = chunksize print(nobjchunk) # Build and write out the simulated object catalog. simcat = build_simcat(nobjchunk, brickname, brickwcs, meta) simcatfile = os.path.join( decals_sim_dir, 'simcat-' + brickname + '-' + lobjtype + '-' + chunksuffix + '.fits') log.info('Writing {}'.format(simcatfile)) if os.path.isfile(simcatfile): os.remove(simcatfile) simcat.write(simcatfile) #fitsio.write(simcatfile,simcat) # Use Tractor to just process the blobs containing the simulated sources. simdecals = SimDecals(sim_sources=simcat) # Test code ccdinfo = decals.ccds_touching_wcs(brickwcs) sim = SimImage(simdecals, ccdinfo[0]) tim = sim.get_tractor_image(const2psf=True)
def main(decals=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0,x1,y0,y1) = opt.zoom zoomslice = (slice(y0,y1), slice(x0,x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for decals.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if decals is None: decals = Decals() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from decals-ccds.fits table T = decals.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut(np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert(len(T) == 1) im = decals.get_image_object(T[0]) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True) print('Got tim:', tim) if opt.catfn in ['DR1', 'DR2']: if opt.catalog_path is None: opt.catalog_path = opt.catfn.lower() margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, decals=decals) for b in bricks: # there is some overlap with this brick... read the catalog. fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3], 'tractor-%s.fits' % b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec) W,H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) * (yy >= -margin) * (yy <= (H+margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') TT.append(T) T = merge_tables(TT) T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE) # print('Got cat:', cat) print('Forced photom...') opti = None if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)) ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x-1).astype(np.float32) F.y = (y-1).astype(np.float32) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap ap = 1./(np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap if opt.forced: kwa = {} if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa) if opt.plots: (data,mod,ie,chi,roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([src.getBrightness().getFlux(tim.band) for src in cat]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2 .astype(np.float32) program_name = sys.argv[0] version_hdr = get_version_header(program_name, decals.decals_dir) # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) fname = os.path.join(d2, d1, fname) print('Trimmed filename to', fname) #version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file')) version_hdr.add_record(dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext')) version_hdr.add_record(dict(name='CAMERA', value='DECam', comment='Dark Energy Camera')) version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num')) version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name')) version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record(dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image')) keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV', 'INSTRUME'] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = {'mjd':'sec', 'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'} columns = F.get_columns() for i,col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) print('Finished forced phot:', Time()-t0) return 0
import numpy as np from astrometry.util.plotutils import * from legacyanalysis.ps1cat import ps1cat from legacypipe.common import Decals from tractor import Image, PointSource, PixPos, NanoMaggies, Tractor ps = PlotSequence('rewcs') expnum, ccdname = 431109, 'N14' cat = ps1cat(expnum=expnum, ccdname=ccdname) stars = cat.get_stars() print len(stars), 'stars' decals = Decals() ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0] im = decals.get_image_object(ccd) wcs = im.get_wcs() tim = im.get_tractor_image(pixPsf=True, splinesky=True) margin = 15 ok,stars.xx,stars.yy = wcs.radec2pixelxy(stars.ra, stars.dec) stars.xx -= 1. stars.yy -= 1. W,H = wcs.get_width(), wcs.get_height() stars.ix = np.round(stars.xx).astype(int) stars.iy = np.round(stars.yy).astype(int) stars.cut((stars.ix >= margin) * (stars.ix < (W-margin)) * (stars.iy >= margin) * (stars.iy < (H-margin)))
def main(): decals = Decals() ccds = decals.get_ccds() print(len(ccds), 'CCDs') expnums = np.unique(ccds.expnum) print(len(expnums), 'unique exposures') for expnum in expnums: expnumstr = '%08i' % expnum skyoutfn = os.path.join('splinesky', expnumstr[:5], 'decam-%s.fits' % expnumstr) psfoutfn = os.path.join('psfex', expnumstr[:5], 'decam-%s.fits' % expnumstr) if os.path.exists(skyoutfn) and os.path.exists(psfoutfn): print('Exposure', expnum, 'is done already') continue C = ccds[ccds.expnum == expnum] print(len(C), 'CCDs in expnum', expnum) psfex = [] psfhdrvals = [] splinesky = [] skyhdrvals = [] for ccd in C: im = decals.get_image_object(ccd) fn = im.splineskyfn if os.path.exists(fn): T = fits_table(fn) splinesky.append(T) # print(fn) # T.about() hdr = fitsio.read_header(fn) skyhdrvals.append([hdr[k] for k in [ 'SKY', 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname]) else: print('File not found:', fn) fn = im.psffn if os.path.exists(fn): T = fits_table(fn) hdr = fitsio.read_header(fn, ext=1) keys = ['LOADED', 'ACCEPTED', 'CHI2', 'POLNAXIS', 'POLNGRP', 'PSF_FWHM', 'PSF_SAMP', 'PSFNAXIS', 'PSFAXIS1', 'PSFAXIS2', 'PSFAXIS3',] if hdr['POLNAXIS'] == 0: # No polynomials. Fake it. T.polgrp1 = np.array([0]) T.polgrp2 = np.array([0]) T.polname1 = np.array(['fake']) T.polname2 = np.array(['fake']) T.polzero1 = np.array([0]) T.polzero2 = np.array([0]) T.polscal1 = np.array([1]) T.polscal2 = np.array([1]) T.poldeg1 = np.array([0]) T.poldeg2 = np.array([0]) else: keys.extend([ 'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1', 'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2', 'POLDEG1']) for k in keys: T.set(k.lower(), np.array([hdr[k]])) psfex.append(T) #print(fn) #T.about() hdr = fitsio.read_header(fn) psfhdrvals.append([hdr.get(k,'') for k in [ 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname]) else: print('File not found:', fn) if len(psfex): padded = pad_arrays([p.psf_mask[0] for p in psfex]) cols = psfex[0].columns() cols.remove('psf_mask') T = merge_tables(psfex, columns=cols) T.psf_mask = np.concatenate([[p] for p in padded]) T.legpipev = np.array([h[0] for h in psfhdrvals]) T.plver = np.array([h[1] for h in psfhdrvals]) T.expnum = np.array([h[2] for h in psfhdrvals]) T.ccdname = np.array([h[3] for h in psfhdrvals]) fn = psfoutfn trymakedirs(fn, dir=True) T.writeto(fn) print('Wrote', fn) if len(splinesky): T = fits_table() T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky]) T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky]) padded = pad_arrays([t.gridvals[0] for t in splinesky]) T.gridvals = np.concatenate([[p] for p in padded]) padded = pad_arrays([t.xgrid[0] for t in splinesky]) T.xgrid = np.concatenate([[p] for p in padded]) padded = pad_arrays([t.xgrid[0] for t in splinesky]) T.ygrid = np.concatenate([[p] for p in padded]) cols = splinesky[0].columns() print('Columns:', cols) for c in ['gridvals', 'xgrid', 'ygrid']: cols.remove(c) T.add_columns_from(merge_tables(splinesky, columns=cols)) T.skyclass = np.array([h[0] for h in skyhdrvals]) T.legpipev = np.array([h[1] for h in skyhdrvals]) T.plver = np.array([h[2] for h in skyhdrvals]) T.expnum = np.array([h[3] for h in skyhdrvals]) T.ccdname = np.array([h[4] for h in skyhdrvals]) fn = skyoutfn trymakedirs(fn, dir=True) T.writeto(fn) print('Wrote', fn)