def main(): """Main program. """ import argparse parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--force', action='store_true', help='Run calib processes even if files already exist?') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--expnum', type=int, help='Cut to a single exposure') parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name') parser.add_argument('--no-astrom', dest='astrom', action='store_false', help='Do not compute astrometric calibs') parser.add_argument('--no-psf', dest='psfex', action='store_false', help='Do not compute PsfEx calibs') parser.add_argument('--no-sky', dest='sky', action='store_false', help='Do not compute sky models') parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor') parser.add_argument('--splinesky', action='store_true', help='Spline sky, not constant') parser.add_argument('args',nargs=argparse.REMAINDER) opt = parser.parse_args() D = Decals() if opt.ccds is not None: T = fits_table(opt.ccds) print('Read', len(T), 'from', opt.ccds) else: T = D.get_ccds() #print len(T), 'CCDs' if len(opt.args) == 0: if opt.expnum is not None: T.cut(T.expnum == opt.expnum) print('Cut to', len(T), 'with expnum =', opt.expnum) if opt.extname is not None: T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname])) print('Cut to', len(T), 'with extname =', opt.extname) opt.args = range(len(T)) for a in opt.args: i = int(a) t = T[i] im = D.get_image_object(t) print('Running', im.calname) kwargs = dict(pvastrom=opt.astrom, psfex=opt.psfex, sky=opt.sky) if opt.force: kwargs.update(force=True) if opt.run_se: kwargs.update(se=True) if opt.splinesky: kwargs.update(splinesky=True) run_calibs((im, kwargs)) return 0
def __init__(self,expnum=None,ccdname=None,ccdwcs=None): """Initialize the class with either the exposure number *and* CCD name, or directly with the WCS of the CCD of interest. """ self.ps1dir = os.getenv('PS1CAT_DIR') if self.ps1dir is None: raise ValueError('You must have the PS1CAT_DIR environment variable set to point to Pan-STARRS1 catalogs') self.nside = 32 if ccdwcs is None: from legacypipe.common import Decals decals = Decals() ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0] im = decals.get_image_object(ccd) self.ccdwcs = im.get_wcs() else: self.ccdwcs = ccdwcs
def __init__(self, expnum=None, ccdname=None, ccdwcs=None): """Initialize the class with either the exposure number *and* CCD name, or directly with the WCS of the CCD of interest. """ self.ps1dir = os.getenv('PS1CAT_DIR') if self.ps1dir is None: raise ValueError( 'You must have the PS1CAT_DIR environment variable set to point to Pan-STARRS1 catalogs' ) self.nside = 32 if ccdwcs is None: from legacypipe.common import Decals decals = Decals() ccd = decals.find_ccds(expnum=expnum, ccdname=ccdname)[0] im = decals.get_image_object(ccd) self.ccdwcs = im.get_wcs() else: self.ccdwcs = ccdwcs
from legacyanalysis.ps1cat import ps1cat from legacypipe.common import Decals from tractor import Image, PointSource, PixPos, NanoMaggies, Tractor ps = PlotSequence('rewcs') expnum, ccdname = 431109, 'N14' cat = ps1cat(expnum=expnum, ccdname=ccdname) stars = cat.get_stars() print len(stars), 'stars' decals = Decals() ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0] im = decals.get_image_object(ccd) wcs = im.get_wcs() tim = im.get_tractor_image(pixPsf=True, splinesky=True) margin = 15 ok,stars.xx,stars.yy = wcs.radec2pixelxy(stars.ra, stars.dec) stars.xx -= 1. stars.yy -= 1. W,H = wcs.get_width(), wcs.get_height() stars.ix = np.round(stars.xx).astype(int) stars.iy = np.round(stars.yy).astype(int) stars.cut((stars.ix >= margin) * (stars.ix < (W-margin)) * (stars.iy >= margin) * (stars.iy < (H-margin))) plt.clf() plt.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.95,
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).") parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') opt = parser.parse_args() decals = Decals() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = decals.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = decals.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo,dhi = -15, 40 rlo,rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict(test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo,dhi = -90,90 rlo,rhi = 0, 360 elif opt.region == 'badsky': # A handful of exposures in DR2 with inconsistent sky estimates. #C = decals.get_ccds() #log(len(C), 'CCDs') #C.cut((C.expnum >= 257400) * (C.expnum < 257500)) T.cut(np.array([e in [257460, 257461, 257462, 257463, 257464, 257465, 257466, 257467, 257469, 257472, 257483, 257496] for e in T.expnum])) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25)/2. I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'badsky2': # UGH, missed this one in original 'badsky' definition. T.cut(T.expnum == 257466) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25)/2. I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo,rhi = 240,245 dlo,dhi = 5, 12 elif opt.region == 'edr-south': rlo,rhi = 240,245 dlo,dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo,rhi = 149.75, 150.75 dlo,dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo,rhi = 240,250 dlo,dhi = 10,15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo,rhi = 320., 360. dlo,dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo,rhi = 240,360 dlo,dhi = 20,40 elif opt.region == 'north': rlo,rhi = 120,240 dlo,dhi = 20,40 elif opt.region == 'northeast': rlo,rhi = 0,120 dlo,dhi = 20,40 elif opt.region == 'southwest': rlo,rhi = 240,360 dlo,dhi = -20,0 elif opt.region == 'south': rlo,rhi = 120,240 dlo,dhi = -20,0 elif opt.region == 'southeast': rlo,rhi = 0,120 dlo,dhi = -20,0 elif opt.region == 'midwest': rlo,rhi = 240,360 dlo,dhi = 0,20 elif opt.region == 'middle': rlo,rhi = 120,240 dlo,dhi = 0,20 elif opt.region == 'mideast': rlo,rhi = 0,120 dlo,dhi = 0,20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo,rhi = 250,260 dlo,dhi = 30,35 elif opt.region == 'virgo': rlo,rhi = 185,190 dlo,dhi = 10, 15 elif opt.region == 'virgo2': rlo,rhi = 182,192 dlo,dhi = 8, 18 elif opt.region == 'lsb': rlo,rhi = 147.2, 147.8 dlo,dhi = -0.4, 0.4 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, 0.25) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('decals-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing B.cut(np.argsort(-B.dec)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue print(b.brickname) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): expstr = '%08i' % T.expnum[i] #outdir = os.path.join('forced', expstr[:5], expstr) #trymakedirs(outdir) outfn = os.path.join('forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(decals.decals_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] f.write('python legacypipe/forced-photom-decam.py %s %i DR1 %s\n' % (imgfn, T.cpimage_hdu[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if opt.command: s = '; '.join(batch) else: s = ' '.join(batch) f.write(s + '\n') for j,i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j+1, 'of', len(allI)) im = decals.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j+1, 'of', len(allI)) im = decals.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = ('python legacypipe/run-calib.py --expnum %i --ccdname %s' % (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] if not opt.nper: f.write(s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, opt.command) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, opt.command) f.close() log('Wrote', opt.out) return 0
def psf_residuals(expnum, ccdname, stampsize=35, nstar=30, magrange=(13, 17), verbose=0, splinesky=False): # Set the debugging level. if verbose == 0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) pngprefix = 'qapsf-{}-{}'.format(expnum, ccdname) # Gather all the info we need about this CCD. decals = Decals() ccd = decals.find_ccds(expnum=expnum, ccdname=ccdname)[0] band = ccd.filter ps1band = dict(g=0, r=1, i=2, z=3, Y=4) print('Band {}'.format(band)) #scales = dict(g=0.0066, r=0.01, z=0.025) #vmin, vmax = np.arcsinh(-1), np.arcsinh(100) #print(scales[band]) im = decals.get_image_object(ccd) iminfo = im.get_image_info() H, W = iminfo['dims'] wcs = im.get_wcs() # Choose a uniformly selected subset of PS1 stars on this CCD. ps1 = ps1cat(ccdwcs=wcs) cat = ps1.get_stars(band=band, magrange=magrange) rand = np.random.RandomState(seed=expnum * ccd.ccdnum) these = rand.choice(len(cat) - 1, nstar, replace=False) #these = rand.random_integers(0,len(cat)-1,nstar) cat = cat[these] cat = cat[np.argsort(cat.median[:, ps1band[band]])] # sort by magnitude #print(cat.nmag_ok) get_tim_kwargs = dict(const2psf=True, splinesky=splinesky) # Make a QAplot of the positions of all the stars. tim = im.get_tractor_image(**get_tim_kwargs) img = tim.getImage() #img = tim.getImage()/scales[band] fig = plt.figure(figsize=(5, 10)) ax = fig.gca() ax.get_xaxis().get_major_formatter().set_useOffset(False) #ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest', # origin='lower',vmin=vmax,vmax=vmax) ax.imshow(img, **tim.ima) ax.axis('off') ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.format( band, expnum, ccdname, ccd.airmass, ccd.seeing)) for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ax.text(xpos, ypos, '{:2d}'.format(istar + 1), color='red', horizontalalignment='left') circ = plt.Circle((xpos, ypos), radius=30, color='g', fill=False, lw=1) ax.add_patch(circ) #radec = wcs.radec_bounds() #ax.scatter(cat.ra,cat.dec) #ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998]) #ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015]) #ax.set_xlabel('$RA\ (deg)$',fontsize=18) #ax.set_ylabel('$Dec\ (deg)$',fontsize=18) fig.savefig(pngprefix + '-ccd.png', bbox_inches='tight') # Initialize the many-stamp QAplot ncols = 3 nrows = np.ceil(nstar / ncols).astype('int') inchperstamp = 2.0 fig = plt.figure(figsize=(inchperstamp * 3 * ncols, inchperstamp * nrows)) irow = 0 icol = 0 for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) mag = ps1star.median[ps1band[band]] # r-band ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ix, iy = int(xpos), int(ypos) # create a little tractor Image object around the star slc = (slice(max(iy - stampsize, 0), min(iy + stampsize + 1, H)), slice(max(ix - stampsize, 0), min(ix + stampsize + 1, W))) # The PSF model 'const2Psf' is the one used in DR1: a 2-component # Gaussian fit to PsfEx instantiated in the image center. tim = im.get_tractor_image(slc=slc, **get_tim_kwargs) stamp = tim.getImage() ivarstamp = tim.getInvvar() # Initialize a tractor PointSource from PS1 measurements flux = NanoMaggies.magToNanomaggies(mag) star = PointSource(RaDecPos(ra, dec), NanoMaggies(**{band: flux})) # Fit just the source RA,Dec,flux. tractor = Tractor([tim], [star]) tractor.freezeParam('images') print('2-component MOG:', tim.psf) tractor.printThawedParams() for step in range(50): dlnp, X, alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_mog = tractor.getModelImage(0) chi2_mog = -2.0 * tractor.getLogLikelihood() mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0] # Now change the PSF model to a pixelized PSF model from PsfEx instantiated # at this place in the image. psf = PixelizedPsfEx(im.psffn) tim.psf = psf.constantPsfAt(xpos, ypos) #print('PSF model:', tim.psf) #tractor.printThawedParams() for step in range(50): dlnp, X, alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_psfex = tractor.getModelImage(0) chi2_psfex = -2.0 * tractor.getLogLikelihood() mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0] #mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95]) sig = np.std((stamp - model_psfex)[ivarstamp > 0]) mn, mx = [-2.0 * sig, 5 * sig] # Generate a QAplot. if (istar > 0) and (istar % (ncols) == 0): irow = irow + 1 icol = 3 * istar - 3 * ncols * irow #print(istar, irow, icol, icol+1, icol+2) ax1 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol), aspect='equal') ax1.axis('off') #ax1.imshow(stamp, **tim.ima) ax1.imshow(stamp, cmap='gray', interpolation='nearest', origin='lower', vmin=mn, vmax=mx) ax1.text(0.1, 0.9, '{:2d}'.format(istar + 1), color='white', horizontalalignment='left', verticalalignment='top', transform=ax1.transAxes) ax2 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 1), aspect='equal') ax2.axis('off') #ax2.imshow(stamp-model_mog, **tim.ima) ax2.imshow(stamp - model_mog, cmap='gray', interpolation='nearest', origin='lower', vmin=mn, vmax=mx) ax2.text(0.1, 0.9, 'MoG', color='white', horizontalalignment='left', verticalalignment='top', transform=ax2.transAxes) ax2.text(0.08, 0.08, '{:.3f}'.format(mag_mog), color='white', horizontalalignment='left', verticalalignment='bottom', transform=ax2.transAxes) #ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14) #ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex)) ax3 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 2), aspect='equal') ax3.axis('off') #ax3.imshow(stamp-model_psfex, **tim.ima) ax3.imshow(stamp - model_psfex, cmap='gray', interpolation='nearest', origin='lower', vmin=mn, vmax=mx) ax3.text(0.1, 0.9, 'PSFEx', color='white', horizontalalignment='left', verticalalignment='top', transform=ax3.transAxes) ax3.text(0.08, 0.08, '{:.3f}'.format(mag_psfex), color='white', horizontalalignment='left', verticalalignment='bottom', transform=ax3.transAxes) if istar == (nstar - 1): break fig.savefig(pngprefix + '-stargrid.png', bbox_inches='tight')
def main(): decals = Decals() ccds = decals.get_ccds() print(len(ccds), 'CCDs') expnums = np.unique(ccds.expnum) print(len(expnums), 'unique exposures') for expnum in expnums: expnumstr = '%08i' % expnum skyoutfn = os.path.join('splinesky', expnumstr[:5], 'decam-%s.fits' % expnumstr) psfoutfn = os.path.join('psfex', expnumstr[:5], 'decam-%s.fits' % expnumstr) if os.path.exists(skyoutfn) and os.path.exists(psfoutfn): print('Exposure', expnum, 'is done already') continue C = ccds[ccds.expnum == expnum] print(len(C), 'CCDs in expnum', expnum) psfex = [] psfhdrvals = [] splinesky = [] skyhdrvals = [] for ccd in C: im = decals.get_image_object(ccd) fn = im.splineskyfn if os.path.exists(fn): T = fits_table(fn) splinesky.append(T) # print(fn) # T.about() hdr = fitsio.read_header(fn) skyhdrvals.append( [hdr[k] for k in ['SKY', 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname]) else: print('File not found:', fn) fn = im.psffn if os.path.exists(fn): T = fits_table(fn) hdr = fitsio.read_header(fn, ext=1) keys = [ 'LOADED', 'ACCEPTED', 'CHI2', 'POLNAXIS', 'POLNGRP', 'PSF_FWHM', 'PSF_SAMP', 'PSFNAXIS', 'PSFAXIS1', 'PSFAXIS2', 'PSFAXIS3', ] if hdr['POLNAXIS'] == 0: # No polynomials. Fake it. T.polgrp1 = np.array([0]) T.polgrp2 = np.array([0]) T.polname1 = np.array(['fake']) T.polname2 = np.array(['fake']) T.polzero1 = np.array([0]) T.polzero2 = np.array([0]) T.polscal1 = np.array([1]) T.polscal2 = np.array([1]) T.poldeg1 = np.array([0]) T.poldeg2 = np.array([0]) else: keys.extend([ 'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1', 'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2', 'POLDEG1' ]) for k in keys: T.set(k.lower(), np.array([hdr[k]])) psfex.append(T) #print(fn) #T.about() hdr = fitsio.read_header(fn) psfhdrvals.append( [hdr.get(k, '') for k in ['LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname]) else: print('File not found:', fn) if len(psfex): padded = pad_arrays([p.psf_mask[0] for p in psfex]) cols = psfex[0].columns() cols.remove('psf_mask') T = merge_tables(psfex, columns=cols) T.psf_mask = np.concatenate([[p] for p in padded]) T.legpipev = np.array([h[0] for h in psfhdrvals]) T.plver = np.array([h[1] for h in psfhdrvals]) T.expnum = np.array([h[2] for h in psfhdrvals]) T.ccdname = np.array([h[3] for h in psfhdrvals]) fn = psfoutfn trymakedirs(fn, dir=True) T.writeto(fn) print('Wrote', fn) if len(splinesky): T = fits_table() T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky]) T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky]) padded = pad_arrays([t.gridvals[0] for t in splinesky]) T.gridvals = np.concatenate([[p] for p in padded]) padded = pad_arrays([t.xgrid[0] for t in splinesky]) T.xgrid = np.concatenate([[p] for p in padded]) padded = pad_arrays([t.xgrid[0] for t in splinesky]) T.ygrid = np.concatenate([[p] for p in padded]) cols = splinesky[0].columns() print('Columns:', cols) for c in ['gridvals', 'xgrid', 'ygrid']: cols.remove(c) T.add_columns_from(merge_tables(splinesky, columns=cols)) T.skyclass = np.array([h[0] for h in skyhdrvals]) T.legpipev = np.array([h[1] for h in skyhdrvals]) T.plver = np.array([h[2] for h in skyhdrvals]) T.expnum = np.array([h[3] for h in skyhdrvals]) T.ccdname = np.array([h[4] for h in skyhdrvals]) fn = skyoutfn trymakedirs(fn, dir=True) T.writeto(fn) print('Wrote', fn)
def main(outfn='ccds-annotated.fits', ccds=None): decals = Decals() if ccds is None: ccds = decals.get_ccds() # File from the "observing" svn repo: # https://desi.lbl.gov/svn/decam/code/observing/trunk tiles = fits_table('decam-tiles_obstatus.fits') #ccds.cut(np.arange(100)) #print("HACK!") #ccds.cut(np.array([name in ['N15', 'N16', 'N21', 'N9'] # for name in ccds.ccdname]) * # ccds.expnum == 229683) I = decals.photometric_ccds(ccds) ccds.photometric = np.zeros(len(ccds), bool) ccds.photometric[I] = True I = decals.apply_blacklist(ccds) ccds.blacklist_ok = np.zeros(len(ccds), bool) ccds.blacklist_ok[I] = True ccds.good_region = np.empty((len(ccds), 4), np.int16) ccds.good_region[:,:] = -1 ccds.ra0 = np.zeros(len(ccds), np.float64) ccds.dec0 = np.zeros(len(ccds), np.float64) ccds.ra1 = np.zeros(len(ccds), np.float64) ccds.dec1 = np.zeros(len(ccds), np.float64) ccds.ra2 = np.zeros(len(ccds), np.float64) ccds.dec2 = np.zeros(len(ccds), np.float64) ccds.ra3 = np.zeros(len(ccds), np.float64) ccds.dec3 = np.zeros(len(ccds), np.float64) ccds.dra = np.zeros(len(ccds), np.float32) ccds.ddec = np.zeros(len(ccds), np.float32) ccds.ra_center = np.zeros(len(ccds), np.float64) ccds.dec_center = np.zeros(len(ccds), np.float64) ccds.sig1 = np.zeros(len(ccds), np.float32) ccds.meansky = np.zeros(len(ccds), np.float32) ccds.stdsky = np.zeros(len(ccds), np.float32) ccds.maxsky = np.zeros(len(ccds), np.float32) ccds.minsky = np.zeros(len(ccds), np.float32) ccds.pixscale_mean = np.zeros(len(ccds), np.float32) ccds.pixscale_std = np.zeros(len(ccds), np.float32) ccds.pixscale_max = np.zeros(len(ccds), np.float32) ccds.pixscale_min = np.zeros(len(ccds), np.float32) ccds.psfnorm_mean = np.zeros(len(ccds), np.float32) ccds.psfnorm_std = np.zeros(len(ccds), np.float32) ccds.galnorm_mean = np.zeros(len(ccds), np.float32) ccds.galnorm_std = np.zeros(len(ccds), np.float32) gaussgalnorm = np.zeros(len(ccds), np.float32) # 2nd moments ccds.psf_mx2 = np.zeros(len(ccds), np.float32) ccds.psf_my2 = np.zeros(len(ccds), np.float32) ccds.psf_mxy = np.zeros(len(ccds), np.float32) # ccds.psf_a = np.zeros(len(ccds), np.float32) ccds.psf_b = np.zeros(len(ccds), np.float32) ccds.psf_theta = np.zeros(len(ccds), np.float32) ccds.psf_ell = np.zeros(len(ccds), np.float32) ccds.humidity = np.zeros(len(ccds), np.float32) ccds.outtemp = np.zeros(len(ccds), np.float32) ccds.tileid = np.zeros(len(ccds), np.int32) ccds.tilepass = np.zeros(len(ccds), np.uint8) ccds.tileebv = np.zeros(len(ccds), np.float32) plvers = [] for iccd,ccd in enumerate(ccds): im = decals.get_image_object(ccd) print('Reading CCD %i of %i:' % (iccd+1, len(ccds)), im) X = im.get_good_image_subregion() for i,x in enumerate(X): if x is not None: ccds.good_region[iccd,i] = x W,H = ccd.width, ccd.height psf = None wcs = None sky = None try: tim = im.get_tractor_image(pixPsf=True, splinesky=True, subsky=False, pixels=False) except: import traceback traceback.print_exc() plvers.append('') continue if tim is None: plvers.append('') continue psf = tim.psf wcs = tim.wcs.wcs sky = tim.sky hdr = tim.primhdr # print('Got PSF', psf) # print('Got sky', type(sky)) # print('Got WCS', wcs) ccds.humidity[iccd] = hdr.get('HUMIDITY') ccds.outtemp[iccd] = hdr.get('OUTTEMP') ccds.sig1[iccd] = tim.sig1 plvers.append(tim.plver) obj = hdr.get('OBJECT') # parse 'DECaLS_15150_r' words = obj.split('_') tile = None if len(words) == 3 and words[0] == 'DECaLS': try: tileid = int(words[1]) tile = tiles[tileid - 1] if tile.tileid != tileid: I = np.flatnonzero(tile.tileid == tileid) tile = tiles[I[0]] except: pass if tile is not None: ccds.tileid [iccd] = tile.tileid ccds.tilepass[iccd] = tile.get('pass') ccds.tileebv [iccd] = tile.ebv_med # Instantiate PSF on a grid S = 32 xx = np.linspace(1+S, W-S, 5) yy = np.linspace(1+S, H-S, 5) xx,yy = np.meshgrid(xx, yy) psfnorms = [] galnorms = [] for x,y in zip(xx.ravel(), yy.ravel()): p = im.psf_norm(tim, x=x, y=y) g = im.galaxy_norm(tim, x=x, y=y) psfnorms.append(p) galnorms.append(g) ccds.psfnorm_mean[iccd] = np.mean(psfnorms) ccds.psfnorm_std [iccd] = np.std (psfnorms) ccds.galnorm_mean[iccd] = np.mean(galnorms) ccds.galnorm_std [iccd] = np.std (galnorms) # PSF in center of field cx,cy = (W+1)/2., (H+1)/2. p = psf.getPointSourcePatch(cx, cy).patch ph,pw = p.shape px,py = np.meshgrid(np.arange(pw), np.arange(ph)) psum = np.sum(p) # print('psum', psum) p /= psum # centroids cenx = np.sum(p * px) ceny = np.sum(p * py) # print('cenx,ceny', cenx,ceny) # second moments x2 = np.sum(p * (px - cenx)**2) y2 = np.sum(p * (py - ceny)**2) xy = np.sum(p * (px - cenx)*(py - ceny)) # semi-major/minor axes and position angle theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.) theta = np.abs(theta) * np.sign(xy) s = np.sqrt(((x2 - y2)/2.)**2 + xy**2) a = np.sqrt((x2 + y2) / 2. + s) b = np.sqrt((x2 + y2) / 2. - s) ell = 1. - b/a # print('PSF second moments', x2, y2, xy) # print('PSF position angle', theta) # print('PSF semi-axes', a, b) # print('PSF ellipticity', ell) ccds.psf_mx2[iccd] = x2 ccds.psf_my2[iccd] = y2 ccds.psf_mxy[iccd] = xy ccds.psf_a[iccd] = a ccds.psf_b[iccd] = b ccds.psf_theta[iccd] = theta ccds.psf_ell [iccd] = ell # Galaxy norm using Gaussian approximation of PSF. realpsf = tim.psf tim.psf = im.read_psf_model(0, 0, gaussPsf=True, psf_sigma=tim.psf_sigma) gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy) tim.psf = realpsf # Sky mod = np.zeros((ccd.height, ccd.width), np.float32) sky.addTo(mod) ccds.meansky[iccd] = np.mean(mod) ccds.stdsky[iccd] = np.std(mod) ccds.maxsky[iccd] = mod.max() ccds.minsky[iccd] = mod.min() # WCS ccds.ra0[iccd],ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1) ccds.ra1[iccd],ccds.dec1[iccd] = wcs.pixelxy2radec(1, H) ccds.ra2[iccd],ccds.dec2[iccd] = wcs.pixelxy2radec(W, H) ccds.ra3[iccd],ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1) midx, midy = (W+1)/2., (H+1)/2. rc,dc = wcs.pixelxy2radec(midx, midy) ra,dec = wcs.pixelxy2radec([1,W,midx,midx], [midy,midy,1,H]) ccds.dra [iccd] = max(degrees_between(ra, dc+np.zeros_like(ra), rc, dc)) ccds.ddec[iccd] = max(degrees_between(rc+np.zeros_like(dec), dec, rc, dc)) ccds.ra_center [iccd] = rc ccds.dec_center[iccd] = dc # Compute scale change across the chip # how many pixels to step step = 10 xx = np.linspace(1+step, W-step, 5) yy = np.linspace(1+step, H-step, 5) xx,yy = np.meshgrid(xx, yy) pixscale = [] for x,y in zip(xx.ravel(), yy.ravel()): sx = [x-step, x-step, x+step, x+step, x-step] sy = [y-step, y+step, y+step, y-step, y-step] sr,sd = wcs.pixelxy2radec(sx, sy) rc,dc = wcs.pixelxy2radec(x, y) # project around a tiny little TAN WCS at (x,y), with 1" pixels locwcs = Tan(rc, dc, 0., 0., 1./3600, 0., 0., 1./3600, 1., 1.) ok,lx,ly = locwcs.radec2pixelxy(sr, sd) #print('local x,y:', lx, ly) A = polygon_area((lx, ly)) pixscale.append(np.sqrt(A / (2*step)**2)) # print('Pixel scales:', pixscale) ccds.pixscale_mean[iccd] = np.mean(pixscale) ccds.pixscale_min[iccd] = min(pixscale) ccds.pixscale_max[iccd] = max(pixscale) ccds.pixscale_std[iccd] = np.std(pixscale) ccds.plver = np.array(plvers) sfd = tractor.sfd.SFDMap() allbands = 'ugrizY' filts = ['%s %s' % ('DES', f) for f in allbands] wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4'] ebv,ext = sfd.extinction(filts + wisebands, ccds.ra_center, ccds.dec_center, get_ebv=True) ext = ext.astype(np.float32) ccds.ebv = ebv.astype(np.float32) ccds.decam_extinction = ext[:,:len(allbands)] ccds.wise_extinction = ext[:,len(allbands):] # Depth detsig1 = ccds.sig1 / ccds.psfnorm_mean depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.psfdepth = -2.5 * (np.log10(depth) - 9) detsig1 = ccds.sig1 / ccds.galnorm_mean depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.galdepth = -2.5 * (np.log10(depth) - 9) # Depth using Gaussian FWHM. psf_sigma = ccds.fwhm / 2.35 gnorm = 1./(2. * np.sqrt(np.pi) * psf_sigma) detsig1 = ccds.sig1 / gnorm depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9) # Gaussian galaxy depth detsig1 = ccds.sig1 / gaussgalnorm depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9) ccds.writeto(outfn)
def main(decals=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0,x1,y0,y1) = opt.zoom zoomslice = (slice(y0,y1), slice(x0,x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for decals.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if decals is None: decals = Decals() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from decals-ccds.fits table T = decals.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut(np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert(len(T) == 1) im = decals.get_image_object(T[0]) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True) print('Got tim:', tim) if opt.catfn in ['DR1', 'DR2']: if opt.catalog_path is None: opt.catalog_path = opt.catfn.lower() margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, decals=decals) for b in bricks: # there is some overlap with this brick... read the catalog. fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3], 'tractor-%s.fits' % b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec) W,H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) * (yy >= -margin) * (yy <= (H+margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') TT.append(T) T = merge_tables(TT) T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE) # print('Got cat:', cat) print('Forced photom...') opti = None if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)) ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x-1).astype(np.float32) F.y = (y-1).astype(np.float32) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap ap = 1./(np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap if opt.forced: kwa = {} if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa) if opt.plots: (data,mod,ie,chi,roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([src.getBrightness().getFlux(tim.band) for src in cat]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2 .astype(np.float32) program_name = sys.argv[0] version_hdr = get_version_header(program_name, decals.decals_dir) # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) fname = os.path.join(d2, d1, fname) print('Trimmed filename to', fname) #version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file')) version_hdr.add_record(dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext')) version_hdr.add_record(dict(name='CAMERA', value='DECam', comment='Dark Energy Camera')) version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num')) version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name')) version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record(dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image')) keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV', 'INSTRUME'] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = {'mjd':'sec', 'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'} columns = F.get_columns() for i,col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) print('Finished forced phot:', Time()-t0) return 0
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser( description= "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)." ) parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') opt = parser.parse_args() decals = Decals() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = decals.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = decals.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo, dhi = -15, 40 rlo, rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict( test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo, dhi = -90, 90 rlo, rhi = 0, 360 elif opt.region == 'badsky': # A handful of exposures in DR2 with inconsistent sky estimates. #C = decals.get_ccds() #log(len(C), 'CCDs') #C.cut((C.expnum >= 257400) * (C.expnum < 257500)) T.cut( np.array([ e in [ 257460, 257461, 257462, 257463, 257464, 257465, 257466, 257467, 257469, 257472, 257483, 257496 ] for e in T.expnum ])) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25) / 2. I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'badsky2': # UGH, missed this one in original 'badsky' definition. T.cut(T.expnum == 257466) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25) / 2. I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo, rhi = 240, 245 dlo, dhi = 5, 12 elif opt.region == 'edr-south': rlo, rhi = 240, 245 dlo, dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo, rhi = 149.75, 150.75 dlo, dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo, rhi = 240, 250 dlo, dhi = 10, 15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo, rhi = 320., 360. dlo, dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo, rhi = 240, 360 dlo, dhi = 20, 40 elif opt.region == 'north': rlo, rhi = 120, 240 dlo, dhi = 20, 40 elif opt.region == 'northeast': rlo, rhi = 0, 120 dlo, dhi = 20, 40 elif opt.region == 'southwest': rlo, rhi = 240, 360 dlo, dhi = -20, 0 elif opt.region == 'south': rlo, rhi = 120, 240 dlo, dhi = -20, 0 elif opt.region == 'southeast': rlo, rhi = 0, 120 dlo, dhi = -20, 0 elif opt.region == 'midwest': rlo, rhi = 240, 360 dlo, dhi = 0, 20 elif opt.region == 'middle': rlo, rhi = 120, 240 dlo, dhi = 0, 20 elif opt.region == 'mideast': rlo, rhi = 0, 120 dlo, dhi = 0, 20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo, rhi = 250, 260 dlo, dhi = 30, 35 elif opt.region == 'virgo': rlo, rhi = 185, 190 dlo, dhi = 10, 15 elif opt.region == 'virgo2': rlo, rhi = 182, 192 dlo, dhi = 8, 18 elif opt.region == 'lsb': rlo, rhi = 147.2, 147.8 dlo, dhi = -0.4, 0.4 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut( np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, 0.25) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('decals-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing B.cut(np.argsort(-B.dec)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % ( b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue print(b.brickname) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): expstr = '%08i' % T.expnum[i] #outdir = os.path.join('forced', expstr[:5], expstr) #trymakedirs(outdir) outfn = os.path.join( 'forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(decals.decals_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] f.write('python legacypipe/forced-photom-decam.py %s %i DR1 %s\n' % (imgfn, T.cpimage_hdu[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write( 'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if opt.command: s = '; '.join(batch) else: s = ' '.join(batch) f.write(s + '\n') for j, i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j + 1, 'of', len(allI)) im = decals.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j + 1, 'of', len(allI)) im = decals.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = ('python legacypipe/run-calib.py --expnum %i --ccdname %s' % (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] if not opt.nper: f.write(s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, opt.command) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, opt.command) f.close() log('Wrote', opt.out) return 0
def main(outfn='ccds-annotated.fits', ccds=None): decals = Decals() if ccds is None: ccds = decals.get_ccds() # File from the "observing" svn repo: # https://desi.lbl.gov/svn/decam/code/observing/trunk tiles = fits_table('decam-tiles_obstatus.fits') #ccds.cut(np.arange(100)) #print("HACK!") #ccds.cut(np.array([name in ['N15', 'N16', 'N21', 'N9'] # for name in ccds.ccdname]) * # ccds.expnum == 229683) I = decals.photometric_ccds(ccds) ccds.photometric = np.zeros(len(ccds), bool) ccds.photometric[I] = True I = decals.apply_blacklist(ccds) ccds.blacklist_ok = np.zeros(len(ccds), bool) ccds.blacklist_ok[I] = True ccds.good_region = np.empty((len(ccds), 4), np.int16) ccds.good_region[:, :] = -1 ccds.ra0 = np.zeros(len(ccds), np.float64) ccds.dec0 = np.zeros(len(ccds), np.float64) ccds.ra1 = np.zeros(len(ccds), np.float64) ccds.dec1 = np.zeros(len(ccds), np.float64) ccds.ra2 = np.zeros(len(ccds), np.float64) ccds.dec2 = np.zeros(len(ccds), np.float64) ccds.ra3 = np.zeros(len(ccds), np.float64) ccds.dec3 = np.zeros(len(ccds), np.float64) ccds.dra = np.zeros(len(ccds), np.float32) ccds.ddec = np.zeros(len(ccds), np.float32) ccds.ra_center = np.zeros(len(ccds), np.float64) ccds.dec_center = np.zeros(len(ccds), np.float64) ccds.sig1 = np.zeros(len(ccds), np.float32) ccds.meansky = np.zeros(len(ccds), np.float32) ccds.stdsky = np.zeros(len(ccds), np.float32) ccds.maxsky = np.zeros(len(ccds), np.float32) ccds.minsky = np.zeros(len(ccds), np.float32) ccds.pixscale_mean = np.zeros(len(ccds), np.float32) ccds.pixscale_std = np.zeros(len(ccds), np.float32) ccds.pixscale_max = np.zeros(len(ccds), np.float32) ccds.pixscale_min = np.zeros(len(ccds), np.float32) ccds.psfnorm_mean = np.zeros(len(ccds), np.float32) ccds.psfnorm_std = np.zeros(len(ccds), np.float32) ccds.galnorm_mean = np.zeros(len(ccds), np.float32) ccds.galnorm_std = np.zeros(len(ccds), np.float32) gaussgalnorm = np.zeros(len(ccds), np.float32) # 2nd moments ccds.psf_mx2 = np.zeros(len(ccds), np.float32) ccds.psf_my2 = np.zeros(len(ccds), np.float32) ccds.psf_mxy = np.zeros(len(ccds), np.float32) # ccds.psf_a = np.zeros(len(ccds), np.float32) ccds.psf_b = np.zeros(len(ccds), np.float32) ccds.psf_theta = np.zeros(len(ccds), np.float32) ccds.psf_ell = np.zeros(len(ccds), np.float32) ccds.humidity = np.zeros(len(ccds), np.float32) ccds.outtemp = np.zeros(len(ccds), np.float32) ccds.tileid = np.zeros(len(ccds), np.int32) ccds.tilepass = np.zeros(len(ccds), np.uint8) ccds.tileebv = np.zeros(len(ccds), np.float32) plvers = [] for iccd, ccd in enumerate(ccds): im = decals.get_image_object(ccd) print('Reading CCD %i of %i:' % (iccd + 1, len(ccds)), im) X = im.get_good_image_subregion() for i, x in enumerate(X): if x is not None: ccds.good_region[iccd, i] = x W, H = ccd.width, ccd.height psf = None wcs = None sky = None try: tim = im.get_tractor_image(pixPsf=True, splinesky=True, subsky=False, pixels=False) except: import traceback traceback.print_exc() plvers.append('') continue if tim is None: plvers.append('') continue psf = tim.psf wcs = tim.wcs.wcs sky = tim.sky hdr = tim.primhdr # print('Got PSF', psf) # print('Got sky', type(sky)) # print('Got WCS', wcs) ccds.humidity[iccd] = hdr.get('HUMIDITY') ccds.outtemp[iccd] = hdr.get('OUTTEMP') ccds.sig1[iccd] = tim.sig1 plvers.append(tim.plver) obj = hdr.get('OBJECT') # parse 'DECaLS_15150_r' words = obj.split('_') tile = None if len(words) == 3 and words[0] == 'DECaLS': try: tileid = int(words[1]) tile = tiles[tileid - 1] if tile.tileid != tileid: I = np.flatnonzero(tile.tileid == tileid) tile = tiles[I[0]] except: pass if tile is not None: ccds.tileid[iccd] = tile.tileid ccds.tilepass[iccd] = tile.get('pass') ccds.tileebv[iccd] = tile.ebv_med # Instantiate PSF on a grid S = 32 xx = np.linspace(1 + S, W - S, 5) yy = np.linspace(1 + S, H - S, 5) xx, yy = np.meshgrid(xx, yy) psfnorms = [] galnorms = [] for x, y in zip(xx.ravel(), yy.ravel()): p = im.psf_norm(tim, x=x, y=y) g = im.galaxy_norm(tim, x=x, y=y) psfnorms.append(p) galnorms.append(g) ccds.psfnorm_mean[iccd] = np.mean(psfnorms) ccds.psfnorm_std[iccd] = np.std(psfnorms) ccds.galnorm_mean[iccd] = np.mean(galnorms) ccds.galnorm_std[iccd] = np.std(galnorms) # PSF in center of field cx, cy = (W + 1) / 2., (H + 1) / 2. p = psf.getPointSourcePatch(cx, cy).patch ph, pw = p.shape px, py = np.meshgrid(np.arange(pw), np.arange(ph)) psum = np.sum(p) # print('psum', psum) p /= psum # centroids cenx = np.sum(p * px) ceny = np.sum(p * py) # print('cenx,ceny', cenx,ceny) # second moments x2 = np.sum(p * (px - cenx)**2) y2 = np.sum(p * (py - ceny)**2) xy = np.sum(p * (px - cenx) * (py - ceny)) # semi-major/minor axes and position angle theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.) theta = np.abs(theta) * np.sign(xy) s = np.sqrt(((x2 - y2) / 2.)**2 + xy**2) a = np.sqrt((x2 + y2) / 2. + s) b = np.sqrt((x2 + y2) / 2. - s) ell = 1. - b / a # print('PSF second moments', x2, y2, xy) # print('PSF position angle', theta) # print('PSF semi-axes', a, b) # print('PSF ellipticity', ell) ccds.psf_mx2[iccd] = x2 ccds.psf_my2[iccd] = y2 ccds.psf_mxy[iccd] = xy ccds.psf_a[iccd] = a ccds.psf_b[iccd] = b ccds.psf_theta[iccd] = theta ccds.psf_ell[iccd] = ell # Galaxy norm using Gaussian approximation of PSF. realpsf = tim.psf tim.psf = im.read_psf_model(0, 0, gaussPsf=True, psf_sigma=tim.psf_sigma) gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy) tim.psf = realpsf # Sky mod = np.zeros((ccd.height, ccd.width), np.float32) sky.addTo(mod) ccds.meansky[iccd] = np.mean(mod) ccds.stdsky[iccd] = np.std(mod) ccds.maxsky[iccd] = mod.max() ccds.minsky[iccd] = mod.min() # WCS ccds.ra0[iccd], ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1) ccds.ra1[iccd], ccds.dec1[iccd] = wcs.pixelxy2radec(1, H) ccds.ra2[iccd], ccds.dec2[iccd] = wcs.pixelxy2radec(W, H) ccds.ra3[iccd], ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1) midx, midy = (W + 1) / 2., (H + 1) / 2. rc, dc = wcs.pixelxy2radec(midx, midy) ra, dec = wcs.pixelxy2radec([1, W, midx, midx], [midy, midy, 1, H]) ccds.dra[iccd] = max( degrees_between(ra, dc + np.zeros_like(ra), rc, dc)) ccds.ddec[iccd] = max( degrees_between(rc + np.zeros_like(dec), dec, rc, dc)) ccds.ra_center[iccd] = rc ccds.dec_center[iccd] = dc # Compute scale change across the chip # how many pixels to step step = 10 xx = np.linspace(1 + step, W - step, 5) yy = np.linspace(1 + step, H - step, 5) xx, yy = np.meshgrid(xx, yy) pixscale = [] for x, y in zip(xx.ravel(), yy.ravel()): sx = [x - step, x - step, x + step, x + step, x - step] sy = [y - step, y + step, y + step, y - step, y - step] sr, sd = wcs.pixelxy2radec(sx, sy) rc, dc = wcs.pixelxy2radec(x, y) # project around a tiny little TAN WCS at (x,y), with 1" pixels locwcs = Tan(rc, dc, 0., 0., 1. / 3600, 0., 0., 1. / 3600, 1., 1.) ok, lx, ly = locwcs.radec2pixelxy(sr, sd) #print('local x,y:', lx, ly) A = polygon_area((lx, ly)) pixscale.append(np.sqrt(A / (2 * step)**2)) # print('Pixel scales:', pixscale) ccds.pixscale_mean[iccd] = np.mean(pixscale) ccds.pixscale_min[iccd] = min(pixscale) ccds.pixscale_max[iccd] = max(pixscale) ccds.pixscale_std[iccd] = np.std(pixscale) ccds.plver = np.array(plvers) sfd = tractor.sfd.SFDMap() allbands = 'ugrizY' filts = ['%s %s' % ('DES', f) for f in allbands] wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4'] ebv, ext = sfd.extinction(filts + wisebands, ccds.ra_center, ccds.dec_center, get_ebv=True) ext = ext.astype(np.float32) ccds.ebv = ebv.astype(np.float32) ccds.decam_extinction = ext[:, :len(allbands)] ccds.wise_extinction = ext[:, len(allbands):] # Depth detsig1 = ccds.sig1 / ccds.psfnorm_mean depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.psfdepth = -2.5 * (np.log10(depth) - 9) detsig1 = ccds.sig1 / ccds.galnorm_mean depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.galdepth = -2.5 * (np.log10(depth) - 9) # Depth using Gaussian FWHM. psf_sigma = ccds.fwhm / 2.35 gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) detsig1 = ccds.sig1 / gnorm depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9) # Gaussian galaxy depth detsig1 = ccds.sig1 / gaussgalnorm depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9) ccds.writeto(outfn)
def main(decals=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0, x1, y0, y1) = opt.zoom zoomslice = (slice(y0, y1), slice(x0, x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for decals.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if decals is None: decals = Decals() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from decals-ccds.fits table T = decals.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut( np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert (len(T) == 1) im = decals.get_image_object(T[0]) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True) print('Got tim:', tim) if opt.catfn in ['DR1', 'DR2']: if opt.catalog_path is None: opt.catalog_path = opt.catfn.lower() margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, decals=decals) for b in bricks: # there is some overlap with this brick... read the catalog. fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3], 'tractor-%s.fits' % b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec) W, H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) * (yy >= -margin) * (yy <= (H + margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') TT.append(T) T = merge_tables(TT) T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE) # print('Got cat:', cat) print('Forced photom...') opti = None if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)) ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x - 1).astype(np.float32) F.y = (y - 1).astype(np.float32) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack( [tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap ap = 1. / (np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap if opt.forced: kwa = {} if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa) if opt.plots: (data, mod, ie, chi, roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([ src.getBrightness().getFlux(tim.band) for src in cat ]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2.astype(np.float32) program_name = sys.argv[0] version_hdr = get_version_header(program_name, decals.decals_dir) # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) fname = os.path.join(d2, d1, fname) print('Trimmed filename to', fname) #version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file')) version_hdr.add_record( dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file')) version_hdr.add_record( dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext')) version_hdr.add_record( dict(name='CAMERA', value='DECam', comment='Dark Energy Camera')) version_hdr.add_record( dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num')) version_hdr.add_record( dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name')) version_hdr.add_record( dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record( dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image')) keys = [ 'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME' ] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = { 'mjd': 'sec', 'exptime': 'sec', 'flux': 'nanomaggy', 'flux_ivar': '1/nanomaggy^2' } columns = F.get_columns() for i, col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) print('Finished forced phot:', Time() - t0) return 0
def psf_residuals(expnum,ccdname,stampsize=35,nstar=30, magrange=(13,17),verbose=0, splinesky=False): # Set the debugging level. if verbose==0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout) pngprefix = 'qapsf-{}-{}'.format(expnum,ccdname) # Gather all the info we need about this CCD. decals = Decals() ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0] band = ccd.filter ps1band = dict(g=0,r=1,i=2,z=3,Y=4) print('Band {}'.format(band)) #scales = dict(g=0.0066, r=0.01, z=0.025) #vmin, vmax = np.arcsinh(-1), np.arcsinh(100) #print(scales[band]) im = decals.get_image_object(ccd) iminfo = im.get_image_info() H,W = iminfo['dims'] wcs = im.get_wcs() # Choose a uniformly selected subset of PS1 stars on this CCD. ps1 = ps1cat(ccdwcs=wcs) cat = ps1.get_stars(band=band,magrange=magrange) rand = np.random.RandomState(seed=expnum*ccd.ccdnum) these = rand.choice(len(cat)-1,nstar,replace=False) #these = rand.random_integers(0,len(cat)-1,nstar) cat = cat[these] cat = cat[np.argsort(cat.median[:,ps1band[band]])] # sort by magnitude #print(cat.nmag_ok) get_tim_kwargs = dict(const2psf=True, splinesky=splinesky) # Make a QAplot of the positions of all the stars. tim = im.get_tractor_image(**get_tim_kwargs) img = tim.getImage() #img = tim.getImage()/scales[band] fig = plt.figure(figsize=(5,10)) ax = fig.gca() ax.get_xaxis().get_major_formatter().set_useOffset(False) #ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest', # origin='lower',vmin=vmax,vmax=vmax) ax.imshow(img, **tim.ima) ax.axis('off') ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'. format(band,expnum,ccdname,ccd.airmass,ccd.seeing)) for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ax.text(xpos,ypos,'{:2d}'.format(istar+1),color='red', horizontalalignment='left') circ = plt.Circle((xpos,ypos),radius=30,color='g',fill=False,lw=1) ax.add_patch(circ) #radec = wcs.radec_bounds() #ax.scatter(cat.ra,cat.dec) #ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998]) #ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015]) #ax.set_xlabel('$RA\ (deg)$',fontsize=18) #ax.set_ylabel('$Dec\ (deg)$',fontsize=18) fig.savefig(pngprefix+'-ccd.png',bbox_inches='tight') # Initialize the many-stamp QAplot ncols = 3 nrows = np.ceil(nstar/ncols).astype('int') inchperstamp = 2.0 fig = plt.figure(figsize=(inchperstamp*3*ncols,inchperstamp*nrows)) irow = 0 icol = 0 for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) mag = ps1star.median[ps1band[band]] # r-band ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ix,iy = int(xpos), int(ypos) # create a little tractor Image object around the star slc = (slice(max(iy-stampsize, 0), min(iy+stampsize+1, H)), slice(max(ix-stampsize, 0), min(ix+stampsize+1, W))) # The PSF model 'const2Psf' is the one used in DR1: a 2-component # Gaussian fit to PsfEx instantiated in the image center. tim = im.get_tractor_image(slc=slc, **get_tim_kwargs) stamp = tim.getImage() ivarstamp = tim.getInvvar() # Initialize a tractor PointSource from PS1 measurements flux = NanoMaggies.magToNanomaggies(mag) star = PointSource(RaDecPos(ra,dec), NanoMaggies(**{band: flux})) # Fit just the source RA,Dec,flux. tractor = Tractor([tim], [star]) tractor.freezeParam('images') print('2-component MOG:', tim.psf) tractor.printThawedParams() for step in range(50): dlnp,X,alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_mog = tractor.getModelImage(0) chi2_mog = -2.0*tractor.getLogLikelihood() mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0] # Now change the PSF model to a pixelized PSF model from PsfEx instantiated # at this place in the image. psf = PixelizedPsfEx(im.psffn) tim.psf = psf.constantPsfAt(xpos, ypos) #print('PSF model:', tim.psf) #tractor.printThawedParams() for step in range(50): dlnp,X,alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_psfex = tractor.getModelImage(0) chi2_psfex = -2.0*tractor.getLogLikelihood() mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0] #mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95]) sig = np.std((stamp-model_psfex)[ivarstamp>0]) mn, mx = [-2.0*sig,5*sig] # Generate a QAplot. if (istar>0) and (istar%(ncols)==0): irow = irow+1 icol = 3*istar - 3*ncols*irow #print(istar, irow, icol, icol+1, icol+2) ax1 = plt.subplot2grid((nrows,3*ncols), (irow,icol), aspect='equal') ax1.axis('off') #ax1.imshow(stamp, **tim.ima) ax1.imshow(stamp,cmap='gray',interpolation='nearest', origin='lower',vmin=mn,vmax=mx) ax1.text(0.1,0.9,'{:2d}'.format(istar+1),color='white', horizontalalignment='left',verticalalignment='top', transform=ax1.transAxes) ax2 = plt.subplot2grid((nrows,3*ncols), (irow,icol+1), aspect='equal') ax2.axis('off') #ax2.imshow(stamp-model_mog, **tim.ima) ax2.imshow(stamp-model_mog,cmap='gray',interpolation='nearest', origin='lower',vmin=mn,vmax=mx) ax2.text(0.1,0.9,'MoG',color='white', horizontalalignment='left',verticalalignment='top', transform=ax2.transAxes) ax2.text(0.08,0.08,'{:.3f}'.format(mag_mog),color='white', horizontalalignment='left',verticalalignment='bottom', transform=ax2.transAxes) #ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14) #ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex)) ax3 = plt.subplot2grid((nrows,3*ncols), (irow,icol+2), aspect='equal') ax3.axis('off') #ax3.imshow(stamp-model_psfex, **tim.ima) ax3.imshow(stamp-model_psfex,cmap='gray',interpolation='nearest', origin='lower',vmin=mn,vmax=mx) ax3.text(0.1,0.9,'PSFEx',color='white', horizontalalignment='left',verticalalignment='top', transform=ax3.transAxes) ax3.text(0.08,0.08,'{:.3f}'.format(mag_psfex),color='white', horizontalalignment='left',verticalalignment='bottom', transform=ax3.transAxes) if istar==(nstar-1): break fig.savefig(pngprefix+'-stargrid.png',bbox_inches='tight')
def main(): decals = Decals() ccds = decals.get_ccds() print(len(ccds), 'CCDs') expnums = np.unique(ccds.expnum) print(len(expnums), 'unique exposures') for expnum in expnums: expnumstr = '%08i' % expnum skyoutfn = os.path.join('splinesky', expnumstr[:5], 'decam-%s.fits' % expnumstr) psfoutfn = os.path.join('psfex', expnumstr[:5], 'decam-%s.fits' % expnumstr) if os.path.exists(skyoutfn) and os.path.exists(psfoutfn): print('Exposure', expnum, 'is done already') continue C = ccds[ccds.expnum == expnum] print(len(C), 'CCDs in expnum', expnum) psfex = [] psfhdrvals = [] splinesky = [] skyhdrvals = [] for ccd in C: im = decals.get_image_object(ccd) fn = im.splineskyfn if os.path.exists(fn): T = fits_table(fn) splinesky.append(T) # print(fn) # T.about() hdr = fitsio.read_header(fn) skyhdrvals.append([hdr[k] for k in [ 'SKY', 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname]) else: print('File not found:', fn) fn = im.psffn if os.path.exists(fn): T = fits_table(fn) hdr = fitsio.read_header(fn, ext=1) keys = ['LOADED', 'ACCEPTED', 'CHI2', 'POLNAXIS', 'POLNGRP', 'PSF_FWHM', 'PSF_SAMP', 'PSFNAXIS', 'PSFAXIS1', 'PSFAXIS2', 'PSFAXIS3',] if hdr['POLNAXIS'] == 0: # No polynomials. Fake it. T.polgrp1 = np.array([0]) T.polgrp2 = np.array([0]) T.polname1 = np.array(['fake']) T.polname2 = np.array(['fake']) T.polzero1 = np.array([0]) T.polzero2 = np.array([0]) T.polscal1 = np.array([1]) T.polscal2 = np.array([1]) T.poldeg1 = np.array([0]) T.poldeg2 = np.array([0]) else: keys.extend([ 'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1', 'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2', 'POLDEG1']) for k in keys: T.set(k.lower(), np.array([hdr[k]])) psfex.append(T) #print(fn) #T.about() hdr = fitsio.read_header(fn) psfhdrvals.append([hdr.get(k,'') for k in [ 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname]) else: print('File not found:', fn) if len(psfex): padded = pad_arrays([p.psf_mask[0] for p in psfex]) cols = psfex[0].columns() cols.remove('psf_mask') T = merge_tables(psfex, columns=cols) T.psf_mask = np.concatenate([[p] for p in padded]) T.legpipev = np.array([h[0] for h in psfhdrvals]) T.plver = np.array([h[1] for h in psfhdrvals]) T.expnum = np.array([h[2] for h in psfhdrvals]) T.ccdname = np.array([h[3] for h in psfhdrvals]) fn = psfoutfn trymakedirs(fn, dir=True) T.writeto(fn) print('Wrote', fn) if len(splinesky): T = fits_table() T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky]) T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky]) padded = pad_arrays([t.gridvals[0] for t in splinesky]) T.gridvals = np.concatenate([[p] for p in padded]) padded = pad_arrays([t.xgrid[0] for t in splinesky]) T.xgrid = np.concatenate([[p] for p in padded]) padded = pad_arrays([t.xgrid[0] for t in splinesky]) T.ygrid = np.concatenate([[p] for p in padded]) cols = splinesky[0].columns() print('Columns:', cols) for c in ['gridvals', 'xgrid', 'ygrid']: cols.remove(c) T.add_columns_from(merge_tables(splinesky, columns=cols)) T.skyclass = np.array([h[0] for h in skyhdrvals]) T.legpipev = np.array([h[1] for h in skyhdrvals]) T.plver = np.array([h[2] for h in skyhdrvals]) T.expnum = np.array([h[3] for h in skyhdrvals]) T.ccdname = np.array([h[4] for h in skyhdrvals]) fn = skyoutfn trymakedirs(fn, dir=True) T.writeto(fn) print('Wrote', fn)