def main(): """Main program. """ import argparse parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--force', action='store_true', help='Run calib processes even if files already exist?') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--expnum', type=int, help='Cut to a single exposure') parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name') parser.add_argument('--no-astrom', dest='astrom', action='store_false', help='Do not compute astrometric calibs') parser.add_argument('--no-psf', dest='psfex', action='store_false', help='Do not compute PsfEx calibs') parser.add_argument('--no-sky', dest='sky', action='store_false', help='Do not compute sky models') parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor') parser.add_argument('--splinesky', action='store_true', help='Spline sky, not constant') parser.add_argument('args',nargs=argparse.REMAINDER) opt = parser.parse_args() D = Decals() if opt.ccds is not None: T = fits_table(opt.ccds) print('Read', len(T), 'from', opt.ccds) else: T = D.get_ccds() #print len(T), 'CCDs' if len(opt.args) == 0: if opt.expnum is not None: T.cut(T.expnum == opt.expnum) print('Cut to', len(T), 'with expnum =', opt.expnum) if opt.extname is not None: T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname])) print('Cut to', len(T), 'with extname =', opt.extname) opt.args = range(len(T)) for a in opt.args: i = int(a) t = T[i] im = D.get_image_object(t) print('Running', im.calname) kwargs = dict(pvastrom=opt.astrom, psfex=opt.psfex, sky=opt.sky) if opt.force: kwargs.update(force=True) if opt.run_se: kwargs.update(se=True) if opt.splinesky: kwargs.update(splinesky=True) run_calibs((im, kwargs)) return 0
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).") parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') opt = parser.parse_args() decals = Decals() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = decals.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = decals.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo,dhi = -15, 40 rlo,rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict(test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo,dhi = -90,90 rlo,rhi = 0, 360 elif opt.region == 'badsky': # A handful of exposures in DR2 with inconsistent sky estimates. #C = decals.get_ccds() #log(len(C), 'CCDs') #C.cut((C.expnum >= 257400) * (C.expnum < 257500)) T.cut(np.array([e in [257460, 257461, 257462, 257463, 257464, 257465, 257466, 257467, 257469, 257472, 257483, 257496] for e in T.expnum])) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25)/2. I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'badsky2': # UGH, missed this one in original 'badsky' definition. T.cut(T.expnum == 257466) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25)/2. I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo,rhi = 240,245 dlo,dhi = 5, 12 elif opt.region == 'edr-south': rlo,rhi = 240,245 dlo,dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo,rhi = 149.75, 150.75 dlo,dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo,rhi = 240,250 dlo,dhi = 10,15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo,rhi = 320., 360. dlo,dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo,rhi = 240,360 dlo,dhi = 20,40 elif opt.region == 'north': rlo,rhi = 120,240 dlo,dhi = 20,40 elif opt.region == 'northeast': rlo,rhi = 0,120 dlo,dhi = 20,40 elif opt.region == 'southwest': rlo,rhi = 240,360 dlo,dhi = -20,0 elif opt.region == 'south': rlo,rhi = 120,240 dlo,dhi = -20,0 elif opt.region == 'southeast': rlo,rhi = 0,120 dlo,dhi = -20,0 elif opt.region == 'midwest': rlo,rhi = 240,360 dlo,dhi = 0,20 elif opt.region == 'middle': rlo,rhi = 120,240 dlo,dhi = 0,20 elif opt.region == 'mideast': rlo,rhi = 0,120 dlo,dhi = 0,20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo,rhi = 250,260 dlo,dhi = 30,35 elif opt.region == 'virgo': rlo,rhi = 185,190 dlo,dhi = 10, 15 elif opt.region == 'virgo2': rlo,rhi = 182,192 dlo,dhi = 8, 18 elif opt.region == 'lsb': rlo,rhi = 147.2, 147.8 dlo,dhi = -0.4, 0.4 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, 0.25) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('decals-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing B.cut(np.argsort(-B.dec)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue print(b.brickname) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): expstr = '%08i' % T.expnum[i] #outdir = os.path.join('forced', expstr[:5], expstr) #trymakedirs(outdir) outfn = os.path.join('forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(decals.decals_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] f.write('python legacypipe/forced-photom-decam.py %s %i DR1 %s\n' % (imgfn, T.cpimage_hdu[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if opt.command: s = '; '.join(batch) else: s = ' '.join(batch) f.write(s + '\n') for j,i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j+1, 'of', len(allI)) im = decals.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j+1, 'of', len(allI)) im = decals.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = ('python legacypipe/run-calib.py --expnum %i --ccdname %s' % (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] if not opt.nper: f.write(s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, opt.command) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, opt.command) f.close() log('Wrote', opt.out) return 0
def main(): decals = Decals() ccds = decals.get_ccds() print(len(ccds), 'CCDs') expnums = np.unique(ccds.expnum) print(len(expnums), 'unique exposures') for expnum in expnums: expnumstr = '%08i' % expnum skyoutfn = os.path.join('splinesky', expnumstr[:5], 'decam-%s.fits' % expnumstr) psfoutfn = os.path.join('psfex', expnumstr[:5], 'decam-%s.fits' % expnumstr) if os.path.exists(skyoutfn) and os.path.exists(psfoutfn): print('Exposure', expnum, 'is done already') continue C = ccds[ccds.expnum == expnum] print(len(C), 'CCDs in expnum', expnum) psfex = [] psfhdrvals = [] splinesky = [] skyhdrvals = [] for ccd in C: im = decals.get_image_object(ccd) fn = im.splineskyfn if os.path.exists(fn): T = fits_table(fn) splinesky.append(T) # print(fn) # T.about() hdr = fitsio.read_header(fn) skyhdrvals.append( [hdr[k] for k in ['SKY', 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname]) else: print('File not found:', fn) fn = im.psffn if os.path.exists(fn): T = fits_table(fn) hdr = fitsio.read_header(fn, ext=1) keys = [ 'LOADED', 'ACCEPTED', 'CHI2', 'POLNAXIS', 'POLNGRP', 'PSF_FWHM', 'PSF_SAMP', 'PSFNAXIS', 'PSFAXIS1', 'PSFAXIS2', 'PSFAXIS3', ] if hdr['POLNAXIS'] == 0: # No polynomials. Fake it. T.polgrp1 = np.array([0]) T.polgrp2 = np.array([0]) T.polname1 = np.array(['fake']) T.polname2 = np.array(['fake']) T.polzero1 = np.array([0]) T.polzero2 = np.array([0]) T.polscal1 = np.array([1]) T.polscal2 = np.array([1]) T.poldeg1 = np.array([0]) T.poldeg2 = np.array([0]) else: keys.extend([ 'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1', 'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2', 'POLDEG1' ]) for k in keys: T.set(k.lower(), np.array([hdr[k]])) psfex.append(T) #print(fn) #T.about() hdr = fitsio.read_header(fn) psfhdrvals.append( [hdr.get(k, '') for k in ['LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname]) else: print('File not found:', fn) if len(psfex): padded = pad_arrays([p.psf_mask[0] for p in psfex]) cols = psfex[0].columns() cols.remove('psf_mask') T = merge_tables(psfex, columns=cols) T.psf_mask = np.concatenate([[p] for p in padded]) T.legpipev = np.array([h[0] for h in psfhdrvals]) T.plver = np.array([h[1] for h in psfhdrvals]) T.expnum = np.array([h[2] for h in psfhdrvals]) T.ccdname = np.array([h[3] for h in psfhdrvals]) fn = psfoutfn trymakedirs(fn, dir=True) T.writeto(fn) print('Wrote', fn) if len(splinesky): T = fits_table() T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky]) T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky]) padded = pad_arrays([t.gridvals[0] for t in splinesky]) T.gridvals = np.concatenate([[p] for p in padded]) padded = pad_arrays([t.xgrid[0] for t in splinesky]) T.xgrid = np.concatenate([[p] for p in padded]) padded = pad_arrays([t.xgrid[0] for t in splinesky]) T.ygrid = np.concatenate([[p] for p in padded]) cols = splinesky[0].columns() print('Columns:', cols) for c in ['gridvals', 'xgrid', 'ygrid']: cols.remove(c) T.add_columns_from(merge_tables(splinesky, columns=cols)) T.skyclass = np.array([h[0] for h in skyhdrvals]) T.legpipev = np.array([h[1] for h in skyhdrvals]) T.plver = np.array([h[2] for h in skyhdrvals]) T.expnum = np.array([h[3] for h in skyhdrvals]) T.ccdname = np.array([h[4] for h in skyhdrvals]) fn = skyoutfn trymakedirs(fn, dir=True) T.writeto(fn) print('Wrote', fn)
def main(outfn='ccds-annotated.fits', ccds=None): decals = Decals() if ccds is None: ccds = decals.get_ccds() # File from the "observing" svn repo: # https://desi.lbl.gov/svn/decam/code/observing/trunk tiles = fits_table('decam-tiles_obstatus.fits') #ccds.cut(np.arange(100)) #print("HACK!") #ccds.cut(np.array([name in ['N15', 'N16', 'N21', 'N9'] # for name in ccds.ccdname]) * # ccds.expnum == 229683) I = decals.photometric_ccds(ccds) ccds.photometric = np.zeros(len(ccds), bool) ccds.photometric[I] = True I = decals.apply_blacklist(ccds) ccds.blacklist_ok = np.zeros(len(ccds), bool) ccds.blacklist_ok[I] = True ccds.good_region = np.empty((len(ccds), 4), np.int16) ccds.good_region[:,:] = -1 ccds.ra0 = np.zeros(len(ccds), np.float64) ccds.dec0 = np.zeros(len(ccds), np.float64) ccds.ra1 = np.zeros(len(ccds), np.float64) ccds.dec1 = np.zeros(len(ccds), np.float64) ccds.ra2 = np.zeros(len(ccds), np.float64) ccds.dec2 = np.zeros(len(ccds), np.float64) ccds.ra3 = np.zeros(len(ccds), np.float64) ccds.dec3 = np.zeros(len(ccds), np.float64) ccds.dra = np.zeros(len(ccds), np.float32) ccds.ddec = np.zeros(len(ccds), np.float32) ccds.ra_center = np.zeros(len(ccds), np.float64) ccds.dec_center = np.zeros(len(ccds), np.float64) ccds.sig1 = np.zeros(len(ccds), np.float32) ccds.meansky = np.zeros(len(ccds), np.float32) ccds.stdsky = np.zeros(len(ccds), np.float32) ccds.maxsky = np.zeros(len(ccds), np.float32) ccds.minsky = np.zeros(len(ccds), np.float32) ccds.pixscale_mean = np.zeros(len(ccds), np.float32) ccds.pixscale_std = np.zeros(len(ccds), np.float32) ccds.pixscale_max = np.zeros(len(ccds), np.float32) ccds.pixscale_min = np.zeros(len(ccds), np.float32) ccds.psfnorm_mean = np.zeros(len(ccds), np.float32) ccds.psfnorm_std = np.zeros(len(ccds), np.float32) ccds.galnorm_mean = np.zeros(len(ccds), np.float32) ccds.galnorm_std = np.zeros(len(ccds), np.float32) gaussgalnorm = np.zeros(len(ccds), np.float32) # 2nd moments ccds.psf_mx2 = np.zeros(len(ccds), np.float32) ccds.psf_my2 = np.zeros(len(ccds), np.float32) ccds.psf_mxy = np.zeros(len(ccds), np.float32) # ccds.psf_a = np.zeros(len(ccds), np.float32) ccds.psf_b = np.zeros(len(ccds), np.float32) ccds.psf_theta = np.zeros(len(ccds), np.float32) ccds.psf_ell = np.zeros(len(ccds), np.float32) ccds.humidity = np.zeros(len(ccds), np.float32) ccds.outtemp = np.zeros(len(ccds), np.float32) ccds.tileid = np.zeros(len(ccds), np.int32) ccds.tilepass = np.zeros(len(ccds), np.uint8) ccds.tileebv = np.zeros(len(ccds), np.float32) plvers = [] for iccd,ccd in enumerate(ccds): im = decals.get_image_object(ccd) print('Reading CCD %i of %i:' % (iccd+1, len(ccds)), im) X = im.get_good_image_subregion() for i,x in enumerate(X): if x is not None: ccds.good_region[iccd,i] = x W,H = ccd.width, ccd.height psf = None wcs = None sky = None try: tim = im.get_tractor_image(pixPsf=True, splinesky=True, subsky=False, pixels=False) except: import traceback traceback.print_exc() plvers.append('') continue if tim is None: plvers.append('') continue psf = tim.psf wcs = tim.wcs.wcs sky = tim.sky hdr = tim.primhdr # print('Got PSF', psf) # print('Got sky', type(sky)) # print('Got WCS', wcs) ccds.humidity[iccd] = hdr.get('HUMIDITY') ccds.outtemp[iccd] = hdr.get('OUTTEMP') ccds.sig1[iccd] = tim.sig1 plvers.append(tim.plver) obj = hdr.get('OBJECT') # parse 'DECaLS_15150_r' words = obj.split('_') tile = None if len(words) == 3 and words[0] == 'DECaLS': try: tileid = int(words[1]) tile = tiles[tileid - 1] if tile.tileid != tileid: I = np.flatnonzero(tile.tileid == tileid) tile = tiles[I[0]] except: pass if tile is not None: ccds.tileid [iccd] = tile.tileid ccds.tilepass[iccd] = tile.get('pass') ccds.tileebv [iccd] = tile.ebv_med # Instantiate PSF on a grid S = 32 xx = np.linspace(1+S, W-S, 5) yy = np.linspace(1+S, H-S, 5) xx,yy = np.meshgrid(xx, yy) psfnorms = [] galnorms = [] for x,y in zip(xx.ravel(), yy.ravel()): p = im.psf_norm(tim, x=x, y=y) g = im.galaxy_norm(tim, x=x, y=y) psfnorms.append(p) galnorms.append(g) ccds.psfnorm_mean[iccd] = np.mean(psfnorms) ccds.psfnorm_std [iccd] = np.std (psfnorms) ccds.galnorm_mean[iccd] = np.mean(galnorms) ccds.galnorm_std [iccd] = np.std (galnorms) # PSF in center of field cx,cy = (W+1)/2., (H+1)/2. p = psf.getPointSourcePatch(cx, cy).patch ph,pw = p.shape px,py = np.meshgrid(np.arange(pw), np.arange(ph)) psum = np.sum(p) # print('psum', psum) p /= psum # centroids cenx = np.sum(p * px) ceny = np.sum(p * py) # print('cenx,ceny', cenx,ceny) # second moments x2 = np.sum(p * (px - cenx)**2) y2 = np.sum(p * (py - ceny)**2) xy = np.sum(p * (px - cenx)*(py - ceny)) # semi-major/minor axes and position angle theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.) theta = np.abs(theta) * np.sign(xy) s = np.sqrt(((x2 - y2)/2.)**2 + xy**2) a = np.sqrt((x2 + y2) / 2. + s) b = np.sqrt((x2 + y2) / 2. - s) ell = 1. - b/a # print('PSF second moments', x2, y2, xy) # print('PSF position angle', theta) # print('PSF semi-axes', a, b) # print('PSF ellipticity', ell) ccds.psf_mx2[iccd] = x2 ccds.psf_my2[iccd] = y2 ccds.psf_mxy[iccd] = xy ccds.psf_a[iccd] = a ccds.psf_b[iccd] = b ccds.psf_theta[iccd] = theta ccds.psf_ell [iccd] = ell # Galaxy norm using Gaussian approximation of PSF. realpsf = tim.psf tim.psf = im.read_psf_model(0, 0, gaussPsf=True, psf_sigma=tim.psf_sigma) gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy) tim.psf = realpsf # Sky mod = np.zeros((ccd.height, ccd.width), np.float32) sky.addTo(mod) ccds.meansky[iccd] = np.mean(mod) ccds.stdsky[iccd] = np.std(mod) ccds.maxsky[iccd] = mod.max() ccds.minsky[iccd] = mod.min() # WCS ccds.ra0[iccd],ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1) ccds.ra1[iccd],ccds.dec1[iccd] = wcs.pixelxy2radec(1, H) ccds.ra2[iccd],ccds.dec2[iccd] = wcs.pixelxy2radec(W, H) ccds.ra3[iccd],ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1) midx, midy = (W+1)/2., (H+1)/2. rc,dc = wcs.pixelxy2radec(midx, midy) ra,dec = wcs.pixelxy2radec([1,W,midx,midx], [midy,midy,1,H]) ccds.dra [iccd] = max(degrees_between(ra, dc+np.zeros_like(ra), rc, dc)) ccds.ddec[iccd] = max(degrees_between(rc+np.zeros_like(dec), dec, rc, dc)) ccds.ra_center [iccd] = rc ccds.dec_center[iccd] = dc # Compute scale change across the chip # how many pixels to step step = 10 xx = np.linspace(1+step, W-step, 5) yy = np.linspace(1+step, H-step, 5) xx,yy = np.meshgrid(xx, yy) pixscale = [] for x,y in zip(xx.ravel(), yy.ravel()): sx = [x-step, x-step, x+step, x+step, x-step] sy = [y-step, y+step, y+step, y-step, y-step] sr,sd = wcs.pixelxy2radec(sx, sy) rc,dc = wcs.pixelxy2radec(x, y) # project around a tiny little TAN WCS at (x,y), with 1" pixels locwcs = Tan(rc, dc, 0., 0., 1./3600, 0., 0., 1./3600, 1., 1.) ok,lx,ly = locwcs.radec2pixelxy(sr, sd) #print('local x,y:', lx, ly) A = polygon_area((lx, ly)) pixscale.append(np.sqrt(A / (2*step)**2)) # print('Pixel scales:', pixscale) ccds.pixscale_mean[iccd] = np.mean(pixscale) ccds.pixscale_min[iccd] = min(pixscale) ccds.pixscale_max[iccd] = max(pixscale) ccds.pixscale_std[iccd] = np.std(pixscale) ccds.plver = np.array(plvers) sfd = tractor.sfd.SFDMap() allbands = 'ugrizY' filts = ['%s %s' % ('DES', f) for f in allbands] wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4'] ebv,ext = sfd.extinction(filts + wisebands, ccds.ra_center, ccds.dec_center, get_ebv=True) ext = ext.astype(np.float32) ccds.ebv = ebv.astype(np.float32) ccds.decam_extinction = ext[:,:len(allbands)] ccds.wise_extinction = ext[:,len(allbands):] # Depth detsig1 = ccds.sig1 / ccds.psfnorm_mean depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.psfdepth = -2.5 * (np.log10(depth) - 9) detsig1 = ccds.sig1 / ccds.galnorm_mean depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.galdepth = -2.5 * (np.log10(depth) - 9) # Depth using Gaussian FWHM. psf_sigma = ccds.fwhm / 2.35 gnorm = 1./(2. * np.sqrt(np.pi) * psf_sigma) detsig1 = ccds.sig1 / gnorm depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9) # Gaussian galaxy depth detsig1 = ccds.sig1 / gaussgalnorm depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9) ccds.writeto(outfn)
except: import traceback traceback.print_exc() if __name__ == '__main__': TT = [fits_table('ccds-annotated/ccds-annotated-%03i.fits' % i) for i in range(515)] T = merge_tables(TT) T.writeto('ccds-annotated.fits') import sys sys.exit() #sys.exit(main()) decals = Decals() ccds = decals.get_ccds() from astrometry.util.multiproc import * #mp = multiproc(8) mp = multiproc(4) N = 1000 args = [] i = 0 while len(ccds): c = ccds[:N] ccds = ccds[N:] args.append((i, c)) i += 1 print('Split CCDs file into', len(args), 'pieces') print('sizes:', [len(c) for i,c in args]) mp.map(_bounce_main, args)
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser( description= "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)." ) parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') opt = parser.parse_args() decals = Decals() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = decals.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = decals.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo, dhi = -15, 40 rlo, rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict( test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo, dhi = -90, 90 rlo, rhi = 0, 360 elif opt.region == 'badsky': # A handful of exposures in DR2 with inconsistent sky estimates. #C = decals.get_ccds() #log(len(C), 'CCDs') #C.cut((C.expnum >= 257400) * (C.expnum < 257500)) T.cut( np.array([ e in [ 257460, 257461, 257462, 257463, 257464, 257465, 257466, 257467, 257469, 257472, 257483, 257496 ] for e in T.expnum ])) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25) / 2. I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'badsky2': # UGH, missed this one in original 'badsky' definition. T.cut(T.expnum == 257466) log(len(T), 'CCDs with bad sky') # CCD radius radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600. # Brick radius radius += np.hypot(0.25, 0.25) / 2. I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05) keep = np.zeros(len(B), bool) keep[I] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs with bad sky') elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo, rhi = 240, 245 dlo, dhi = 5, 12 elif opt.region == 'edr-south': rlo, rhi = 240, 245 dlo, dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo, rhi = 149.75, 150.75 dlo, dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo, rhi = 240, 250 dlo, dhi = 10, 15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo, rhi = 320., 360. dlo, dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo, rhi = 240, 360 dlo, dhi = 20, 40 elif opt.region == 'north': rlo, rhi = 120, 240 dlo, dhi = 20, 40 elif opt.region == 'northeast': rlo, rhi = 0, 120 dlo, dhi = 20, 40 elif opt.region == 'southwest': rlo, rhi = 240, 360 dlo, dhi = -20, 0 elif opt.region == 'south': rlo, rhi = 120, 240 dlo, dhi = -20, 0 elif opt.region == 'southeast': rlo, rhi = 0, 120 dlo, dhi = -20, 0 elif opt.region == 'midwest': rlo, rhi = 240, 360 dlo, dhi = 0, 20 elif opt.region == 'middle': rlo, rhi = 120, 240 dlo, dhi = 0, 20 elif opt.region == 'mideast': rlo, rhi = 0, 120 dlo, dhi = 0, 20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks decals-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo, rhi = 250, 260 dlo, dhi = 30, 35 elif opt.region == 'virgo': rlo, rhi = 185, 190 dlo, dhi = 10, 15 elif opt.region == 'virgo2': rlo, rhi = 182, 192 dlo, dhi = 8, 18 elif opt.region == 'lsb': rlo, rhi = 147.2, 147.8 dlo, dhi = -0.4, 0.4 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut( np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, 0.25) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, 0.25) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('decals-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing B.cut(np.argsort(-B.dec)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % ( b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue print(b.brickname) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): expstr = '%08i' % T.expnum[i] #outdir = os.path.join('forced', expstr[:5], expstr) #trymakedirs(outdir) outfn = os.path.join( 'forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(decals.decals_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] f.write('python legacypipe/forced-photom-decam.py %s %i DR1 %s\n' % (imgfn, T.cpimage_hdu[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write( 'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if opt.command: s = '; '.join(batch) else: s = ' '.join(batch) f.write(s + '\n') for j, i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j + 1, 'of', len(allI)) im = decals.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j + 1, 'of', len(allI)) im = decals.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = ('python legacypipe/run-calib.py --expnum %i --ccdname %s' % (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] if not opt.nper: f.write(s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, opt.command) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, opt.command) f.close() log('Wrote', opt.out) return 0
def main(outfn='ccds-annotated.fits', ccds=None): decals = Decals() if ccds is None: ccds = decals.get_ccds() # File from the "observing" svn repo: # https://desi.lbl.gov/svn/decam/code/observing/trunk tiles = fits_table('decam-tiles_obstatus.fits') #ccds.cut(np.arange(100)) #print("HACK!") #ccds.cut(np.array([name in ['N15', 'N16', 'N21', 'N9'] # for name in ccds.ccdname]) * # ccds.expnum == 229683) I = decals.photometric_ccds(ccds) ccds.photometric = np.zeros(len(ccds), bool) ccds.photometric[I] = True I = decals.apply_blacklist(ccds) ccds.blacklist_ok = np.zeros(len(ccds), bool) ccds.blacklist_ok[I] = True ccds.good_region = np.empty((len(ccds), 4), np.int16) ccds.good_region[:, :] = -1 ccds.ra0 = np.zeros(len(ccds), np.float64) ccds.dec0 = np.zeros(len(ccds), np.float64) ccds.ra1 = np.zeros(len(ccds), np.float64) ccds.dec1 = np.zeros(len(ccds), np.float64) ccds.ra2 = np.zeros(len(ccds), np.float64) ccds.dec2 = np.zeros(len(ccds), np.float64) ccds.ra3 = np.zeros(len(ccds), np.float64) ccds.dec3 = np.zeros(len(ccds), np.float64) ccds.dra = np.zeros(len(ccds), np.float32) ccds.ddec = np.zeros(len(ccds), np.float32) ccds.ra_center = np.zeros(len(ccds), np.float64) ccds.dec_center = np.zeros(len(ccds), np.float64) ccds.sig1 = np.zeros(len(ccds), np.float32) ccds.meansky = np.zeros(len(ccds), np.float32) ccds.stdsky = np.zeros(len(ccds), np.float32) ccds.maxsky = np.zeros(len(ccds), np.float32) ccds.minsky = np.zeros(len(ccds), np.float32) ccds.pixscale_mean = np.zeros(len(ccds), np.float32) ccds.pixscale_std = np.zeros(len(ccds), np.float32) ccds.pixscale_max = np.zeros(len(ccds), np.float32) ccds.pixscale_min = np.zeros(len(ccds), np.float32) ccds.psfnorm_mean = np.zeros(len(ccds), np.float32) ccds.psfnorm_std = np.zeros(len(ccds), np.float32) ccds.galnorm_mean = np.zeros(len(ccds), np.float32) ccds.galnorm_std = np.zeros(len(ccds), np.float32) gaussgalnorm = np.zeros(len(ccds), np.float32) # 2nd moments ccds.psf_mx2 = np.zeros(len(ccds), np.float32) ccds.psf_my2 = np.zeros(len(ccds), np.float32) ccds.psf_mxy = np.zeros(len(ccds), np.float32) # ccds.psf_a = np.zeros(len(ccds), np.float32) ccds.psf_b = np.zeros(len(ccds), np.float32) ccds.psf_theta = np.zeros(len(ccds), np.float32) ccds.psf_ell = np.zeros(len(ccds), np.float32) ccds.humidity = np.zeros(len(ccds), np.float32) ccds.outtemp = np.zeros(len(ccds), np.float32) ccds.tileid = np.zeros(len(ccds), np.int32) ccds.tilepass = np.zeros(len(ccds), np.uint8) ccds.tileebv = np.zeros(len(ccds), np.float32) plvers = [] for iccd, ccd in enumerate(ccds): im = decals.get_image_object(ccd) print('Reading CCD %i of %i:' % (iccd + 1, len(ccds)), im) X = im.get_good_image_subregion() for i, x in enumerate(X): if x is not None: ccds.good_region[iccd, i] = x W, H = ccd.width, ccd.height psf = None wcs = None sky = None try: tim = im.get_tractor_image(pixPsf=True, splinesky=True, subsky=False, pixels=False) except: import traceback traceback.print_exc() plvers.append('') continue if tim is None: plvers.append('') continue psf = tim.psf wcs = tim.wcs.wcs sky = tim.sky hdr = tim.primhdr # print('Got PSF', psf) # print('Got sky', type(sky)) # print('Got WCS', wcs) ccds.humidity[iccd] = hdr.get('HUMIDITY') ccds.outtemp[iccd] = hdr.get('OUTTEMP') ccds.sig1[iccd] = tim.sig1 plvers.append(tim.plver) obj = hdr.get('OBJECT') # parse 'DECaLS_15150_r' words = obj.split('_') tile = None if len(words) == 3 and words[0] == 'DECaLS': try: tileid = int(words[1]) tile = tiles[tileid - 1] if tile.tileid != tileid: I = np.flatnonzero(tile.tileid == tileid) tile = tiles[I[0]] except: pass if tile is not None: ccds.tileid[iccd] = tile.tileid ccds.tilepass[iccd] = tile.get('pass') ccds.tileebv[iccd] = tile.ebv_med # Instantiate PSF on a grid S = 32 xx = np.linspace(1 + S, W - S, 5) yy = np.linspace(1 + S, H - S, 5) xx, yy = np.meshgrid(xx, yy) psfnorms = [] galnorms = [] for x, y in zip(xx.ravel(), yy.ravel()): p = im.psf_norm(tim, x=x, y=y) g = im.galaxy_norm(tim, x=x, y=y) psfnorms.append(p) galnorms.append(g) ccds.psfnorm_mean[iccd] = np.mean(psfnorms) ccds.psfnorm_std[iccd] = np.std(psfnorms) ccds.galnorm_mean[iccd] = np.mean(galnorms) ccds.galnorm_std[iccd] = np.std(galnorms) # PSF in center of field cx, cy = (W + 1) / 2., (H + 1) / 2. p = psf.getPointSourcePatch(cx, cy).patch ph, pw = p.shape px, py = np.meshgrid(np.arange(pw), np.arange(ph)) psum = np.sum(p) # print('psum', psum) p /= psum # centroids cenx = np.sum(p * px) ceny = np.sum(p * py) # print('cenx,ceny', cenx,ceny) # second moments x2 = np.sum(p * (px - cenx)**2) y2 = np.sum(p * (py - ceny)**2) xy = np.sum(p * (px - cenx) * (py - ceny)) # semi-major/minor axes and position angle theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.) theta = np.abs(theta) * np.sign(xy) s = np.sqrt(((x2 - y2) / 2.)**2 + xy**2) a = np.sqrt((x2 + y2) / 2. + s) b = np.sqrt((x2 + y2) / 2. - s) ell = 1. - b / a # print('PSF second moments', x2, y2, xy) # print('PSF position angle', theta) # print('PSF semi-axes', a, b) # print('PSF ellipticity', ell) ccds.psf_mx2[iccd] = x2 ccds.psf_my2[iccd] = y2 ccds.psf_mxy[iccd] = xy ccds.psf_a[iccd] = a ccds.psf_b[iccd] = b ccds.psf_theta[iccd] = theta ccds.psf_ell[iccd] = ell # Galaxy norm using Gaussian approximation of PSF. realpsf = tim.psf tim.psf = im.read_psf_model(0, 0, gaussPsf=True, psf_sigma=tim.psf_sigma) gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy) tim.psf = realpsf # Sky mod = np.zeros((ccd.height, ccd.width), np.float32) sky.addTo(mod) ccds.meansky[iccd] = np.mean(mod) ccds.stdsky[iccd] = np.std(mod) ccds.maxsky[iccd] = mod.max() ccds.minsky[iccd] = mod.min() # WCS ccds.ra0[iccd], ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1) ccds.ra1[iccd], ccds.dec1[iccd] = wcs.pixelxy2radec(1, H) ccds.ra2[iccd], ccds.dec2[iccd] = wcs.pixelxy2radec(W, H) ccds.ra3[iccd], ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1) midx, midy = (W + 1) / 2., (H + 1) / 2. rc, dc = wcs.pixelxy2radec(midx, midy) ra, dec = wcs.pixelxy2radec([1, W, midx, midx], [midy, midy, 1, H]) ccds.dra[iccd] = max( degrees_between(ra, dc + np.zeros_like(ra), rc, dc)) ccds.ddec[iccd] = max( degrees_between(rc + np.zeros_like(dec), dec, rc, dc)) ccds.ra_center[iccd] = rc ccds.dec_center[iccd] = dc # Compute scale change across the chip # how many pixels to step step = 10 xx = np.linspace(1 + step, W - step, 5) yy = np.linspace(1 + step, H - step, 5) xx, yy = np.meshgrid(xx, yy) pixscale = [] for x, y in zip(xx.ravel(), yy.ravel()): sx = [x - step, x - step, x + step, x + step, x - step] sy = [y - step, y + step, y + step, y - step, y - step] sr, sd = wcs.pixelxy2radec(sx, sy) rc, dc = wcs.pixelxy2radec(x, y) # project around a tiny little TAN WCS at (x,y), with 1" pixels locwcs = Tan(rc, dc, 0., 0., 1. / 3600, 0., 0., 1. / 3600, 1., 1.) ok, lx, ly = locwcs.radec2pixelxy(sr, sd) #print('local x,y:', lx, ly) A = polygon_area((lx, ly)) pixscale.append(np.sqrt(A / (2 * step)**2)) # print('Pixel scales:', pixscale) ccds.pixscale_mean[iccd] = np.mean(pixscale) ccds.pixscale_min[iccd] = min(pixscale) ccds.pixscale_max[iccd] = max(pixscale) ccds.pixscale_std[iccd] = np.std(pixscale) ccds.plver = np.array(plvers) sfd = tractor.sfd.SFDMap() allbands = 'ugrizY' filts = ['%s %s' % ('DES', f) for f in allbands] wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4'] ebv, ext = sfd.extinction(filts + wisebands, ccds.ra_center, ccds.dec_center, get_ebv=True) ext = ext.astype(np.float32) ccds.ebv = ebv.astype(np.float32) ccds.decam_extinction = ext[:, :len(allbands)] ccds.wise_extinction = ext[:, len(allbands):] # Depth detsig1 = ccds.sig1 / ccds.psfnorm_mean depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.psfdepth = -2.5 * (np.log10(depth) - 9) detsig1 = ccds.sig1 / ccds.galnorm_mean depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.galdepth = -2.5 * (np.log10(depth) - 9) # Depth using Gaussian FWHM. psf_sigma = ccds.fwhm / 2.35 gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) detsig1 = ccds.sig1 / gnorm depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9) # Gaussian galaxy depth detsig1 = ccds.sig1 / gaussgalnorm depth = 5. * detsig1 # that's flux in nanomaggies -- convert to mag ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9) ccds.writeto(outfn)
if __name__ == '__main__': TT = [ fits_table('ccds-annotated/ccds-annotated-%03i.fits' % i) for i in range(515) ] T = merge_tables(TT) T.writeto('ccds-annotated.fits') import sys sys.exit() #sys.exit(main()) decals = Decals() ccds = decals.get_ccds() from astrometry.util.multiproc import * #mp = multiproc(8) mp = multiproc(4) N = 1000 args = [] i = 0 while len(ccds): c = ccds[:N] ccds = ccds[N:] args.append((i, c)) i += 1 print('Split CCDs file into', len(args), 'pieces') print('sizes:', [len(c) for i, c in args]) mp.map(_bounce_main, args)
def main(): import optparse parser = optparse.OptionParser() parser.add_option('--zoom', '-z', type=int, action='append', default=[], help='Add zoom level; default 13') parser.add_option('--threads', type=int, default=1, help='Number of threads') parser.add_option('--y0', type=int, default=0, help='Start row') parser.add_option('--y1', type=int, default=None, help='End row (non-inclusive)') parser.add_option('--x0', type=int, default=None) parser.add_option('--x1', type=int, default=None) parser.add_option('-x', type=int) parser.add_option('-y', type=int) parser.add_option('--mindec', type=float, default=None, help='Minimum Dec to run') parser.add_option('--maxdec', type=float, default=None, help='Maximum Dec to run') parser.add_option('--minra', type=float, default=0., help='Minimum RA to run') parser.add_option('--maxra', type=float, default=360., help='Maximum RA to run') parser.add_option('--near', action='store_true', help='Only run tiles near bricks') parser.add_option('--near-ccds', action='store_true', help='Only run tiles near CCDs') parser.add_option('--queue', action='store_true', default=False, help='Print qdo commands') parser.add_option('--all', action='store_true', help='Render all tiles') parser.add_option('--ignore', action='store_true', help='Ignore cached tile files', default=False) parser.add_option('--top', action='store_true', help='Top levels of the pyramid') parser.add_option('--kind', default='image') parser.add_option('--scale', action='store_true', help='Scale images?') parser.add_option('--coadd', action='store_true', help='Create SDSS coadd images?') parser.add_option('--grass', action='store_true', help='progress plot') opt,args = parser.parse_args() if len(opt.zoom) == 0: opt.zoom = [13] mp = multiproc(opt.threads) if opt.kind == 'sdss': if opt.maxdec is None: opt.maxdec = 90 if opt.mindec is None: opt.mindec = -25 elif opt.kind in ['halpha', 'unwise-neo1']: if opt.maxdec is None: opt.maxdec = 90 if opt.mindec is None: opt.mindec = -90 else: if opt.maxdec is None: opt.maxdec = 40 if opt.mindec is None: opt.mindec = -20 if opt.top: top_levels(mp, opt) sys.exit(0) from legacypipe.common import Decals decals = Decals() if opt.near: if opt.kind == 'sdss': B = fits_table(os.path.join(settings.DATA_DIR, 'bricks-sdssco.fits')) else: B = decals.get_bricks() print len(B), 'bricks' if opt.scale: opt.near_ccds = True if opt.near_ccds: if opt.kind == 'sdss': C = fits_table(os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'), columns=['rerun','ra','dec', 'run', 'camcol', 'field', 'score']) C.cut(C.rerun == '301') C.cut(C.score >= 0.6) #C.delete_column('rerun') # SDSS field size radius = 1.01 * np.hypot(10., 14.)/2. / 60. ccdsize = radius print len(C), 'SDSS fields' else: C = decals.get_ccds() print len(C), 'CCDs' ccdsize = 0.2 if opt.x is not None: opt.x0 = opt.x opt.x1 = opt.x + 1 if opt.y is not None: opt.y0 = opt.y opt.y1 = opt.y + 1 if opt.coadd and opt.kind == 'sdss': from legacypipe.common import wcs_for_brick from map.views import trymakedirs B = decals.get_bricks() print len(B), 'bricks' B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec)) print len(B), 'in Dec range' B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra)) print len(B), 'in RA range' if opt.queue: # ~ square-degree tiles # RA slices rr = np.arange(opt.minra , opt.maxra +1) dd = np.arange(opt.mindec, opt.maxdec+1) for rlo,rhi in zip(rr, rr[1:]): for dlo,dhi in zip(dd, dd[1:]): print 'time python render-tiles.py --kind sdss --coadd --minra %f --maxra %f --mindec %f --maxdec %f' % (rlo, rhi, dlo, dhi) sys.exit(0) if opt.grass: basedir = settings.DATA_DIR codir = os.path.join(basedir, 'coadd', 'sdssco') rr,dd = [],[] exist = [] for i,b in enumerate(B): print 'Brick', b.brickname, fn = os.path.join(codir, b.brickname[:3], 'sdssco-%s-%s.fits' % (b.brickname, 'r')) print '-->', fn, if not os.path.exists(fn): print continue print 'found' rr.append(b.ra) dd.append(b.dec) exist.append(i) exist = np.array(exist) B.cut(exist) B.writeto('bricks-sdssco-exist.fits') import pylab as plt plt.clf() plt.plot(rr, dd, 'k.') plt.title('SDSS coadd tiles') plt.savefig('sdss.png') sys.exit(0) basedir = settings.DATA_DIR codir = os.path.join(basedir, 'coadd', 'sdssco') for b in B: print 'Brick', b.brickname wcs = wcs_for_brick(b, W=2400, H=2400, pixscale=0.396) bands = 'gri' dirnm = os.path.join(codir, b.brickname[:3]) fns = [os.path.join(dirnm, 'sdssco-%s-%s.fits' % (b.brickname, band)) for band in bands] hdr = fitsio.FITSHDR() hdr['SURVEY'] = 'SDSS' wcs.add_to_header(hdr) if all([os.path.exists(fn) for fn in fns]): print 'Already exist' continue ims = map_sdss(req, 1, 0, 0, 0, get_images=True, wcs=wcs, ignoreCached=True, forcescale=0) if ims is None: print 'No overlap' continue trymakedirs(os.path.join(dirnm, 'xxx')) for fn,band,im in zip(fns,bands, ims): fitsio.write(fn, im, header=hdr, clobber=True) print 'Wrote', fn # Also write scaled versions dirnm = os.path.join(basedir, 'scaled', 'sdssco') scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(brickname).3s', 'sdssco-%(brickname)s-%(band)s.fits') for im,band in zip(ims,bands): scalekwargs = dict(band=band, brick=b.brickid, brickname=b.brickname) imwcs = wcs for scale in range(1, 7): print 'Writing scale level', scale im,imwcs,sfn = get_scaled(scalepat, scalekwargs, scale, None, wcs=imwcs, img=im, return_data=True) sys.exit(0) if opt.scale: if opt.kind == 'sdss': C.cut((C.dec >= opt.mindec) * (C.dec < opt.maxdec)) print len(C), 'in Dec range' C.cut((C.ra >= opt.minra) * (C.ra < opt.maxra)) print len(C), 'in RA range' from astrometry.sdss import AsTransWrapper, DR9 sdss = DR9(basedir=settings.SDSS_DIR) sdss.saveUnzippedFiles(settings.SDSS_DIR) sdss.setFitsioReadBZ2() if settings.SDSS_PHOTOOBJS: sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS, resolve=settings.SDSS_RESOLVE) basedir = settings.DATA_DIR scaledir = 'sdss' dirnm = os.path.join(basedir, 'scaled', scaledir) for im in C: from map.views import _read_sip_wcs #if im.rerun != '301': # continue for band in 'gri': scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i', 'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits') tmpsuff = '.tmp%08i' % np.random.randint(100000000) basefn = sdss.retrieve('frame', im.run, im.camcol, field=im.field, band=band, rerun=im.rerun, tempsuffix=tmpsuff) fnargs = dict(band=band, rerun=im.rerun, run=im.run, camcol=im.camcol, field=im.field) scaled = 1 fn = get_scaled(scalepat, fnargs, scaled, basefn, read_base_wcs=read_astrans, read_wcs=_read_sip_wcs) print 'get_scaled:', fn return 0 else: assert(False) for zoom in opt.zoom: N = 2**zoom if opt.y1 is None: y1 = N else: y1 = opt.y1 if opt.x0 is None: opt.x0 = 0 x1 = opt.x1 if x1 is None: x1 = N # Find grid of Ra,Dec tile centers and select the ones near DECaLS bricks. rr,dd = [],[] yy = np.arange(opt.y0, y1) xx = np.arange(opt.x0, x1) if opt.grass: import pylab as plt tileexists = np.zeros((len(yy),len(xx)), bool) basedir = settings.DATA_DIR ver = tileversions[opt.kind][-1] tiledir = os.path.join(basedir, 'tiles', opt.kind, '%i'%ver, '%i'%zoom) for dirpath,dirnames,filenames in os.walk(tiledir): # change walk order dirnames.sort() if len(filenames) == 0: continue print 'Dirpath', dirpath #print 'Dirnames', dirnames #print 'Filenames', filenames # check for symlinks if False: fns = [] for fn in filenames: fullfn = os.path.join(tiledir, dirpath, fn) if os.path.isfile(fullfn) and not os.path.islink(fullfn): fns.append(fn) print len(fns), 'of', len(filenames), 'are files (not symlinks)' filenames = fns x = os.path.basename(dirpath) x = int(x) #print 'x', x yy = [int(fn.replace('.jpg','')) for fn in filenames] #print 'yy', yy print len(yy), 'tiles' for y in yy: tileexists[y - opt.y0, x - opt.x0] = True plt.clf() plt.imshow(tileexists, interpolation='nearest', origin='upper', vmin=0, vmax=1, cmap='gray') fn = 'exist-%s-z%02i' % (opt.kind, zoom) plt.savefig(fn+'.png') fitsio.write(fn+'.fits', tileexists, clobber=True) print 'Wrote', fn+'.png and', fn+'.fits' continue if not opt.all: for y in yy: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, y) r,d = wcs.get_center() dd.append(d) for x in xx: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, x, 0) r,d = wcs.get_center() rr.append(r) dd = np.array(dd) rr = np.array(rr) if len(dd) > 1: tilesize = max(np.abs(np.diff(dd))) print 'Tile size:', tilesize else: if opt.near_ccds or opt.near: try: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0+1) r2,d2 = wcs.get_center() except: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0-1) r2,d2 = wcs.get_center() tilesize = np.abs(dd[0] - d2) print 'Tile size:', tilesize else: tilesize = 180. I = np.flatnonzero((dd >= opt.mindec) * (dd <= opt.maxdec)) print 'Keeping', len(I), 'Dec points between', opt.mindec, 'and', opt.maxdec dd = dd[I] yy = yy[I] I = np.flatnonzero((rr >= opt.minra) * (rr <= opt.maxra)) print 'Keeping', len(I), 'RA points between', opt.minra, 'and', opt.maxra rr = rr[I] xx = xx[I] print len(rr), 'RA points x', len(dd), 'Dec points' print 'x tile range:', xx.min(), xx.max(), 'y tile range:', yy.min(), yy.max() for iy,y in enumerate(yy): print print 'Y row', y if opt.near: d = dd[iy] I,J,dist = match_radec(rr, d+np.zeros_like(rr), B.ra, B.dec, 0.25 + tilesize, nearest=True) if len(I) == 0: print 'No matches to bricks' continue keep = np.zeros(len(rr), bool) keep[I] = True print 'Keeping', sum(keep), 'tiles in row', y, 'Dec', d x = xx[keep] elif opt.near_ccds: d = dd[iy] print 'RA range of tiles:', rr.min(), rr.max() print 'Dec of tile row:', d I,J,dist = match_radec(rr, d+np.zeros_like(rr), C.ra, C.dec, ccdsize + tilesize, nearest=True) if len(I) == 0: print 'No matches to CCDs' continue keep = np.zeros(len(rr), bool) keep[I] = True print 'Keeping', sum(keep), 'tiles in row', y, 'Dec', d x = xx[keep] else: x = xx if opt.queue: cmd = 'python -u render-tiles.py --zoom %i --y0 %i --y1 %i --kind %s' % (zoom, y, y+1, opt.kind) if opt.near_ccds: cmd += ' --near-ccds' if opt.all: cmd += ' --all' if opt.ignore: cmd += ' --ignore' print cmd continue # if opt.grass: # for xi in x: # basedir = settings.DATA_DIR # ver = tileversions[opt.kind][-1] # tilefn = os.path.join(basedir, 'tiles', opt.kind, # '%i/%i/%i/%i.jpg' % (ver, zoom, xi, y)) # print 'Checking for', tilefn # if os.path.exists(tilefn): # print 'EXISTS' # tileexists[yi-opt.y0, xi-opt.x0] # continue args = [] for xi in x: args.append((opt.kind,zoom,xi,y, opt.ignore)) print 'Rendering', len(args), 'tiles in row y =', y mp.map(_bounce_one_tile, args, chunksize=min(100, max(1, len(args)/opt.threads))) print 'Rendered', len(args), 'tiles'
def main(): decals = Decals() ccds = decals.get_ccds() print(len(ccds), 'CCDs') expnums = np.unique(ccds.expnum) print(len(expnums), 'unique exposures') for expnum in expnums: expnumstr = '%08i' % expnum skyoutfn = os.path.join('splinesky', expnumstr[:5], 'decam-%s.fits' % expnumstr) psfoutfn = os.path.join('psfex', expnumstr[:5], 'decam-%s.fits' % expnumstr) if os.path.exists(skyoutfn) and os.path.exists(psfoutfn): print('Exposure', expnum, 'is done already') continue C = ccds[ccds.expnum == expnum] print(len(C), 'CCDs in expnum', expnum) psfex = [] psfhdrvals = [] splinesky = [] skyhdrvals = [] for ccd in C: im = decals.get_image_object(ccd) fn = im.splineskyfn if os.path.exists(fn): T = fits_table(fn) splinesky.append(T) # print(fn) # T.about() hdr = fitsio.read_header(fn) skyhdrvals.append([hdr[k] for k in [ 'SKY', 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname]) else: print('File not found:', fn) fn = im.psffn if os.path.exists(fn): T = fits_table(fn) hdr = fitsio.read_header(fn, ext=1) keys = ['LOADED', 'ACCEPTED', 'CHI2', 'POLNAXIS', 'POLNGRP', 'PSF_FWHM', 'PSF_SAMP', 'PSFNAXIS', 'PSFAXIS1', 'PSFAXIS2', 'PSFAXIS3',] if hdr['POLNAXIS'] == 0: # No polynomials. Fake it. T.polgrp1 = np.array([0]) T.polgrp2 = np.array([0]) T.polname1 = np.array(['fake']) T.polname2 = np.array(['fake']) T.polzero1 = np.array([0]) T.polzero2 = np.array([0]) T.polscal1 = np.array([1]) T.polscal2 = np.array([1]) T.poldeg1 = np.array([0]) T.poldeg2 = np.array([0]) else: keys.extend([ 'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1', 'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2', 'POLDEG1']) for k in keys: T.set(k.lower(), np.array([hdr[k]])) psfex.append(T) #print(fn) #T.about() hdr = fitsio.read_header(fn) psfhdrvals.append([hdr.get(k,'') for k in [ 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname]) else: print('File not found:', fn) if len(psfex): padded = pad_arrays([p.psf_mask[0] for p in psfex]) cols = psfex[0].columns() cols.remove('psf_mask') T = merge_tables(psfex, columns=cols) T.psf_mask = np.concatenate([[p] for p in padded]) T.legpipev = np.array([h[0] for h in psfhdrvals]) T.plver = np.array([h[1] for h in psfhdrvals]) T.expnum = np.array([h[2] for h in psfhdrvals]) T.ccdname = np.array([h[3] for h in psfhdrvals]) fn = psfoutfn trymakedirs(fn, dir=True) T.writeto(fn) print('Wrote', fn) if len(splinesky): T = fits_table() T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky]) T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky]) padded = pad_arrays([t.gridvals[0] for t in splinesky]) T.gridvals = np.concatenate([[p] for p in padded]) padded = pad_arrays([t.xgrid[0] for t in splinesky]) T.xgrid = np.concatenate([[p] for p in padded]) padded = pad_arrays([t.xgrid[0] for t in splinesky]) T.ygrid = np.concatenate([[p] for p in padded]) cols = splinesky[0].columns() print('Columns:', cols) for c in ['gridvals', 'xgrid', 'ygrid']: cols.remove(c) T.add_columns_from(merge_tables(splinesky, columns=cols)) T.skyclass = np.array([h[0] for h in skyhdrvals]) T.legpipev = np.array([h[1] for h in skyhdrvals]) T.plver = np.array([h[2] for h in skyhdrvals]) T.expnum = np.array([h[3] for h in skyhdrvals]) T.ccdname = np.array([h[4] for h in skyhdrvals]) fn = skyoutfn trymakedirs(fn, dir=True) T.writeto(fn) print('Wrote', fn)