def getbrickfiles(brickname=None): survey = LegacySurveyData() brickinfo = survey.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brickinfo) ccdinfo = survey.ccds_touching_wcs(brickwcs) nccd = len(ccdinfo) calibdir = survey.get_calib_dir() imagedir = survey.survey_dir # Construct image file names and the calibration file names. expnum = ccdinfo.expnum ccdname = ccdinfo.ccdname psffiles = list() skyfiles = list() imagefiles = list() for ccd in ccdinfo: info = survey.get_image_object(ccd) for attr in ['imgfn', 'dqfn', 'wtfn']: fn = getattr(info, attr).replace(imagedir+'/', '') #if '160108_073601' in fn: # pdb.set_trace() imagefiles.append(fn) psffiles.append(info.psffn.replace(calibdir, 'calib')) skyfiles.append(info.splineskyfn.replace(calibdir, 'calib')) #for ii in range(nccd): #exp = '{0:08d}'.format(expnum[ii]) #rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits') #psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile)) #skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile)) #imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii])))) #print(np.array(imagefiles)) #print(np.array(psffiles)) #print(np.array(skyfiles)) return imagefiles, psffiles, skyfiles
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).") parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--near', action='store_true', help='Quick cut to only CCDs near selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--opt', help='With --command, extra options to add') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric or blacklist cuts') parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table') parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') parser.add_argument('--brickq', type=int, default=None, help='Queue only bricks with the given "brickq" value [0 to 3]') parser.add_argument('--brickq-deps', action='store_true', default=False, help='Queue bricks directly using qdo API, setting brickq dependencies') parser.add_argument('--queue', default='bricks', help='With --brickq-deps, the QDO queue name to use') opt = parser.parse_args() survey = LegacySurveyData() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = survey.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = survey.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) if opt.ignore_cuts == False: I = survey.photometric_ccds(T) print(len(I), 'CCDs are photometric') T.cut(I) I = survey.apply_blacklist(T) print(len(I), 'CCDs are not blacklisted') T.cut(I) print(len(T), 'CCDs remain') # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo,dhi = -25, 40 rlo,rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict(test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo,dhi = -90,90 rlo,rhi = 0, 360 elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo,rhi = 240,245 dlo,dhi = 5, 12 elif opt.region == 'edrplus': rlo,rhi = 235,248 dlo,dhi = 5, 15 elif opt.region == 'edr-south': rlo,rhi = 240,245 dlo,dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo,rhi = 149.75, 150.75 dlo,dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo,rhi = 240,250 dlo,dhi = 10,15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo,rhi = 320., 360. dlo,dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo,rhi = 240,360 dlo,dhi = 20,40 elif opt.region == 'north': rlo,rhi = 120,240 dlo,dhi = 20,40 elif opt.region == 'northeast': rlo,rhi = 0,120 dlo,dhi = 20,40 elif opt.region == 'southwest': rlo,rhi = 240,360 dlo,dhi = -20,0 elif opt.region == 'south': rlo,rhi = 120,240 dlo,dhi = -20,0 elif opt.region == 'southeast': rlo,rhi = 0,120 dlo,dhi = -20,0 elif opt.region == 'southsoutheast': rlo,rhi = 0,120 dlo,dhi = -20,-10 elif opt.region == 'midwest': rlo,rhi = 240,360 dlo,dhi = 0,20 elif opt.region == 'middle': rlo,rhi = 120,240 dlo,dhi = 0,20 elif opt.region == 'mideast': rlo,rhi = 0,120 dlo,dhi = 0,20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo,rhi = 250,260 dlo,dhi = 30,35 elif opt.region == 'deep2f3': rlo,rhi = 351.25, 353.75 dlo,dhi = 0, 0.5 elif opt.region == 'virgo': rlo,rhi = 185,190 dlo,dhi = 10, 15 elif opt.region == 'virgo2': rlo,rhi = 182,192 dlo,dhi = 8, 18 elif opt.region == 'lsb': rlo,rhi = 147.2, 147.8 dlo,dhi = -0.4, 0.4 elif opt.region == 'eboss-elg': # RA -45 to +45 # Dec -5 to +7 rlo,rhi = 315., 45. dlo,dhi = -5., 7. elif opt.region == 'eboss-ngc': # NGC ELGs # RA 115 to 175 # Dec 15 to 30 rlo,rhi = 115., 175. dlo,dhi = 15., 30. elif opt.region == 'mzls': dlo,dhi = 30., 90. elif opt.region == 'dr4-bootes': # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched #dlo,dhi = 34., 35. #rlo,rhi = 209.5, 210.5 dlo,dhi = 33., 36. rlo,rhi = 216.5, 219.5 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') for name in B.get('brickname'): print(name) B.writeto('bricks-cut.fits') I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') plt.clf() plt.plot(B.ra, B.dec, 'b.') plt.title('DR3 bricks') plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1]) plt.savefig('bricks.png') if opt.brickq is not None: B.cut(B.brickq == opt.brickq) log('Cut to', len(B), 'with brickq =', opt.brickq) if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, survey.bricksize) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('survey-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing #B.cut(np.argsort(-B.dec)) # RA increasing B.cut(np.argsort(B.ra)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue print(b.brickname) if opt.save_to_fits: assert(opt.touching) # Write cut tables to file for tab,typ in zip([B,T],['bricks','ccds']): fn='%s-%s-cut.fits' % (typ,opt.name) if os.path.exists(fn): os.remove(fn) tab.writeto(fn) print('Wrote %s' % fn) # Write text files listing ccd and filename names nm1,nm2= 'ccds-%s.txt'% opt.name,'filenames-%s.txt' % opt.name if os.path.exists(nm1): os.remove(nm1) if os.path.exists(nm2): os.remove(nm2) f1,f2=open(nm1,'w'),open(nm2,'w') fns= list(set(T.get('image_filename'))) for fn in fns: f2.write('%s\n' % fn.strip()) for ti in T: f1.write('%s\n' % ti.get('image_filename').strip()) f1.close() f2.close() print('Wrote *-names.txt') if opt.brickq_deps: import qdo from legacypipe.survey import on_bricks_dependencies #... find Queue... q = qdo.connect(opt.queue, create_ok=True) print('Connected to QDO queue', opt.queue, q) brick_to_task = dict() I = survey.photometric_ccds(T) print(len(I), 'CCDs are photometric') T.cut(I) I = survey.apply_blacklist(T) print(len(I), 'CCDs are not blacklisted') T.cut(I) print(len(T), 'CCDs remaining') T.wra = T.ra + (T.ra > 180) * -360 wra = rlo - 360 plt.clf() plt.plot(T.wra, T.dec, 'b.') ax = [wra, rhi, dlo, dhi] plt.axis(ax) plt.title('CCDs') plt.savefig('q-ccds.png') B.wra = B.ra + (B.ra > 180) * -360 # this slight overestimate (for DECam images) is fine radius = 0.3 Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius, indexlist=True) ikeep = [] for ib,(b,Iccd) in enumerate(zip(B, Iccds)): if Iccd is None or len(Iccd) == 0: print('No matched CCDs to brick', b.brickname) continue wcs = wcs_for_brick(b) cI = ccds_touching_wcs(wcs, T[np.array(Iccd)]) print(len(cI), 'CCDs touching brick', b.brickname) if len(cI) == 0: continue ikeep.append(ib) B.cut(np.array(ikeep)) print('Cut to', len(B), 'bricks touched by CCDs') for brickq in range(4): I = np.flatnonzero(B.brickq == brickq) print(len(I), 'bricks with brickq =', brickq) J = np.flatnonzero(B.brickq < brickq) preB = B[J] reqs = [] if brickq > 0: for b in B[I]: # find brick dependencies brickdeps = on_bricks_dependencies(b, survey, bricks=preB) # convert to task ids taskdeps = [brick_to_task.get(b.brickname,None) for b in brickdeps] # If we dropped a dependency brick from a previous brickq because # of no overlapping CCDs, it won't appear in the brick_to_task map. taskdeps = [t for t in taskdeps if t is not None] reqs.append(taskdeps) plt.clf() plt.plot(B.wra, B.dec, '.', color='0.5') plt.plot(B.wra[I], B.dec[I], 'b.') plt.axis(ax) plt.title('Bricks: brickq=%i' % brickq) plt.savefig('q-bricks-%i.png' % brickq) # submit to qdo queue print('Queuing', len(B[I]), 'bricks') if brickq == 0: reqs = None else: assert(len(I) == len(reqs)) taskids = q.add_multiple(B.brickname[I], requires=reqs) assert(len(taskids) == len(I)) print('Queued', len(taskids), 'bricks') brick_to_task.update(dict(zip(B.brickname[I], taskids))) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() elif opt.near: # Roughly brick radius + DECam image radius radius = 0.35 allI,nil,nil = match_radec(T.ra, T.dec, B.ra, B.dec, radius, nearest=True) else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): expstr = '%08i' % T.expnum[i] outfn = os.path.join('forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(survey.survey_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' % # (imgfn, T.image_hdu[i], outfn)) f.write('python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n' % (T.expnum[i], T.ccdname[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if cmd is None: cmd = '' f.write(cmd + ' '.join(batch) + '\n') cmd = None if opt.command: cmd = 'python legacypipe/run-calib.py ' if opt.opt is not None: cmd += opt.opt + ' ' for j,i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j+1, 'of', len(allI)) im = survey.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j+1, 'of', len(allI)) im = survey.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = '%i-%s' % (T.expnum[i], T.ccdname[i]) prefix = 'python legacypipe/run-calib.py ' + opt.opt #('python legacypipe/run-calib.py --expnum %i --ccdname %s' % # (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] prefix = '' if j < 10: print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i], 'filename', T.image_filename[i]) if not opt.nper: f.write(prefix + s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, cmd) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, cmd) f.close() log('Wrote', opt.out) return 0
def main(args=None): """Main routine which parses the optional inputs.""" # Command line options parser= get_parser() args = parser.parse_args(args=args) # Setup loggers if args.verbose: lvl = logging.DEBUG else: lvl = logging.INFO logging.basicConfig(level=lvl, stream=sys.stdout) #,format='%(message)s') log = logging.getLogger('decals_sim') # Sort through args log.info('decals_sim.py args={}'.format(args)) max_nobj=500 max_nchunk=1000 if args.ith_chunk is not None: assert(args.ith_chunk <= max_nchunk-1) assert(args.nchunk <= max_nchunk) assert(args.nobj <= max_nobj) if args.ith_chunk is not None: assert(args.nchunk == 1) #if choose a chunk, only doing 1 chunk if args.nobj is None: parser.print_help() sys.exit(1) brickname = args.brick objtype = args.objtype.upper() lobjtype = objtype.lower() for obj in ('LRG', 'LSB', 'QSO'): if objtype == obj: log.warning('{} objtype not yet supported!'.format(objtype)) return 0 # Deal with the paths. if 'DECALS_SIM_DIR' in os.environ: decals_sim_dir = os.getenv('DECALS_SIM_DIR') else: decals_sim_dir = '.' nobj = args.nobj nchunk = args.nchunk rand = np.random.RandomState(args.seed) # determines seed for all chunks seeds = rand.random_integers(0,2**18, max_nchunk) log.info('Object type = {}'.format(objtype)) log.info('Number of objects = {}'.format(nobj)) log.info('Number of chunks = {}'.format(nchunk)) # Optionally zoom into a portion of the brick survey = LegacySurveyData() brickinfo = survey.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brickinfo) W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale() log.info('Brick = {}'.format(brickname)) if args.zoom is not None: # See also runbrick.stage_tims() (x0, x1, y0, y1) = args.zoom W = x1 - x0 H = y1 - y0 brickwcs = brickwcs.get_subimage(x0, y0, W, H) log.info('Zoom (pixel boundaries) = {}'.format(args.zoom)) targetrd = np.array([brickwcs.pixelxy2radec(x, y) for x, y in [(1,1), (W,1), (W,H), (1,H), (1,1)]]) radec_center = brickwcs.radec_center() log.info('RA, Dec center = {}'.format(radec_center)) log.info('Brick = {}'.format(brickname)) if args.ith_chunk is not None: chunk_list= [args.ith_chunk] else: chunk_list= range(nchunk) # Store args in dict for easy func passing kwargs=dict(seeds=seeds,\ brickname=brickname, \ decals_sim_dir= decals_sim_dir,\ brickwcs= brickwcs, \ objtype=objtype,\ lobjtype=lobjtype,\ nobj=nobj,\ nchunk=nchunk,\ args=args) # Create simulated catalogues and run Tractor create_metadata(kwargs=kwargs) # do chunks for ith_chunk in chunk_list: log.info('Working on chunk {:02d}/{:02d}'.format(ith_chunk,kwargs['nchunk']-1)) # Random ra,dec and source properties create_ith_simcat(ith_chunk, d=kwargs) # Run tractor do_one_chunk(d=kwargs) # Clean up output do_ith_cleanup(ith_chunk=ith_chunk, d=kwargs) log.info('All done!')
def main(): import argparse parser = argparse.ArgumentParser( description='This script creates small self-contained data sets that ' 'are useful for test cases of the pipeline codes.') parser.add_argument('ccds', help='CCDs table describing region to grab') parser.add_argument('outdir', help='Output directory name') parser.add_argument('brick', help='Brick containing these images') parser.add_argument('--cache-dir', type=str, default=None, help='Directory to search for cached files') parser.add_argument( '--wise', help= 'For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image' ) parser.add_argument( '--wise-wcs-hdu', help= 'For WISE outputs, the HDU to read the WCS from in the file given by --wise.', type=int, default=0) parser.add_argument('--fpack', action='store_true', default=False) parser.add_argument('--gzip', action='store_true', default=False) parser.add_argument( '--pad', action='store_true', default=False, help='Keep original image size, but zero out pixels outside ROI') args = parser.parse_args() C = fits_table(args.ccds) print(len(C), 'CCDs in', args.ccds) C.camera = np.array([c.strip() for c in C.camera]) survey = LegacySurveyData(cache_dir=args.cache_dir) if ',' in args.brick: ra, dec = args.brick.split(',') ra = float(ra) dec = float(dec) fakebricks = fits_table() fakebricks.brickname = np.array([( 'custom-%06i%s%05i' % (int(1000 * ra), 'm' if dec < 0 else 'p', int(1000 * np.abs(dec)))) ]) fakebricks.ra = np.array([ra]) fakebricks.dec = np.array([dec]) bricks = fakebricks outbricks = bricks else: bricks = survey.get_bricks_readonly() outbricks = bricks[np.array( [n == args.brick for n in bricks.brickname])] assert (len(outbricks) == 1) outsurvey = LegacySurveyData(survey_dir=args.outdir) trymakedirs(args.outdir) outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz')) targetwcs = wcs_for_brick(outbricks[0]) H, W = targetwcs.shape tycho2fn = survey.find_file('tycho2') kd = tree_open(tycho2fn, 'stars') radius = 1. rc, dc = targetwcs.radec_center() I = tree_search_radec(kd, rc, dc, radius) print(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc, dc)) # Read only the rows within range. tycho = fits_table(tycho2fn, rows=I) del kd print('Read', len(tycho), 'Tycho-2 stars') ok, tx, ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec) #margin = 100 #tycho.cut(ok * (tx > -margin) * (tx < W+margin) * # (ty > -margin) * (ty < H+margin)) print('Cut to', len(tycho), 'Tycho-2 stars within brick') del ok, tx, ty #tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz')) f, tfn = tempfile.mkstemp(suffix='.fits') os.close(f) tycho.writeto(tfn) outfn = os.path.join(args.outdir, 'tycho2.kd.fits') cmd = 'startree -i %s -o %s -P -k -n stars -T' % (tfn, outfn) print(cmd) rtn = os.system(cmd) assert (rtn == 0) os.unlink(tfn) outccds = C.copy() cols = outccds.get_columns() for c in [ 'ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1', 'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1', 'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver', 'psfplver' ]: if c in cols: outccds.delete_column(c) outccds.image_hdu[:] = 1 # Convert to list to avoid truncating filenames outccds.image_filename = [fn for fn in outccds.image_filename] for iccd, ccd in enumerate(C): decam = (ccd.camera.strip() == 'decam') bok = (ccd.camera.strip() == '90prime') im = survey.get_image_object(ccd) print('Got', im) if survey.cache_dir is not None: im.check_for_cached_files(survey) slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1)) psfkwargs = dict(pixPsf=True, gaussPsf=False, hybridPsf=False, normalizePsf=False) tim = im.get_tractor_image(slc, pixPsf=True, splinesky=True, subsky=False, nanomaggies=False, no_remap_invvar=True, old_calibs_ok=True) print('Tim:', tim.shape) psfrow = psfhdr = None if args.pad: psf = im.read_psf_model(0, 0, w=im.width, h=im.height, **psfkwargs) psfex = psf.psfex else: psf = tim.getPsf() psfex = psf.psfex # Did the PSF model come from a merged file? mpsf = im.read_merged_psfex_model(old_calibs_ok=True) if mpsf is not None: T = fits_table(im.merged_psffn) I, = np.nonzero( (T.expnum == im.expnum) * np.array([c.strip() == im.ccdname for c in T.ccdname])) psfrow = T[I] x0 = ccd.ccd_x0 y0 = ccd.ccd_y0 psfrow.polzero1[0] += x0 psfrow.polzero2[0] += y0 psfhdr = fitsio.read_header(im.merged_psffn) psfex.fwhm = tim.psf_fwhm if psfrow is not None: print('PSF row:', psfrow) else: print('PSF:', psf) print('PsfEx:', psfex) skyrow = skyhdr = None if args.pad: primhdr = fitsio.read_header(im.imgfn) imghdr = fitsio.read_header(im.imgfn, hdu=im.hdu) sky = im.read_sky_model(splinesky=True, primhdr=primhdr, imghdr=imghdr) else: sky = tim.getSky() # Did the sky model come from a merged file? msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True) if msky is not None: T = fits_table(im.merged_splineskyfn) I, = np.nonzero( (T.expnum == im.expnum) * np.array([c.strip() == im.ccdname for c in T.ccdname])) skyrow = T[I] skyrow.x0[0] = ccd.ccd_x0 skyrow.y0[0] = ccd.ccd_y0 skyhdr = fitsio.read_header(im.merged_splineskyfn) if skyrow is not None: print('Sky row:', skyrow) else: print('Sky:', sky) outim = outsurvey.get_image_object(ccd) print('Output image:', outim) print('Image filename:', outim.imgfn) trymakedirs(outim.imgfn, dir=True) imgdata = tim.getImage() ivdata = tim.getInvvar() # Since we remap DQ codes (always with Mosaic and Bok, sometimes with DECam), # re-read from the FITS file rather than using tim.dq. print('Reading data quality from', im.dqfn, 'hdu', im.hdu) dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice) print('Tim shape:', tim.shape, 'Slice', tim.slice) print('image shape:', imgdata.shape, 'iv', ivdata.shape, 'DQ', dqdata.shape) from collections import Counter dqvals = Counter(dqdata.ravel()) print('DQ pixel counts:') for k, n in dqvals.most_common(): print(' 0x%x' % k, ':', n) if args.pad: # Create zero image of full size, copy in data. fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype) fullsize[slc] = imgdata imgdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype) fullsize[slc] = dqdata dqdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype) fullsize[slc] = ivdata ivdata = fullsize else: # Adjust the header WCS by x0,y0 crpix1 = tim.hdr['CRPIX1'] crpix2 = tim.hdr['CRPIX2'] tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0 tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0 # Add image extension to filename # fitsio doesn't compress .fz by default, so drop .fz suffix outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits') if args.gzip: outim.imgfn = outim.imgfn.replace('.fits', '.fits.gz') outim.wtfn = outim.wtfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.wtfn = outim.wtfn.replace('.fits.fz', '.fits') if args.gzip: outim.wtfn = outim.wtfn.replace('.fits', '.fits.gz') if outim.dqfn is not None: outim.dqfn = outim.dqfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.dqfn = outim.dqfn.replace('.fits.fz', '.fits') if args.gzip: outim.dqfn = outim.dqfn.replace('.fits', '.fits.gz') if bok: outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname) ccdfn = outim.imgfn ccdfn = ccdfn.replace(outsurvey.get_image_dir(), '') if ccdfn.startswith('/'): ccdfn = ccdfn[1:] outccds.image_filename[iccd] = ccdfn print('Changed output filenames to:') print(outim.imgfn) print(outim.dqfn) ofn = outim.imgfn if args.fpack: f, ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(imgdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert (rtn == 0) h, w = tim.shape if not args.pad: outccds.width[iccd] = w outccds.height[iccd] = h outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0 outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0 wcs = Tan(*[ float(x) for x in [ ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2, ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height ] ]) if args.pad: subwcs = wcs else: subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h) outccds.ra[iccd], outccds.dec[iccd] = subwcs.radec_center() print('Weight filename:', outim.wtfn) wfn = outim.wtfn trymakedirs(wfn, dir=True) ofn = wfn if args.fpack: f, ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(ivdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert (rtn == 0) if outim.dqfn is not None: print('DQ filename', outim.dqfn) trymakedirs(outim.dqfn, dir=True) ofn = outim.dqfn if args.fpack: f, ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(dqdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert (rtn == 0) psfout = outim.psffn if psfrow: psfout = outim.merged_psffn print('PSF output filename:', psfout) trymakedirs(psfout, dir=True) if psfrow: psfrow.writeto(psfout, primhdr=psfhdr) else: print('Writing PsfEx:', psfout) psfex.writeto(psfout) # update header F = fitsio.FITS(psfout, 'rw') F[0].write_keys([ dict(name='EXPNUM', value=ccd.expnum), dict(name='PLVER', value=psf.plver), dict(name='PROCDATE', value=psf.procdate), ]) F.close() skyout = outim.splineskyfn if skyrow: skyout = outim.merged_splineskyfn print('Sky output filename:', skyout) trymakedirs(skyout, dir=True) if skyrow is not None: skyrow.writeto(skyout, primhdr=skyhdr) else: primhdr = fitsio.FITSHDR() primhdr['PLVER'] = sky.plver primhdr['PROCDATE'] = sky.procdate primhdr['EXPNUM'] = ccd.expnum primhdr['IMGDSUM'] = sky.datasum sky.write_fits(skyout, primhdr=primhdr) # HACK -- check result immediately. outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz')) outsurvey.ccds = None outC = outsurvey.get_ccds_readonly() occd = outC[iccd] outim = outsurvey.get_image_object(occd) print('Got output image:', outim) otim = outim.get_tractor_image(pixPsf=True, splinesky=True, hybridPsf=True, old_calibs_ok=True) print('Got output tim:', otim) outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz')) # WISE if args.wise is not None: from wise.forcedphot import unwise_tiles_touching_wcs from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs, get_unwise_tractor_image, get_unwise_tile_dir) # Read WCS... print('Reading TAN wcs header from', args.wise, 'HDU', args.wise_wcs_hdu) targetwcs = Tan(args.wise, args.wise_wcs_hdu) tiles = unwise_tiles_touching_wcs(targetwcs) print('Cut to', len(tiles), 'unWISE tiles') H, W = targetwcs.shape r, d = targetwcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]), np.array([H / 2, H / 2, 1, H])) roiradec = [r[0], r[1], d[2], d[3]] unwise_dir = os.environ['UNWISE_COADDS_DIR'] wise_out = os.path.join(args.outdir, 'images', 'unwise') print('Will write WISE outputs to', wise_out) unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR'] wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr') print('Will write WISE time-resolved outputs to', wise_tr_out) trymakedirs(wise_tr_out) W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_atlas.fits')) print('Read', len(W), 'time-resolved WISE coadd tiles') W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id])) print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth') # Write the time-resolved index subset. W.writeto(os.path.join(wise_tr_out, 'time_resolved_atlas.fits')) # this ought to be enough for anyone =) _, Nepochs = W.epoch_bitmask.shape print('N epochs in time-resolved atlas:', Nepochs) wisedata = [] # full depth for band in [1, 2, 3, 4]: wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band)) # time-resolved for band in [1, 2]: # W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2) bitmask = (1 << (band - 1)) for e in range(Nepochs): # Which tiles have images for this epoch? I = np.flatnonzero(W.epoch_bitmask[:, e] & bitmask) if len(I) == 0: continue print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I]) edir = os.path.join(unwise_tr_dir, 'e%03i' % e) eoutdir = os.path.join(wise_tr_out, 'e%03i' % e) wisedata.append((edir, eoutdir, tiles.coadd_id[I], band)) wrote_masks = set() for indir, outdir, tiles, band in wisedata: for tile in tiles: wanyband = 'w' tim = get_unwise_tractor_image(indir, tile, band, bandname=wanyband, roiradecbox=roiradec) print('Got unWISE tim', tim) print(tim.shape) thisdir = get_unwise_tile_dir(outdir, tile) print('Directory for this WISE tile:', thisdir) base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band)) print('Base filename:', base) masked = True mu = 'm' if masked else 'u' imfn = base + 'img-%s.fits' % mu ivfn = base + 'invvar-%s.fits.gz' % mu nifn = base + 'n-%s.fits.gz' % mu nufn = base + 'n-u.fits.gz' #print('WISE image header:', tim.hdr) # Adjust the header WCS by x0,y0 wcs = tim.wcs.wcs tim.hdr['CRPIX1'] = wcs.crpix[0] tim.hdr['CRPIX2'] = wcs.crpix[1] H, W = tim.shape tim.hdr['IMAGEW'] = W tim.hdr['IMAGEH'] = H print('WCS:', wcs) print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2']) trymakedirs(imfn, dir=True) fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True) print('Wrote', imfn) fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True) print('Wrote', ivfn) fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True) print('Wrote', nifn) fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True) print('Wrote', nufn) if not (indir, tile) in wrote_masks: print('Looking for mask file for', indir, tile) # record that we tried this dir/tile combo wrote_masks.add((indir, tile)) for idir in indir.split(':'): tdir = get_unwise_tile_dir(idir, tile) maskfn = 'unwise-%s-msk.fits.gz' % tile fn = os.path.join(tdir, maskfn) print('Mask file:', fn) if os.path.exists(fn): print('Reading', fn) (x0, x1, y0, y1) = tim.roi roislice = (slice(y0, y1), slice(x0, x1)) F = fitsio.FITS(fn)[0] hdr = F.read_header() M = F[roislice] outfn = os.path.join(thisdir, maskfn) fitsio.write(outfn, M, header=tim.hdr, clobber=True) print('Wrote', outfn) break outC = outsurvey.get_ccds_readonly() for iccd, ccd in enumerate(outC): outim = outsurvey.get_image_object(ccd) print('Got output image:', outim) otim = outim.get_tractor_image(pixPsf=True, splinesky=True, hybridPsf=True, old_calibs_ok=True) print('Got output tim:', otim)
def main(): import optparse parser = optparse.OptionParser() parser.add_option('--zoom', '-z', type=int, action='append', default=[], help='Add zoom level; default 13') parser.add_option('--threads', type=int, default=1, help='Number of threads') parser.add_option('--y0', type=int, default=0, help='Start row') parser.add_option('--y1', type=int, default=None, help='End row (non-inclusive)') parser.add_option('--x0', type=int, default=None) parser.add_option('--x1', type=int, default=None) parser.add_option('-x', type=int) parser.add_option('-y', type=int) parser.add_option('--mindec', type=float, default=None, help='Minimum Dec to run') parser.add_option('--maxdec', type=float, default=None, help='Maximum Dec to run') parser.add_option('--minra', type=float, default=None, help='Minimum RA to run') parser.add_option('--maxra', type=float, default=None, help='Maximum RA to run') parser.add_option('--near', action='store_true', help='Only run tiles near bricks') parser.add_option('--near-ccds', action='store_true', help='Only run tiles near CCDs') parser.add_option('--queue', action='store_true', default=False, help='Print qdo commands') parser.add_option('--all', action='store_true', help='Render all tiles') parser.add_option('--ignore', action='store_true', help='Ignore cached tile files', default=False) parser.add_option('--top', action='store_true', help='Top levels of the pyramid') parser.add_option('--split', action='store_true', help='For split layers (DR6+DR7), only compute one y strip per zoom level') parser.add_option('--bricks-exist', action='store_true', help='Create table of bricks that exist') parser.add_option('--kind', default='image') parser.add_option('--scale', action='store_true', help='Scale images?') parser.add_option('--bricks', action='store_true', help='Compute scaled brick tables?') parser.add_option('--coadd', action='store_true', help='Create SDSS coadd images?') parser.add_option('--grass', action='store_true', help='progress plot') parser.add_option('--bands', default=None) parser.add_option('-v', '--verbose', dest='verbose', action='count', default=0, help='Make more verbose') opt,args = parser.parse_args() if opt.verbose == 0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) mp = multiproc(opt.threads) if opt.kind in ['sdss', 'sdss2']: if opt.maxdec is None: opt.maxdec = 90 if opt.mindec is None: opt.mindec = -25 elif opt.kind in ['halpha', 'unwise-neo1', 'unwise-neo2', 'unwise-neo3', 'unwise-neo4', 'unwise-cat-model', 'galex', 'wssa', 'vlass', 'hsc']: if opt.maxdec is None: opt.maxdec = 90. if opt.mindec is None: opt.mindec = -90. if opt.maxra is None: opt.maxra = 360. if opt.minra is None: opt.minra = 0. if opt.kind == 'galex' and opt.bands is None: opt.bands = 'nf' if 'unwise' in opt.kind and opt.bands is None: opt.bands = '12' if 'vlass' in opt.kind: opt.bands = [1] elif opt.kind == 'm33': if opt.mindec is None: opt.mindec = 30.40 if opt.maxdec is None: opt.maxdec = 30.90 if opt.minra is None: opt.minra = 23.29 if opt.maxra is None: opt.maxra = 23.73 elif opt.kind in ['des-dr1']: if opt.maxdec is None: opt.maxdec = 6 if opt.mindec is None: opt.mindec = -68 if opt.maxra is None: opt.maxra = 360 if opt.minra is None: opt.minra = 0 elif opt.kind in ['mzls+bass-dr4', 'mzls+bass-dr4-model', 'mzls+bass-dr4-resid']: if opt.maxdec is None: opt.maxdec = 90 if opt.mindec is None: opt.mindec = 30 if opt.maxra is None: opt.maxra = 54 if opt.minra is None: opt.minra = 301 elif opt.kind in ['mzls+bass-dr6', 'mzls+bass-dr6-model', 'mzls+bass-dr6-resid']: if opt.maxdec is None: opt.maxdec = 90 if opt.mindec is None: opt.mindec = -20 if opt.maxra is None: opt.maxra = 360 if opt.minra is None: opt.minra = 0 elif opt.kind in ['decaps2', 'decaps2-model', 'decaps2-resid']: if opt.maxdec is None: opt.maxdec = -20 if opt.mindec is None: opt.mindec = -70 if opt.maxra is None: opt.maxra = 280 if opt.minra is None: opt.minra = 90 else: if opt.maxdec is None: opt.maxdec = 40 if opt.mindec is None: opt.mindec = -25 if opt.maxra is None: opt.maxra = 360 if opt.minra is None: opt.minra = 0 if opt.bands is None: opt.bands = 'grz' if opt.top: top_levels(mp, opt) sys.exit(0) if opt.bricks: from map.views import get_layer layer = get_layer(opt.kind) for scale in range(1,8): B = layer.get_bricks_for_scale(scale) sys.exit(0) if opt.scale: if opt.kind == 'ps1': from map.views import get_layer fns = glob('data/ps1/skycells/*/ps1-*.fits') fns.sort() #ps1-1561-021-r.fits if len(opt.zoom) == 0: opt.zoom = [1,2,3,4,5,6,7] print(len(fns), 'PS1 image files') layer = get_layer(opt.kind) B = layer.get_bricks() for i,brick in enumerate(B): for band in opt.bands: fn0 = layer.get_filename(brick, band, 0) if not os.path.exists(fn0): continue for scale in opt.zoom: fn = layer.get_filename(brick, band, scale) layer.create_scaled_image(brick, band, scale, fn) sys.exit(0) # Rebricked if opt.kind in ['decals-dr5', 'decals-dr5-model', 'decals-dr7', 'decals-dr7-model', 'eboss', 'mzls+bass-dr6', 'mzls+bass-dr6-model', 'unwise-neo3', 'unwise-neo4', 'unwise-cat-model', 'galex', 'wssa', 'des-dr1', 'hsc', ] or opt.kind.startswith('dr8-test'): # or True: from map.views import get_layer layer = get_layer(opt.kind) if opt.queue: if len(opt.zoom) == 0: opt.zoom = [1,2,3,4,5,6,7] #step = 0.1 #ras = np.arange(opt.minra, opt.maxra+step, step) step = 3. #ras = np.arange(opt.minra, opt.maxra+step, step) decs = np.arange(opt.mindec, opt.maxdec+step, step) for zoom in opt.zoom: for declo,dechi in zip(decs, np.clip(decs[1:], opt.mindec, opt.maxdec)): rstep = step / np.maximum(0.05, np.cos(np.deg2rad((declo+dechi)/2.))) ras = np.arange(opt.minra, opt.maxra+rstep, rstep) for ralo,rahi in zip(ras, np.clip(ras[1:], opt.minra, opt.maxra)): cmd = ('python render-tiles.py --kind %s --scale --minra %f --maxra %f --mindec %f --maxdec %f -z %i' % (opt.kind, ralo, rahi, declo, dechi, zoom)) print(cmd) sys.exit(0) if len(opt.zoom) == 0: opt.zoom = [1] for scale in opt.zoom: B = layer.get_bricks_for_scale(scale) print(len(B), 'bricks for scale', scale) B.cut((B.dec >= opt.mindec) * (B.dec <= opt.maxdec)) print(len(B), 'in Dec range') B.cut((B.ra >= opt.minra) * (B.ra <= opt.maxra)) print(len(B), 'in RA range') bands = opt.bands has = {} for band in bands: if 'has_%s' % band in B.get_columns(): has[band] = B.get('has_%s' % band) else: # assume yes has[band] = np.ones(len(B), bool) # Run one scale at a time args = [] for ibrick,brick in enumerate(B): for band in bands: if has[band][ibrick]: args.append((layer, brick, band, scale, opt.ignore)) print(len(args), 'bricks for scale', scale) mp.map(_layer_get_filename, args) sys.exit(0) if (opt.kind in ['decals-dr3', 'decals-dr3-model', 'mzls+bass-dr4', 'mzls+bass-dr4-model', 'decaps2', 'decaps2-model', 'eboss', 'ps1'] or 'dr8b' in opt.kind or 'dr8c' in opt.kind): from map.views import get_survey, get_layer surveyname = opt.kind # *-model -> * # for prefix in ['decals-dr3', 'mzls+bass-dr4', 'decaps2', 'decals-dr5']: # if prefix in surveyname: # surveyname = prefix for suffix in ['-model', '-resid']: if surveyname.endswith(suffix): surveyname = surveyname[:-len(suffix)] survey = get_survey(surveyname) print('Survey:', survey) print(' cache_dir:', survey.cache_dir) B = survey.get_bricks() print(len(B), 'bricks') B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec)) print(len(B), 'in Dec range') B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra)) print(len(B), 'in RA range') # find all image files filetype = 'image' model = False if '-model' in opt.kind: model = True filetype = 'model' bands = opt.bands layer = get_layer(opt.kind) has = {} for band in bands: if 'has_%s' % band in B.get_columns(): has[band] = B.get('has_%s' % band) else: # assume yes has[band] = np.ones(len(B), bool) for scale in opt.zoom: args = [] for ibrick,brick in enumerate(B): for band in bands: if not has[band][ibrick]: print('Brick', brick.brickname, 'does not have', band) continue args.append((layer, brick, band, scale, opt.ignore)) mp.map(_layer_get_filename, args) sys.exit(0) elif opt.kind == 'sdss2': layer = get_layer(opt.kind) bands = 'gri' scales = opt.zoom if len(scales) == 0: scales = list(range(1,8)) for scale in scales: #maxscale = 7 bricks = layer.get_bricks_for_scale(scale) print('Got', len(bricks), 'bricks for scale', scale) bricks.cut((bricks.dec > opt.mindec) * (bricks.dec <= opt.maxdec) * (bricks.ra > opt.minra ) * (bricks.ra <= opt.maxra)) print('Cut to', len(bricks), 'bricks within RA,Dec box') for ibrick,brick in enumerate(bricks): for band in bands: print('Scaling brick', ibrick, 'of', len(bricks), 'scale', scale, 'brick', brick.brickname, 'band', band) fn = layer.get_filename(brick, band, scale) sys.exit(0) if opt.kind in ['unwise-w1w2', 'unwise-neo2']: # scaledir = opt.kind # basedir = settings.DATA_DIR # dirnm = os.path.join(basedir, 'scaled', scaledir) # B = fits_table(os.path.join(basedir, 'unwise-bricks.fits')) layer = get_layer(opt.kind) B = layer.get_bricks() print(len(B), 'unWISE tiles') for b in B: for band in ['1','2']: for scale in [1,2,3,4,5,6,7]: print('Get brick', b.brickname, 'band', band, 'scale', scale) layer.get_filename(b, band, scale) else: assert(False) if opt.bricks_exist: from map.views import get_survey surveyname = opt.kind filetype = 'image' survey = get_survey(surveyname) B = survey.get_bricks() print(len(B), 'bricks') B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec)) print(len(B), 'in Dec range') B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra)) print(len(B), 'in RA range') # find all image files bands = opt.bands has_band = {} for b in bands: B.set('has_%s' % b, np.zeros(len(B), bool)) has_band[b] = B.get('has_%s' % b) exists = np.zeros(len(B), bool) for i,brick in enumerate(B.brickname): found = False for band in bands: fn = survey.find_file(filetype, brick=brick, band=band) ex = os.path.exists(fn) print('Brick', brick, 'band', band, 'exists?', ex) has_band[band][i] = ex if ex: found = True exists[i] = found B.cut(exists) B.writeto('bricks-exist-%s.fits' % opt.kind) sys.exit(0) from map.views import get_survey surveyname = opt.kind if surveyname.endswith('-model'): surveyname = surveyname.replace('-model','') if surveyname.endswith('-resid'): surveyname = surveyname.replace('-resid','') survey = get_survey(surveyname) if len(opt.zoom) == 0: opt.zoom = [13] if opt.near: if opt.kind == 'sdss': B = fits_table(os.path.join(settings.DATA_DIR, 'bricks-sdssco.fits')) else: B = survey.get_bricks() print(len(B), 'bricks') #if opt.scale: # opt.near_ccds = True if opt.near_ccds: if opt.kind == 'sdss': C = fits_table(os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'), columns=['rerun','ra','dec', 'run', 'camcol', 'field', 'score']) C.cut(C.rerun == '301') C.cut(C.score >= 0.6) #C.delete_column('rerun') # SDSS field size radius = 1.01 * np.hypot(10., 14.)/2. / 60. ccdsize = radius print(len(C), 'SDSS fields') else: C = survey.get_ccds() print(len(C), 'CCDs') ccdsize = 0.2 if opt.x is not None: opt.x0 = opt.x opt.x1 = opt.x + 1 if opt.y is not None: opt.y0 = opt.y opt.y1 = opt.y + 1 if opt.coadd and opt.kind == 'galex': layer = GalexLayer('galex') # base-level (coadd) bricks B = layer.get_bricks() print(len(B), 'bricks') B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec)) print(len(B), 'in Dec range') B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra)) print(len(B), 'in RA range') pat = layer.get_scaled_pattern() tempfiles = [] for b in B: for band in ['n','f']: fn = pat % dict(scale=0, band=band, brickname=b.brickname) layer.create_coadd_image(b, band, 0, fn, tempfiles=tempfiles) for fn in tempfiles: os.unlink(fn) sys.exit(0) if opt.coadd and opt.kind == 'sdss': from legacypipe.survey import wcs_for_brick from map.views import trymakedirs #B = survey.get_bricks() B = fits_table(os.path.join(settings.DATA_DIR, 'sdss2', 'bricks-sdssco.fits')) print(len(B), 'bricks') B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec)) print(len(B), 'in Dec range') B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra)) print(len(B), 'in RA range') if opt.queue: # ~ square-degree tiles # RA slices rr = np.arange(opt.minra , opt.maxra +1) dd = np.arange(opt.mindec, opt.maxdec+1) for rlo,rhi in zip(rr, rr[1:]): for dlo,dhi in zip(dd, dd[1:]): print('time python render-tiles.py --kind sdss --coadd --minra %f --maxra %f --mindec %f --maxdec %f' % (rlo, rhi, dlo, dhi)) sys.exit(0) if opt.grass: basedir = settings.DATA_DIR codir = os.path.join(basedir, 'coadd', 'sdssco') rr,dd = [],[] exist = [] for i,b in enumerate(B): print('Brick', b.brickname,) fn = os.path.join(codir, b.brickname[:3], 'sdssco-%s-%s.fits' % (b.brickname, 'r')) print('-->', fn,) if not os.path.exists(fn): print() continue print('found') rr.append(b.ra) dd.append(b.dec) exist.append(i) exist = np.array(exist) B.cut(exist) B.writeto('bricks-sdssco-exist.fits') import pylab as plt plt.clf() plt.plot(rr, dd, 'k.') plt.title('SDSS coadd tiles') plt.savefig('sdss.png') sys.exit(0) basedir = settings.DATA_DIR codir = os.path.join(basedir, 'coadd', 'sdssco') for b in B: print('Brick', b.brickname) wcs = wcs_for_brick(b, W=2400, H=2400, pixscale=0.396) #bands = 'gri' bands = 'z' dirnm = os.path.join(codir, b.brickname[:3]) fns = [os.path.join(dirnm, 'sdssco-%s-%s.fits' % (b.brickname, band)) for band in bands] hdr = fitsio.FITSHDR() hdr['SURVEY'] = 'SDSS' wcs.add_to_header(hdr) if all([os.path.exists(fn) for fn in fns]): print('Already exist') continue from map.oldviews import map_sdss ims = map_sdss(req, 1, 0, 0, 0, get_images=True, wcs=wcs, ignoreCached=True, forcescale=0, bands=bands) if ims is None: print('No overlap') continue trymakedirs(os.path.join(dirnm, 'xxx')) for fn,band,im in zip(fns,bands, ims): fitsio.write(fn, im, header=hdr, clobber=True) print('Wrote', fn) # Also write scaled versions # dirnm = os.path.join(basedir, 'scaled', 'sdssco') # scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(brickname).3s', 'sdssco-%(brickname)s-%(band)s.fits') # for im,band in zip(ims,bands): # scalekwargs = dict(band=band, brick=b.brickid, brickname=b.brickname) # imwcs = wcs # for scale in range(1, 7): # print('Writing scale level', scale) # im,imwcs,sfn = get_scaled(scalepat, scalekwargs, scale, None, # wcs=imwcs, img=im, return_data=True) sys.exit(0) for zoom in opt.zoom: N = 2**zoom if opt.y1 is None: y1 = N else: y1 = opt.y1 if opt.x0 is None: opt.x0 = 0 x1 = opt.x1 if x1 is None: x1 = N if opt.split: decsplit = 32. y = 2.**zoom/(2.*np.pi) * (np.pi - np.log(np.tan(np.pi/4. + np.deg2rad(decsplit)/2.))) y = int(y) opt.y0 = y y1 = y+1 # Find grid of Ra,Dec tile centers and select the ones near DECaLS bricks. rr,dd = [],[] yy = np.arange(opt.y0, y1) xx = np.arange(opt.x0, x1) if opt.grass: import pylab as plt tileexists = np.zeros((len(yy),len(xx)), bool) basedir = settings.DATA_DIR ver = tileversions[opt.kind][-1] tiledir = os.path.join(basedir, 'tiles', opt.kind, '%i'%ver, '%i'%zoom) for dirpath,dirnames,filenames in os.walk(tiledir): # change walk order dirnames.sort() if len(filenames) == 0: continue print('Dirpath', dirpath) #print 'Dirnames', dirnames #print 'Filenames', filenames # check for symlinks if False: fns = [] for fn in filenames: fullfn = os.path.join(tiledir, dirpath, fn) if os.path.isfile(fullfn) and not os.path.islink(fullfn): fns.append(fn) print(len(fns), 'of', len(filenames), 'are files (not symlinks)') filenames = fns x = os.path.basename(dirpath) x = int(x) #print 'x', x yy = [int(fn.replace('.jpg','')) for fn in filenames] #print 'yy', yy print(len(yy), 'tiles') for y in yy: tileexists[y - opt.y0, x - opt.x0] = True plt.clf() plt.imshow(tileexists, interpolation='nearest', origin='upper', vmin=0, vmax=1, cmap='gray') fn = 'exist-%s-z%02i' % (opt.kind, zoom) plt.savefig(fn+'.png') fitsio.write(fn+'.fits', tileexists, clobber=True) print('Wrote', fn+'.png and', fn+'.fits') continue if not opt.all: for y in yy: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, y) r,d = wcs.get_center() dd.append(d) for x in xx: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, x, 0) r,d = wcs.get_center() rr.append(r) dd = np.array(dd) rr = np.array(rr) if len(dd) > 1: tilesize = max(np.abs(np.diff(dd))) print('Tile size:', tilesize) else: if opt.near_ccds or opt.near: try: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0+1) r2,d2 = wcs.get_center() except: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0-1) r2,d2 = wcs.get_center() tilesize = np.abs(dd[0] - d2) print('Tile size:', tilesize) else: tilesize = 180. I = np.flatnonzero((dd >= opt.mindec) * (dd <= opt.maxdec)) print('Keeping', len(I), 'Dec points between', opt.mindec, 'and', opt.maxdec) dd = dd[I] yy = yy[I] if opt.near_ccds: margin = tilesize + ccdsize I = np.flatnonzero((dd > C.dec.min()-margin) * (dd < C.dec.max()+margin)) if len(I) == 0: print('No Dec points within range of CCDs') continue dd = dd[I] yy = yy[I] print('Keeping', len(I), 'Dec points within range of CCDs: Dec', dd.min(), dd.max()) I = np.flatnonzero((rr >= opt.minra) * (rr <= opt.maxra)) print('Keeping', len(I), 'RA points between', opt.minra, 'and', opt.maxra) rr = rr[I] xx = xx[I] print(len(rr), 'RA points x', len(dd), 'Dec points') print('x tile range:', xx.min(), xx.max(), 'y tile range:', yy.min(), yy.max()) for iy,y in enumerate(yy): print() print('Y row', y) if opt.queue: if 'decaps2' in opt.kind: layer = get_layer(opt.kind) if zoom >= layer.nativescale: oldscale = 0 else: oldscale = (layer.nativescale - zoom) oldscale = np.clip(oldscale, layer.minscale, layer.maxscale) x = 0 wcs, W, H, zoomscale, z,xi,yi = get_tile_wcs(zoom, x, y) newscale = layer.get_scale(zoom, 0, y, wcs) if oldscale == newscale: print('Oldscale = newscale = ', oldscale) continue cmd = 'python -u render-tiles.py --zoom %i --y0 %i --y1 %i --kind %s --mindec %f --maxdec %f' % (zoom, y, y+1, opt.kind, opt.mindec, opt.maxdec) cmd += ' --threads 32' if opt.near_ccds: cmd += ' --near-ccds' if opt.all: cmd += ' --all' if opt.ignore: cmd += ' --ignore' print(cmd) continue if opt.near: d = dd[iy] I,J,dist = match_radec(rr, d+np.zeros_like(rr), B.ra, B.dec, 0.25 + tilesize, nearest=True) if len(I) == 0: print('No matches to bricks') continue keep = np.zeros(len(rr), bool) keep[I] = True print('Keeping', sum(keep), 'tiles in row', y, 'Dec', d) x = xx[keep] elif opt.near_ccds: d = dd[iy] print('RA range of tiles:', rr.min(), rr.max()) print('Dec of tile row:', d) I,J,dist = match_radec(rr, d+np.zeros_like(rr), C.ra, C.dec, ccdsize + tilesize, nearest=True) if len(I) == 0: print('No matches to CCDs') continue keep = np.zeros(len(rr), bool) keep[I] = True print('Keeping', sum(keep), 'tiles in row', y, 'Dec', d) x = xx[keep] else: x = xx # if opt.grass: # for xi in x: # basedir = settings.DATA_DIR # ver = tileversions[opt.kind][-1] # tilefn = os.path.join(basedir, 'tiles', opt.kind, # '%i/%i/%i/%i.jpg' % (ver, zoom, xi, y)) # print 'Checking for', tilefn # if os.path.exists(tilefn): # print 'EXISTS' # tileexists[yi-opt.y0, xi-opt.x0] # continue args = [] for xi in x: args.append((opt.kind,zoom,xi,y, opt.ignore, False)) #args.append((opt.kind,zoom,xi,y, opt.ignore, True)) print('Rendering', len(args), 'tiles in row y =', y) mp.map(_bounce_one_tile, args, chunksize=min(100, max(1, int(len(args)/opt.threads)))) #mp.map(_one_tile, args, chunksize=min(100, max(1, int(len(args)/opt.threads)))) print('Rendered', len(args), 'tiles')
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser( description= "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)." ) parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--near', action='store_true', help='Quick cut to only CCDs near selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--opt', help='With --command, extra options to add') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--ignore_cuts', action='store_true', default=False, help='no photometric or blacklist cuts') parser.add_argument('--save_to_fits', action='store_true', default=False, help='save cut brick,ccd to fits table') parser.add_argument( '--name', action='store', default='dr3', help='save with this suffix, e.g. refers to ccds table') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') parser.add_argument( '--brickq', type=int, default=None, help='Queue only bricks with the given "brickq" value [0 to 3]') parser.add_argument( '--brickq-deps', action='store_true', default=False, help='Queue bricks directly using qdo API, setting brickq dependencies' ) parser.add_argument('--queue', default='bricks', help='With --brickq-deps, the QDO queue name to use') opt = parser.parse_args() survey = LegacySurveyData() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = survey.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = survey.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) if opt.ignore_cuts == False: I = survey.photometric_ccds(T) print(len(I), 'CCDs are photometric') T.cut(I) I = survey.apply_blacklist(T) print(len(I), 'CCDs are not blacklisted') T.cut(I) print(len(T), 'CCDs remain') # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo, dhi = -25, 40 rlo, rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict( test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo, dhi = -90, 90 rlo, rhi = 0, 360 elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo, rhi = 240, 245 dlo, dhi = 5, 12 elif opt.region == 'edrplus': rlo, rhi = 235, 248 dlo, dhi = 5, 15 elif opt.region == 'edr-south': rlo, rhi = 240, 245 dlo, dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo, rhi = 149.75, 150.75 dlo, dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo, rhi = 240, 250 dlo, dhi = 10, 15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo, rhi = 320., 360. dlo, dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo, rhi = 240, 360 dlo, dhi = 20, 40 elif opt.region == 'north': rlo, rhi = 120, 240 dlo, dhi = 20, 40 elif opt.region == 'northeast': rlo, rhi = 0, 120 dlo, dhi = 20, 40 elif opt.region == 'southwest': rlo, rhi = 240, 360 dlo, dhi = -20, 0 elif opt.region == 'south': rlo, rhi = 120, 240 dlo, dhi = -20, 0 elif opt.region == 'southeast': rlo, rhi = 0, 120 dlo, dhi = -20, 0 elif opt.region == 'southsoutheast': rlo, rhi = 0, 120 dlo, dhi = -20, -10 elif opt.region == 'midwest': rlo, rhi = 240, 360 dlo, dhi = 0, 20 elif opt.region == 'middle': rlo, rhi = 120, 240 dlo, dhi = 0, 20 elif opt.region == 'mideast': rlo, rhi = 0, 120 dlo, dhi = 0, 20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo, rhi = 250, 260 dlo, dhi = 30, 35 elif opt.region == 'deep2f3': rlo, rhi = 351.25, 353.75 dlo, dhi = 0, 0.5 elif opt.region == 'virgo': rlo, rhi = 185, 190 dlo, dhi = 10, 15 elif opt.region == 'virgo2': rlo, rhi = 182, 192 dlo, dhi = 8, 18 elif opt.region == 'lsb': rlo, rhi = 147.2, 147.8 dlo, dhi = -0.4, 0.4 elif opt.region == 'eboss-sgc': # generous boundaries to make sure get all relevant images # RA -45 to +45 # Dec -5 to +7 rlo, rhi = 310., 50. dlo, dhi = -6., 6. elif opt.region == 'eboss-ngc': # generous boundaries to make sure get all relevant images # NGC ELGs # RA 115 to 175 # Dec 15 to 30 rlo, rhi = 122., 177. dlo, dhi = 12., 32. elif opt.region == 'mzls': dlo, dhi = 30., 90. elif opt.region == 'dr4-bootes': # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched #dlo,dhi = 34., 35. #rlo,rhi = 209.5, 210.5 dlo, dhi = 33., 36. rlo, rhi = 216.5, 219.5 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut( np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') #for name in B.get('brickname'): #print(name) B.writeto('bricks-cut.fits') I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') plt.clf() plt.plot(B.ra, B.dec, 'b.') plt.title('DR3 bricks') plt.axis([360, 0, np.min(B.dec) - 1, np.max(B.dec) + 1]) plt.savefig('bricks.png') if opt.brickq is not None: B.cut(B.brickq == opt.brickq) log('Cut to', len(B), 'with brickq =', opt.brickq) if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, survey.bricksize) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('survey-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing #B.cut(np.argsort(-B.dec)) # RA increasing B.cut(np.argsort(B.ra)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % ( b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue #print(b.brickname) if opt.save_to_fits: assert (opt.touching) # Write cut tables to file for tab, typ in zip([B, T], ['bricks', 'ccds']): fn = '%s-%s-cut.fits' % (typ, opt.region) if os.path.exists(fn): os.remove(fn) tab.writeto(fn) print('Wrote %s' % fn) # Write text files listing ccd and filename names nm1, nm2 = 'ccds-%s.txt' % opt.region, 'filenames-%s.txt' % opt.region if os.path.exists(nm1): os.remove(nm1) if os.path.exists(nm2): os.remove(nm2) f1, f2 = open(nm1, 'w'), open(nm2, 'w') fns = list(set(T.get('image_filename'))) for fn in fns: f2.write('%s\n' % fn.strip()) for ti in T: f1.write('%s\n' % ti.get('image_filename').strip()) f1.close() f2.close() print('Wrote *-names.txt') if opt.brickq_deps: import qdo from legacypipe.survey import on_bricks_dependencies #... find Queue... q = qdo.connect(opt.queue, create_ok=True) print('Connected to QDO queue', opt.queue, q) brick_to_task = dict() I = survey.photometric_ccds(T) print(len(I), 'CCDs are photometric') T.cut(I) I = survey.apply_blacklist(T) print(len(I), 'CCDs are not blacklisted') T.cut(I) print(len(T), 'CCDs remaining') T.wra = T.ra + (T.ra > 180) * -360 wra = rlo - 360 plt.clf() plt.plot(T.wra, T.dec, 'b.') ax = [wra, rhi, dlo, dhi] plt.axis(ax) plt.title('CCDs') plt.savefig('q-ccds.png') B.wra = B.ra + (B.ra > 180) * -360 # this slight overestimate (for DECam images) is fine radius = 0.3 Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius, indexlist=True) ikeep = [] for ib, (b, Iccd) in enumerate(zip(B, Iccds)): if Iccd is None or len(Iccd) == 0: print('No matched CCDs to brick', b.brickname) continue wcs = wcs_for_brick(b) cI = ccds_touching_wcs(wcs, T[np.array(Iccd)]) print(len(cI), 'CCDs touching brick', b.brickname) if len(cI) == 0: continue ikeep.append(ib) B.cut(np.array(ikeep)) print('Cut to', len(B), 'bricks touched by CCDs') for brickq in range(4): I = np.flatnonzero(B.brickq == brickq) print(len(I), 'bricks with brickq =', brickq) J = np.flatnonzero(B.brickq < brickq) preB = B[J] reqs = [] if brickq > 0: for b in B[I]: # find brick dependencies brickdeps = on_bricks_dependencies(b, survey, bricks=preB) # convert to task ids taskdeps = [ brick_to_task.get(b.brickname, None) for b in brickdeps ] # If we dropped a dependency brick from a previous brickq because # of no overlapping CCDs, it won't appear in the brick_to_task map. taskdeps = [t for t in taskdeps if t is not None] reqs.append(taskdeps) plt.clf() plt.plot(B.wra, B.dec, '.', color='0.5') plt.plot(B.wra[I], B.dec[I], 'b.') plt.axis(ax) plt.title('Bricks: brickq=%i' % brickq) plt.savefig('q-bricks-%i.png' % brickq) # submit to qdo queue print('Queuing', len(B[I]), 'bricks') if brickq == 0: reqs = None else: assert (len(I) == len(reqs)) taskids = q.add_multiple(B.brickname[I], requires=reqs) assert (len(taskids) == len(I)) print('Queued', len(taskids), 'bricks') brick_to_task.update(dict(zip(B.brickname[I], taskids))) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() elif opt.near: # Roughly brick radius + DECam image radius radius = 0.35 allI, nil, nil = match_radec(T.ra, T.dec, B.ra, B.dec, radius, nearest=True) else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): expstr = '%08i' % T.expnum[i] outfn = os.path.join( 'forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(survey.survey_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' % # (imgfn, T.image_hdu[i], outfn)) f.write( 'python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n' % (T.expnum[i], T.ccdname[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write( 'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if cmd is None: cmd = '' f.write(cmd + ' '.join(batch) + '\n') cmd = None if opt.command: cmd = 'python legacypipe/run-calib.py ' if opt.opt is not None: cmd += opt.opt + ' ' for j, i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j + 1, 'of', len(allI)) im = survey.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j + 1, 'of', len(allI)) im = survey.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = '%i-%s' % (T.expnum[i], T.ccdname[i]) prefix = 'python legacypipe/run-calib.py ' + opt.opt #('python legacypipe/run-calib.py --expnum %i --ccdname %s' % # (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] prefix = '' if j < 10: print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i], 'filename', T.image_filename[i]) if not opt.nper: f.write(prefix + s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, cmd) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, cmd) f.close() log('Wrote', opt.out) return 0