def main(): brickname = '2428p117' objtype = 'STAR' lobjtype = objtype.lower() decals_sim_dir = '/Users/ioannis/decals_sim_dir/star/2428p117' zoom = [1800,2400,1800,2400] #zoom = [1800,2000,1800,2000] #zoom = None decals = Decals() brickinfo = decals.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brickinfo) W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale() if zoom is not None: # See also runbrick.stage_tims() #(x0,x1,y0,y1) = args.zoom (x0,x1,y0,y1) = zoom W = x1-x0 H = y1-y0 targetwcs = brickwcs.get_subimage(x0, y0, W, H) bounds = brickwcs.radec_bounds() ra_range = bounds[1]-bounds[0] dec_range = bounds[3]-bounds[2] radec_center = brickwcs.radec_center() print(radec_center, ra_range, dec_range) corners = np.array([brickwcs.pixelxy2radec(x,y) for x,y in [(1,1),(W,1),(W,H),(1,H),(1,1)]]) # Identify the CCDs in the region of interest. ccdinfo = decals.ccds_touching_wcs(brickwcs) ccdinfo.about() log.info('Got {} CCDs'.format(len(ccdinfo))) # Generate the catalog of simulated sources here! simcat = fits_table('simcat-{}-{}-00.fits'.format(brickname,lobjtype)) #simcat = fits.getdata('simcat-{}-{}-00.fits'.format(brickname,lobjtype)) simcat.about() blobxy = zip(simcat.x,simcat.y) simdecals = SimDecals(sim_sources=simcat) # For testing! #sim = SimImage(simdecals,ccdinfo[0]) #tim = sim.get_tractor_image(const2psf=True) run_brick(brickname, decals=simdecals, outdir=decals_sim_dir, threads=8, zoom=zoom, wise=False, sdssInit=False, forceAll=True, writePickles=False, blobxy=blobxy, pixPsf=False, stages=['tims','srcs','fitblobs', 'fitblobs_finish','coadds', 'writecat'])
def main(): """Main routine which parses the optional inputs.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='DECaLS simulations.') parser.add_argument('-n', '--nobj', type=long, default=None, metavar='', help='number of objects to simulate (required input)') parser.add_argument('-c', '--chunksize', type=long, default=500, metavar='', help='divide NOBJ into CHUNKSIZE chunks') parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='', help='simulate objects in this brick') parser.add_argument('-o', '--objtype', type=str, default='STAR', metavar='', help='object type (STAR, ELG, LRG, QSO, LSB)') parser.add_argument( '-t', '--threads', type=int, default=8, metavar='', help='number of threads to use when calling The Tractor') parser.add_argument('-s', '--seed', type=long, default=None, metavar='', help='random number seed') parser.add_argument('-z', '--zoom', nargs=4, type=int, metavar='', help='see runbrick.py; (default is 0 3600 0 3600)') parser.add_argument('--rmag-range', nargs=2, type=float, default=(18, 25), metavar='', help='r-band magnitude range') parser.add_argument('--no-qaplots', action='store_true', help='do not generate QAplots') parser.add_argument('-v', '--verbose', dest='verbose', action='count', default=0, help='toggle on verbose output') args = parser.parse_args() if args.nobj is None: parser.print_help() sys.exit(1) # Set the debugging level if args.verbose: lvl = logging.DEBUG else: lvl = logging.INFO logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout) log = logging.getLogger('__name__') brickname = args.brick objtype = args.objtype.upper() lobjtype = objtype.lower() if objtype == 'LRG': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) elif objtype == 'LSB': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) elif objtype == 'QSO': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) # Determine how many chunks we need nobj = args.nobj chunksize = args.chunksize nchunk = long(np.ceil(nobj / chunksize)) log.info('Object type = {}'.format(objtype)) log.info('Number of objects = {}'.format(nobj)) log.info('Chunksize = {}'.format(chunksize)) log.info('Number of chunks = {}'.format(nchunk)) # Optionally zoom into a portion of the brick decals = Decals() brickinfo = decals.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brickinfo) W, H, pixscale = brickwcs.get_width(), brickwcs.get_height( ), brickwcs.pixel_scale() log.info('Brick = {}'.format(brickname)) if args.zoom is not None: # See also runbrick.stage_tims() (x0, x1, y0, y1) = args.zoom W = x1 - x0 H = y1 - y0 brickwcs = brickwcs.get_subimage(x0, y0, W, H) log.info('Zoom (pixel boundaries) = {}'.format(args.zoom)) radec_center = brickwcs.radec_center() log.info('RA, Dec center = {}'.format(radec_center)) log.info('Brick = {}'.format(brickname)) if args.seed is not None: log.info('Random seed = {}'.format(args.seed)) # Pack the input parameters into a meta-data table. meta = Table() meta['brickname'] = Column([brickname], dtype='S10') meta['objtype'] = Column([objtype], dtype='S10') if args.seed is not None: meta['seed'] = Column([args.seed], dtype='i4') meta['nobj'] = Column([args.nobj], dtype='i4') meta['chunksize'] = Column([args.chunksize], dtype='i2') meta['nchunk'] = Column([nchunk], dtype='i2') #meta['RA'] = Column([ra_range],dtype='f8') #meta['DEC'] = Column([dec_range],dtype='f8') if args.zoom is None: meta['zoom'] = Column([[0, 3600, 0, 3600]], dtype='i4') else: meta['zoom'] = Column([args.zoom], dtype='i4') meta['rmag_range'] = Column([args.rmag_range], dtype='f4') print('Hack!') decals_sim_dir = './' metafile = os.path.join(decals_sim_dir, 'metacat-' + brickname + '-' + lobjtype + '.fits') log.info('Writing {}'.format(metafile)) if os.path.isfile(metafile): os.remove(metafile) meta.write(metafile) # Work in chunks. for ichunk in range(nchunk): log.info('Working on chunk {:02d}/{:02d}'.format(ichunk + 1, nchunk)) chunksuffix = '{:02d}'.format(ichunk) # There's probably a smarter way to do this if ((ichunk + 1) * chunksize) > nobj: nobjchunk = (ichunk + 1) * chunksize - nobj else: nobjchunk = chunksize print(nobjchunk) # Build and write out the simulated object catalog. simcat = build_simcat(nobjchunk, brickname, brickwcs, meta) simcatfile = os.path.join( decals_sim_dir, 'simcat-' + brickname + '-' + lobjtype + '-' + chunksuffix + '.fits') log.info('Writing {}'.format(simcatfile)) if os.path.isfile(simcatfile): os.remove(simcatfile) simcat.write(simcatfile) #fitsio.write(simcatfile,simcat) # Use Tractor to just process the blobs containing the simulated sources. simdecals = SimDecals(sim_sources=simcat) # Test code ccdinfo = decals.ccds_touching_wcs(brickwcs) sim = SimImage(simdecals, ccdinfo[0]) tim = sim.get_tractor_image(const2psf=True)
def main(): """Main routine which parses the optional inputs.""" parser = argparse.ArgumentParser(formatter_class=argparse. ArgumentDefaultsHelpFormatter, description='DECaLS simulations.') parser.add_argument('-n', '--nobj', type=long, default=None, metavar='', help='number of objects to simulate (required input)') parser.add_argument('-c', '--chunksize', type=long, default=500, metavar='', help='divide NOBJ into CHUNKSIZE chunks') parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='', help='simulate objects in this brick') parser.add_argument('-o', '--objtype', type=str, default='STAR', metavar='', help='object type (STAR, ELG, LRG, QSO, LSB)') parser.add_argument('-t', '--threads', type=int, default=8, metavar='', help='number of threads to use when calling The Tractor') parser.add_argument('-s', '--seed', type=long, default=None, metavar='', help='random number seed') parser.add_argument('-z', '--zoom', nargs=4, type=int, metavar='', help='see runbrick.py; (default is 0 3600 0 3600)') parser.add_argument('--rmag-range', nargs=2, type=float, default=(18,25), metavar='', help='r-band magnitude range') parser.add_argument('--test', action='store_true', help='run the test code and write out test images') parser.add_argument('-v', '--verbose', action='store_true', help='toggle on verbose output') args = parser.parse_args() if args.nobj is None: parser.print_help() sys.exit(1) # Set the debugging level if args.verbose: lvl = logging.DEBUG else: lvl = logging.INFO logging.basicConfig(format='%(message)s',level=lvl,stream=sys.stdout) log = logging.getLogger('__name__') brickname = args.brick objtype = args.objtype.upper() lobjtype = objtype.lower() if objtype=='LRG': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) elif objtype=='LSB': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) elif objtype=='QSO': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) # Determine how many chunks we need nobj = args.nobj chunksize = args.chunksize nchunk = long(np.ceil(nobj/chunksize)) log.info('Object type = {}'.format(objtype)) log.info('Number of objects = {}'.format(nobj)) log.info('Chunksize = {}'.format(chunksize)) log.info('Number of chunks = {}'.format(nchunk)) # Optionally zoom into a portion of the brick decals = Decals() brickinfo = decals.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brickinfo) W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale() print(W, H, pixscale) log.info('Brick = {}'.format(brickname)) if args.zoom is not None: # See also runbrick.stage_tims() (x0,x1,y0,y1) = args.zoom W = x1-x0 H = y1-y0 brickwcs = brickwcs.get_subimage(x0, y0, W, H) log.info('Zoom (pixel boundaries) = {}'.format(args.zoom)) targetrd = np.array([brickwcs.pixelxy2radec(x,y) for x,y in [(1,1),(W,1),(W,H),(1,H),(1,1)]]) radec_center = brickwcs.radec_center() log.info('RA, Dec center = {}'.format(radec_center)) log.info('Brick = {}'.format(brickname)) if args.seed is not None: log.info('Random seed = {}'.format(args.seed)) # Pack the input parameters into a meta-data table. metacat = Table() metacat['brickname'] = Column([brickname],dtype='S10') metacat['objtype'] = Column([objtype],dtype='S10') if args.seed is not None: metacat['seed'] = Column([args.seed],dtype='i4') metacat['nobj'] = Column([args.nobj],dtype='i4') metacat['chunksize'] = Column([args.chunksize],dtype='i2') metacat['nchunk'] = Column([nchunk],dtype='i2') #metacat['RA'] = Column([ra_range],dtype='f8') #metacat['DEC'] = Column([dec_range],dtype='f8') if args.zoom is None: metacat['zoom'] = Column([[0,3600,0,3600]],dtype='i4') else: metacat['zoom'] = Column([args.zoom],dtype='i4') metacat['rmag_range'] = Column([args.rmag_range],dtype='f4') print('Hack!') decals_sim_dir = './' metafile = os.path.join(decals_sim_dir,'metacat-'+brickname+'-'+lobjtype+'.fits') log.info('Writing {}'.format(metafile)) if os.path.isfile(metafile): os.remove(metafile) metacat.write(metafile) # Work in chunks. for ichunk in range(nchunk): log.info('Working on chunk {:02d}/{:02d}'.format(ichunk+1,nchunk)) chunksuffix = '{:02d}'.format(ichunk) # There's probably a smarter way to do this nobjchunk = np.min((nobj,chunksize)) if (nchunk>1) and (ichunk==(nchunk-1)): nobjchunk = np.max((nobjchunk,nobj%((nchunk-1)*chunksize))) # Build and write out the simulated object catalog. simcat = build_simcat(nobjchunk,brickname,brickwcs,metacat) simcatfile = os.path.join(decals_sim_dir,'simcat-'+brickname+'-'+ lobjtype+'-'+chunksuffix+'.fits') log.info('Writing {}'.format(simcatfile)) if os.path.isfile(simcatfile): os.remove(simcatfile) simcat.write(simcatfile) # # Use Tractor to just process the blobs containing the simulated sources. if args.test: # Test code simdecals = SimDecals(metacat=metacat, simcat=simcat, test=True) ccdinfo = decals.ccds_touching_wcs(brickwcs) sim = SimImage(simdecals, ccdinfo[19]) tim = sim.get_tractor_image(const2psf=True, radecpoly=targetrd) #for ii, ccd in enumerate(ccdinfo): # log.info('Working on CCD {}'.format(ii)) # sim = SimImage(simdecals,ccd) # tim = sim.get_tractor_image(const2psf=True, radecpoly=targetrd, test=args.test) else: simdecals = SimDecals(metacat=metacat,simcat=simcat) blobxy = zip(simcat['x'],simcat['y']) run_brick(brickname, decals=simdecals, outdir=decals_sim_dir, threads=8, zoom=args.zoom, wise=False, sdssInit=False, forceAll=True, writePickles=False, do_calibs=False, write_metrics=False, pixPsf=False, blobxy=blobxy, early_coadds=False, stages=['writecat'])
(h,t) = os.path.split(fn) (h,t) = os.path.split(h) print 'Brick:', t brickname = t bricknames.append(brickname) for brickname in bricknames: scaledfns = glob('data/scaled/decals-dr2/*/%s/image-%s-*.fits' % (brickname[:3], brickname)) for s in scaledfns: print 'Deleting', s os.unlink(s) for brickname in bricknames: brick = decals.get_brick_by_name(brickname) print 'Got brick:', brick bwcs = wcs_for_brick(brick) for zoom in list(reversed(range(5, 13))): print 'Zoom', zoom xx,yy = tiles_touching_wcs(bwcs, zoom) print len(xx), 'tiles touching WCS' for x,y in zip(xx,yy): v = 2 map_decals_dr2(req, v, zoom, x, y, savecache=True, forcecache=True, ignoreCached=True, hack_jpeg=True, drname='decals-dr2')
(h, t) = os.path.split(h) print 'Brick:', t brickname = t bricknames.append(brickname) for brickname in bricknames: scaledfns = glob('data/scaled/decals-dr2/*/%s/image-%s-*.fits' % (brickname[:3], brickname)) for s in scaledfns: print 'Deleting', s os.unlink(s) for brickname in bricknames: brick = decals.get_brick_by_name(brickname) print 'Got brick:', brick bwcs = wcs_for_brick(brick) for zoom in list(reversed(range(5, 13))): print 'Zoom', zoom xx, yy = tiles_touching_wcs(bwcs, zoom) print len(xx), 'tiles touching WCS' for x, y in zip(xx, yy): v = 2 map_decals_dr2(req, v, zoom, x, y, savecache=True, forcecache=True,
def main(): """Main routine which parses the optional inputs.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='DECaLS simulations.') parser.add_argument('-n', '--nobj', type=long, default=None, metavar='', help='number of objects to simulate (required input)') parser.add_argument('-c', '--chunksize', type=long, default=500, metavar='', help='divide NOBJ into CHUNKSIZE chunks') parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='', help='simulate objects in this brick') parser.add_argument('-o', '--objtype', type=str, default='STAR', metavar='', help='object type (STAR, ELG, LRG, QSO, LSB)') parser.add_argument( '-t', '--threads', type=int, default=8, metavar='', help='number of threads to use when calling The Tractor') parser.add_argument('-s', '--seed', type=long, default=None, metavar='', help='random number seed') parser.add_argument('-z', '--zoom', nargs=4, type=int, metavar='', help='see runbrick.py; (default is 0 3600 0 3600)') parser.add_argument('--rmag-range', nargs=2, type=float, default=(18, 26), metavar='', help='r-band magnitude range') parser.add_argument('-v', '--verbose', action='store_true', help='toggle on verbose output') args = parser.parse_args() if args.nobj is None: parser.print_help() sys.exit(1) # Set the debugging level if args.verbose: lvl = logging.DEBUG else: lvl = logging.INFO logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout) log = logging.getLogger('__name__') brickname = args.brick objtype = args.objtype.upper() lobjtype = objtype.lower() if objtype == 'LRG': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) elif objtype == 'LSB': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) elif objtype == 'QSO': log.warning('{} objtype not yet supported!'.format(objtype)) sys.exit(1) # Determine how many chunks we need nobj = args.nobj chunksize = args.chunksize nchunk = long(np.ceil(nobj / chunksize)) log.info('Object type = {}'.format(objtype)) log.info('Number of objects = {}'.format(nobj)) log.info('Chunksize = {}'.format(chunksize)) log.info('Number of chunks = {}'.format(nchunk)) # Optionally zoom into a portion of the brick decals = Decals() brickinfo = decals.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brickinfo) W, H, pixscale = brickwcs.get_width(), brickwcs.get_height( ), brickwcs.pixel_scale() print(W, H, pixscale) log.info('Brick = {}'.format(brickname)) if args.zoom is not None: # See also runbrick.stage_tims() (x0, x1, y0, y1) = args.zoom W = x1 - x0 H = y1 - y0 brickwcs = brickwcs.get_subimage(x0, y0, W, H) log.info('Zoom (pixel boundaries) = {}'.format(args.zoom)) targetrd = np.array([ brickwcs.pixelxy2radec(x, y) for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)] ]) radec_center = brickwcs.radec_center() log.info('RA, Dec center = {}'.format(radec_center)) log.info('Brick = {}'.format(brickname)) # Pack the input parameters into a meta-data table. metacat = Table() metacat['brickname'] = Column([brickname], dtype='S10') metacat['objtype'] = Column([objtype], dtype='S10') metacat['nobj'] = Column([args.nobj], dtype='i4') metacat['chunksize'] = Column([args.chunksize], dtype='i2') metacat['nchunk'] = Column([nchunk], dtype='i2') #metacat['RA'] = Column([ra_range],dtype='f8') #metacat['DEC'] = Column([dec_range],dtype='f8') if args.zoom is None: metacat['zoom'] = Column([[0, 3600, 0, 3600]], dtype='i4') else: metacat['zoom'] = Column([args.zoom], dtype='i4') metacat['rmag_range'] = Column([args.rmag_range], dtype='f4') # Deal with the random seed if args.seed is not None: log.info('Random seed = {}'.format(args.seed)) metacat['seed'] = Column([args.seed], dtype='i4') rand = np.random.RandomState(args.seed) seeds = rand.random_integers(0, 2**18, nchunk) if 'DECALS_SIM_DIR' in os.environ: decals_sim_dir = os.getenv('DECALS_SIM_DIR') else: decals_sim_dir = '.' if not os.path.exists(os.path.join(decals_sim_dir, brickname)): os.makedirs(os.path.join(decals_sim_dir, brickname)) metafile = os.path.join(decals_sim_dir, brickname, 'metacat-' + brickname + '-' + lobjtype + '.fits') log.info('Writing {}'.format(metafile)) if os.path.isfile(metafile): os.remove(metafile) metacat.write(metafile) # Work in chunks. for ichunk in range(nchunk): log.info('Working on chunk {:02d}/{:02d}'.format(ichunk + 1, nchunk)) chunksuffix = '{:02d}'.format(ichunk) # There's probably a smarter way to do this nobjchunk = np.min((nobj, chunksize)) if (nchunk > 1) and (ichunk == (nchunk - 1)): nobjchunk = np.max((nobjchunk, nobj % ((nchunk - 1) * chunksize))) # Build and write out the simulated object catalog. simcat = build_simcat(nobjchunk, brickname, brickwcs, metacat, seeds[ichunk]) simcatfile = os.path.join( decals_sim_dir, brickname, 'simcat-' + brickname + '-' + lobjtype + '-' + chunksuffix + '.fits') log.info('Writing {}'.format(simcatfile)) if os.path.isfile(simcatfile): os.remove(simcatfile) simcat.write(simcatfile) # # Use Tractor to just process the blobs containing the simulated sources. simdecals = SimDecals(metacat=metacat, simcat=simcat) blobxy = zip(simcat['x'], simcat['y']) run_brick(brickname, decals=simdecals, outdir=os.path.join(decals_sim_dir, brickname), threads=args.threads, zoom=args.zoom, wise=False, sdssInit=False, forceAll=True, writePickles=False, do_calibs=True, write_metrics=False, pixPsf=True, blobxy=blobxy, early_coadds=False, stages=['writecat'], splinesky=True) log.info('Cleaning up...') shutil.move( os.path.join(decals_sim_dir, brickname, 'tractor', brickname[:3], 'tractor-' + brickname + '.fits'), os.path.join( decals_sim_dir, brickname, 'tractor-' + brickname + '-' + lobjtype + '-' + chunksuffix + '.fits')) shutil.move( os.path.join(decals_sim_dir, brickname, 'coadd', brickname[:3], brickname, 'decals-' + brickname + '-image.jpg'), os.path.join( decals_sim_dir, brickname, 'qa-' + brickname + '-' + lobjtype + '-image-' + chunksuffix + '.jpg')) shutil.move( os.path.join(decals_sim_dir, brickname, 'coadd', brickname[:3], brickname, 'decals-' + brickname + '-resid.jpg'), os.path.join( decals_sim_dir, brickname, 'qa-' + brickname + '-' + lobjtype + '-resid-' + chunksuffix + '.jpg')) shutil.rmtree(os.path.join(decals_sim_dir, brickname, 'coadd')) shutil.rmtree(os.path.join(decals_sim_dir, brickname, 'tractor')) #shutil.rmtree(os.path.join(decals_sim_dir,brickname,'images')) #shutil.rmtree(os.path.join(decals_sim_dir,brickname,'metrics')) # Write a log file log.info('All done!')
def main(): import optparse parser = optparse.OptionParser() parser.add_option('--zoom', '-z', type=int, action='append', default=[], help='Add zoom level; default 13') parser.add_option('--threads', type=int, default=1, help='Number of threads') parser.add_option('--y0', type=int, default=0, help='Start row') parser.add_option('--y1', type=int, default=None, help='End row (non-inclusive)') parser.add_option('--x0', type=int, default=None) parser.add_option('--x1', type=int, default=None) parser.add_option('-x', type=int) parser.add_option('-y', type=int) parser.add_option('--mindec', type=float, default=None, help='Minimum Dec to run') parser.add_option('--maxdec', type=float, default=None, help='Maximum Dec to run') parser.add_option('--minra', type=float, default=0., help='Minimum RA to run') parser.add_option('--maxra', type=float, default=360., help='Maximum RA to run') parser.add_option('--near', action='store_true', help='Only run tiles near bricks') parser.add_option('--near-ccds', action='store_true', help='Only run tiles near CCDs') parser.add_option('--queue', action='store_true', default=False, help='Print qdo commands') parser.add_option('--all', action='store_true', help='Render all tiles') parser.add_option('--ignore', action='store_true', help='Ignore cached tile files', default=False) parser.add_option('--top', action='store_true', help='Top levels of the pyramid') parser.add_option('--kind', default='image') parser.add_option('--scale', action='store_true', help='Scale images?') parser.add_option('--coadd', action='store_true', help='Create SDSS coadd images?') parser.add_option('--grass', action='store_true', help='progress plot') opt,args = parser.parse_args() if len(opt.zoom) == 0: opt.zoom = [13] mp = multiproc(opt.threads) if opt.kind == 'sdss': if opt.maxdec is None: opt.maxdec = 90 if opt.mindec is None: opt.mindec = -25 elif opt.kind in ['halpha', 'unwise-neo1']: if opt.maxdec is None: opt.maxdec = 90 if opt.mindec is None: opt.mindec = -90 else: if opt.maxdec is None: opt.maxdec = 40 if opt.mindec is None: opt.mindec = -20 if opt.top: top_levels(mp, opt) sys.exit(0) from legacypipe.common import Decals decals = Decals() if opt.near: if opt.kind == 'sdss': B = fits_table(os.path.join(settings.DATA_DIR, 'bricks-sdssco.fits')) else: B = decals.get_bricks() print len(B), 'bricks' if opt.scale: opt.near_ccds = True if opt.near_ccds: if opt.kind == 'sdss': C = fits_table(os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'), columns=['rerun','ra','dec', 'run', 'camcol', 'field', 'score']) C.cut(C.rerun == '301') C.cut(C.score >= 0.6) #C.delete_column('rerun') # SDSS field size radius = 1.01 * np.hypot(10., 14.)/2. / 60. ccdsize = radius print len(C), 'SDSS fields' else: C = decals.get_ccds() print len(C), 'CCDs' ccdsize = 0.2 if opt.x is not None: opt.x0 = opt.x opt.x1 = opt.x + 1 if opt.y is not None: opt.y0 = opt.y opt.y1 = opt.y + 1 if opt.coadd and opt.kind == 'sdss': from legacypipe.common import wcs_for_brick from map.views import trymakedirs B = decals.get_bricks() print len(B), 'bricks' B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec)) print len(B), 'in Dec range' B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra)) print len(B), 'in RA range' if opt.queue: # ~ square-degree tiles # RA slices rr = np.arange(opt.minra , opt.maxra +1) dd = np.arange(opt.mindec, opt.maxdec+1) for rlo,rhi in zip(rr, rr[1:]): for dlo,dhi in zip(dd, dd[1:]): print 'time python render-tiles.py --kind sdss --coadd --minra %f --maxra %f --mindec %f --maxdec %f' % (rlo, rhi, dlo, dhi) sys.exit(0) if opt.grass: basedir = settings.DATA_DIR codir = os.path.join(basedir, 'coadd', 'sdssco') rr,dd = [],[] exist = [] for i,b in enumerate(B): print 'Brick', b.brickname, fn = os.path.join(codir, b.brickname[:3], 'sdssco-%s-%s.fits' % (b.brickname, 'r')) print '-->', fn, if not os.path.exists(fn): print continue print 'found' rr.append(b.ra) dd.append(b.dec) exist.append(i) exist = np.array(exist) B.cut(exist) B.writeto('bricks-sdssco-exist.fits') import pylab as plt plt.clf() plt.plot(rr, dd, 'k.') plt.title('SDSS coadd tiles') plt.savefig('sdss.png') sys.exit(0) basedir = settings.DATA_DIR codir = os.path.join(basedir, 'coadd', 'sdssco') for b in B: print 'Brick', b.brickname wcs = wcs_for_brick(b, W=2400, H=2400, pixscale=0.396) bands = 'gri' dirnm = os.path.join(codir, b.brickname[:3]) fns = [os.path.join(dirnm, 'sdssco-%s-%s.fits' % (b.brickname, band)) for band in bands] hdr = fitsio.FITSHDR() hdr['SURVEY'] = 'SDSS' wcs.add_to_header(hdr) if all([os.path.exists(fn) for fn in fns]): print 'Already exist' continue ims = map_sdss(req, 1, 0, 0, 0, get_images=True, wcs=wcs, ignoreCached=True, forcescale=0) if ims is None: print 'No overlap' continue trymakedirs(os.path.join(dirnm, 'xxx')) for fn,band,im in zip(fns,bands, ims): fitsio.write(fn, im, header=hdr, clobber=True) print 'Wrote', fn # Also write scaled versions dirnm = os.path.join(basedir, 'scaled', 'sdssco') scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(brickname).3s', 'sdssco-%(brickname)s-%(band)s.fits') for im,band in zip(ims,bands): scalekwargs = dict(band=band, brick=b.brickid, brickname=b.brickname) imwcs = wcs for scale in range(1, 7): print 'Writing scale level', scale im,imwcs,sfn = get_scaled(scalepat, scalekwargs, scale, None, wcs=imwcs, img=im, return_data=True) sys.exit(0) if opt.scale: if opt.kind == 'sdss': C.cut((C.dec >= opt.mindec) * (C.dec < opt.maxdec)) print len(C), 'in Dec range' C.cut((C.ra >= opt.minra) * (C.ra < opt.maxra)) print len(C), 'in RA range' from astrometry.sdss import AsTransWrapper, DR9 sdss = DR9(basedir=settings.SDSS_DIR) sdss.saveUnzippedFiles(settings.SDSS_DIR) sdss.setFitsioReadBZ2() if settings.SDSS_PHOTOOBJS: sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS, resolve=settings.SDSS_RESOLVE) basedir = settings.DATA_DIR scaledir = 'sdss' dirnm = os.path.join(basedir, 'scaled', scaledir) for im in C: from map.views import _read_sip_wcs #if im.rerun != '301': # continue for band in 'gri': scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i', 'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits') tmpsuff = '.tmp%08i' % np.random.randint(100000000) basefn = sdss.retrieve('frame', im.run, im.camcol, field=im.field, band=band, rerun=im.rerun, tempsuffix=tmpsuff) fnargs = dict(band=band, rerun=im.rerun, run=im.run, camcol=im.camcol, field=im.field) scaled = 1 fn = get_scaled(scalepat, fnargs, scaled, basefn, read_base_wcs=read_astrans, read_wcs=_read_sip_wcs) print 'get_scaled:', fn return 0 else: assert(False) for zoom in opt.zoom: N = 2**zoom if opt.y1 is None: y1 = N else: y1 = opt.y1 if opt.x0 is None: opt.x0 = 0 x1 = opt.x1 if x1 is None: x1 = N # Find grid of Ra,Dec tile centers and select the ones near DECaLS bricks. rr,dd = [],[] yy = np.arange(opt.y0, y1) xx = np.arange(opt.x0, x1) if opt.grass: import pylab as plt tileexists = np.zeros((len(yy),len(xx)), bool) basedir = settings.DATA_DIR ver = tileversions[opt.kind][-1] tiledir = os.path.join(basedir, 'tiles', opt.kind, '%i'%ver, '%i'%zoom) for dirpath,dirnames,filenames in os.walk(tiledir): # change walk order dirnames.sort() if len(filenames) == 0: continue print 'Dirpath', dirpath #print 'Dirnames', dirnames #print 'Filenames', filenames # check for symlinks if False: fns = [] for fn in filenames: fullfn = os.path.join(tiledir, dirpath, fn) if os.path.isfile(fullfn) and not os.path.islink(fullfn): fns.append(fn) print len(fns), 'of', len(filenames), 'are files (not symlinks)' filenames = fns x = os.path.basename(dirpath) x = int(x) #print 'x', x yy = [int(fn.replace('.jpg','')) for fn in filenames] #print 'yy', yy print len(yy), 'tiles' for y in yy: tileexists[y - opt.y0, x - opt.x0] = True plt.clf() plt.imshow(tileexists, interpolation='nearest', origin='upper', vmin=0, vmax=1, cmap='gray') fn = 'exist-%s-z%02i' % (opt.kind, zoom) plt.savefig(fn+'.png') fitsio.write(fn+'.fits', tileexists, clobber=True) print 'Wrote', fn+'.png and', fn+'.fits' continue if not opt.all: for y in yy: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, y) r,d = wcs.get_center() dd.append(d) for x in xx: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, x, 0) r,d = wcs.get_center() rr.append(r) dd = np.array(dd) rr = np.array(rr) if len(dd) > 1: tilesize = max(np.abs(np.diff(dd))) print 'Tile size:', tilesize else: if opt.near_ccds or opt.near: try: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0+1) r2,d2 = wcs.get_center() except: wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0-1) r2,d2 = wcs.get_center() tilesize = np.abs(dd[0] - d2) print 'Tile size:', tilesize else: tilesize = 180. I = np.flatnonzero((dd >= opt.mindec) * (dd <= opt.maxdec)) print 'Keeping', len(I), 'Dec points between', opt.mindec, 'and', opt.maxdec dd = dd[I] yy = yy[I] I = np.flatnonzero((rr >= opt.minra) * (rr <= opt.maxra)) print 'Keeping', len(I), 'RA points between', opt.minra, 'and', opt.maxra rr = rr[I] xx = xx[I] print len(rr), 'RA points x', len(dd), 'Dec points' print 'x tile range:', xx.min(), xx.max(), 'y tile range:', yy.min(), yy.max() for iy,y in enumerate(yy): print print 'Y row', y if opt.near: d = dd[iy] I,J,dist = match_radec(rr, d+np.zeros_like(rr), B.ra, B.dec, 0.25 + tilesize, nearest=True) if len(I) == 0: print 'No matches to bricks' continue keep = np.zeros(len(rr), bool) keep[I] = True print 'Keeping', sum(keep), 'tiles in row', y, 'Dec', d x = xx[keep] elif opt.near_ccds: d = dd[iy] print 'RA range of tiles:', rr.min(), rr.max() print 'Dec of tile row:', d I,J,dist = match_radec(rr, d+np.zeros_like(rr), C.ra, C.dec, ccdsize + tilesize, nearest=True) if len(I) == 0: print 'No matches to CCDs' continue keep = np.zeros(len(rr), bool) keep[I] = True print 'Keeping', sum(keep), 'tiles in row', y, 'Dec', d x = xx[keep] else: x = xx if opt.queue: cmd = 'python -u render-tiles.py --zoom %i --y0 %i --y1 %i --kind %s' % (zoom, y, y+1, opt.kind) if opt.near_ccds: cmd += ' --near-ccds' if opt.all: cmd += ' --all' if opt.ignore: cmd += ' --ignore' print cmd continue # if opt.grass: # for xi in x: # basedir = settings.DATA_DIR # ver = tileversions[opt.kind][-1] # tilefn = os.path.join(basedir, 'tiles', opt.kind, # '%i/%i/%i/%i.jpg' % (ver, zoom, xi, y)) # print 'Checking for', tilefn # if os.path.exists(tilefn): # print 'EXISTS' # tileexists[yi-opt.y0, xi-opt.x0] # continue args = [] for xi in x: args.append((opt.kind,zoom,xi,y, opt.ignore)) print 'Rendering', len(args), 'tiles in row y =', y mp.map(_bounce_one_tile, args, chunksize=min(100, max(1, len(args)/opt.threads))) print 'Rendered', len(args), 'tiles'