def main():
brickname = '2428p117'
objtype = 'STAR'
lobjtype = objtype.lower()
decals_sim_dir = '/Users/ioannis/decals_sim_dir/star/2428p117'
zoom = [1800,2400,1800,2400]
#zoom = [1800,2000,1800,2000]
#zoom = None
decals = Decals()
brickinfo = decals.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()
if zoom is not None:
# See also runbrick.stage_tims()
#(x0,x1,y0,y1) = args.zoom
(x0,x1,y0,y1) = zoom
W = x1-x0
H = y1-y0
targetwcs = brickwcs.get_subimage(x0, y0, W, H)
bounds = brickwcs.radec_bounds()
ra_range = bounds[1]-bounds[0]
dec_range = bounds[3]-bounds[2]
radec_center = brickwcs.radec_center()
print(radec_center, ra_range, dec_range)
corners = np.array([brickwcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
# Identify the CCDs in the region of interest.
ccdinfo = decals.ccds_touching_wcs(brickwcs)
ccdinfo.about()
log.info('Got {} CCDs'.format(len(ccdinfo)))
# Generate the catalog of simulated sources here!
simcat = fits_table('simcat-{}-{}-00.fits'.format(brickname,lobjtype))
#simcat = fits.getdata('simcat-{}-{}-00.fits'.format(brickname,lobjtype))
simcat.about()
blobxy = zip(simcat.x,simcat.y)
simdecals = SimDecals(sim_sources=simcat)
# For testing!
#sim = SimImage(simdecals,ccdinfo[0])
#tim = sim.get_tractor_image(const2psf=True)
run_brick(brickname, decals=simdecals, outdir=decals_sim_dir,
threads=8, zoom=zoom, wise=False, sdssInit=False,
forceAll=True, writePickles=False, blobxy=blobxy,
pixPsf=False, stages=['tims','srcs','fitblobs',
'fitblobs_finish','coadds',
'writecat'])
def main():
"""Main routine which parses the optional inputs."""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='DECaLS simulations.')
parser.add_argument('-n',
'--nobj',
type=long,
default=None,
metavar='',
help='number of objects to simulate (required input)')
parser.add_argument('-c',
'--chunksize',
type=long,
default=500,
metavar='',
help='divide NOBJ into CHUNKSIZE chunks')
parser.add_argument('-b',
'--brick',
type=str,
default='2428p117',
metavar='',
help='simulate objects in this brick')
parser.add_argument('-o',
'--objtype',
type=str,
default='STAR',
metavar='',
help='object type (STAR, ELG, LRG, QSO, LSB)')
parser.add_argument(
'-t',
'--threads',
type=int,
default=8,
metavar='',
help='number of threads to use when calling The Tractor')
parser.add_argument('-s',
'--seed',
type=long,
default=None,
metavar='',
help='random number seed')
parser.add_argument('-z',
'--zoom',
nargs=4,
type=int,
metavar='',
help='see runbrick.py; (default is 0 3600 0 3600)')
parser.add_argument('--rmag-range',
nargs=2,
type=float,
default=(18, 25),
metavar='',
help='r-band magnitude range')
parser.add_argument('--no-qaplots',
action='store_true',
help='do not generate QAplots')
parser.add_argument('-v',
'--verbose',
dest='verbose',
action='count',
default=0,
help='toggle on verbose output')
args = parser.parse_args()
if args.nobj is None:
parser.print_help()
sys.exit(1)
# Set the debugging level
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout)
log = logging.getLogger('__name__')
brickname = args.brick
objtype = args.objtype.upper()
lobjtype = objtype.lower()
if objtype == 'LRG':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
elif objtype == 'LSB':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
elif objtype == 'QSO':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
# Determine how many chunks we need
nobj = args.nobj
chunksize = args.chunksize
nchunk = long(np.ceil(nobj / chunksize))
log.info('Object type = {}'.format(objtype))
log.info('Number of objects = {}'.format(nobj))
log.info('Chunksize = {}'.format(chunksize))
log.info('Number of chunks = {}'.format(nchunk))
# Optionally zoom into a portion of the brick
decals = Decals()
brickinfo = decals.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(
), brickwcs.pixel_scale()
log.info('Brick = {}'.format(brickname))
if args.zoom is not None: # See also runbrick.stage_tims()
(x0, x1, y0, y1) = args.zoom
W = x1 - x0
H = y1 - y0
brickwcs = brickwcs.get_subimage(x0, y0, W, H)
log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
radec_center = brickwcs.radec_center()
log.info('RA, Dec center = {}'.format(radec_center))
log.info('Brick = {}'.format(brickname))
if args.seed is not None:
log.info('Random seed = {}'.format(args.seed))
# Pack the input parameters into a meta-data table.
meta = Table()
meta['brickname'] = Column([brickname], dtype='S10')
meta['objtype'] = Column([objtype], dtype='S10')
if args.seed is not None:
meta['seed'] = Column([args.seed], dtype='i4')
meta['nobj'] = Column([args.nobj], dtype='i4')
meta['chunksize'] = Column([args.chunksize], dtype='i2')
meta['nchunk'] = Column([nchunk], dtype='i2')
#meta['RA'] = Column([ra_range],dtype='f8')
#meta['DEC'] = Column([dec_range],dtype='f8')
if args.zoom is None:
meta['zoom'] = Column([[0, 3600, 0, 3600]], dtype='i4')
else:
meta['zoom'] = Column([args.zoom], dtype='i4')
meta['rmag_range'] = Column([args.rmag_range], dtype='f4')
print('Hack!')
decals_sim_dir = './'
metafile = os.path.join(decals_sim_dir,
'metacat-' + brickname + '-' + lobjtype + '.fits')
log.info('Writing {}'.format(metafile))
if os.path.isfile(metafile):
os.remove(metafile)
meta.write(metafile)
# Work in chunks.
for ichunk in range(nchunk):
log.info('Working on chunk {:02d}/{:02d}'.format(ichunk + 1, nchunk))
chunksuffix = '{:02d}'.format(ichunk)
# There's probably a smarter way to do this
if ((ichunk + 1) * chunksize) > nobj:
nobjchunk = (ichunk + 1) * chunksize - nobj
else:
nobjchunk = chunksize
print(nobjchunk)
# Build and write out the simulated object catalog.
simcat = build_simcat(nobjchunk, brickname, brickwcs, meta)
simcatfile = os.path.join(
decals_sim_dir, 'simcat-' + brickname + '-' + lobjtype + '-' +
chunksuffix + '.fits')
log.info('Writing {}'.format(simcatfile))
if os.path.isfile(simcatfile):
os.remove(simcatfile)
simcat.write(simcatfile)
#fitsio.write(simcatfile,simcat)
# Use Tractor to just process the blobs containing the simulated sources.
simdecals = SimDecals(sim_sources=simcat)
# Test code
ccdinfo = decals.ccds_touching_wcs(brickwcs)
sim = SimImage(simdecals, ccdinfo[0])
tim = sim.get_tractor_image(const2psf=True)
def main():
"""Main routine which parses the optional inputs."""
parser = argparse.ArgumentParser(formatter_class=argparse.
ArgumentDefaultsHelpFormatter,
description='DECaLS simulations.')
parser.add_argument('-n', '--nobj', type=long, default=None, metavar='',
help='number of objects to simulate (required input)')
parser.add_argument('-c', '--chunksize', type=long, default=500, metavar='',
help='divide NOBJ into CHUNKSIZE chunks')
parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='',
help='simulate objects in this brick')
parser.add_argument('-o', '--objtype', type=str, default='STAR', metavar='',
help='object type (STAR, ELG, LRG, QSO, LSB)')
parser.add_argument('-t', '--threads', type=int, default=8, metavar='',
help='number of threads to use when calling The Tractor')
parser.add_argument('-s', '--seed', type=long, default=None, metavar='',
help='random number seed')
parser.add_argument('-z', '--zoom', nargs=4, type=int, metavar='',
help='see runbrick.py; (default is 0 3600 0 3600)')
parser.add_argument('--rmag-range', nargs=2, type=float, default=(18,25), metavar='',
help='r-band magnitude range')
parser.add_argument('--test', action='store_true',
help='run the test code and write out test images')
parser.add_argument('-v', '--verbose', action='store_true',
help='toggle on verbose output')
args = parser.parse_args()
if args.nobj is None:
parser.print_help()
sys.exit(1)
# Set the debugging level
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(format='%(message)s',level=lvl,stream=sys.stdout)
log = logging.getLogger('__name__')
brickname = args.brick
objtype = args.objtype.upper()
lobjtype = objtype.lower()
if objtype=='LRG':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
elif objtype=='LSB':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
elif objtype=='QSO':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
# Determine how many chunks we need
nobj = args.nobj
chunksize = args.chunksize
nchunk = long(np.ceil(nobj/chunksize))
log.info('Object type = {}'.format(objtype))
log.info('Number of objects = {}'.format(nobj))
log.info('Chunksize = {}'.format(chunksize))
log.info('Number of chunks = {}'.format(nchunk))
# Optionally zoom into a portion of the brick
decals = Decals()
brickinfo = decals.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()
print(W, H, pixscale)
log.info('Brick = {}'.format(brickname))
if args.zoom is not None: # See also runbrick.stage_tims()
(x0,x1,y0,y1) = args.zoom
W = x1-x0
H = y1-y0
brickwcs = brickwcs.get_subimage(x0, y0, W, H)
log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
targetrd = np.array([brickwcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
radec_center = brickwcs.radec_center()
log.info('RA, Dec center = {}'.format(radec_center))
log.info('Brick = {}'.format(brickname))
if args.seed is not None:
log.info('Random seed = {}'.format(args.seed))
# Pack the input parameters into a meta-data table.
metacat = Table()
metacat['brickname'] = Column([brickname],dtype='S10')
metacat['objtype'] = Column([objtype],dtype='S10')
if args.seed is not None:
metacat['seed'] = Column([args.seed],dtype='i4')
metacat['nobj'] = Column([args.nobj],dtype='i4')
metacat['chunksize'] = Column([args.chunksize],dtype='i2')
metacat['nchunk'] = Column([nchunk],dtype='i2')
#metacat['RA'] = Column([ra_range],dtype='f8')
#metacat['DEC'] = Column([dec_range],dtype='f8')
if args.zoom is None:
metacat['zoom'] = Column([[0,3600,0,3600]],dtype='i4')
else:
metacat['zoom'] = Column([args.zoom],dtype='i4')
metacat['rmag_range'] = Column([args.rmag_range],dtype='f4')
print('Hack!')
decals_sim_dir = './'
metafile = os.path.join(decals_sim_dir,'metacat-'+brickname+'-'+lobjtype+'.fits')
log.info('Writing {}'.format(metafile))
if os.path.isfile(metafile):
os.remove(metafile)
metacat.write(metafile)
# Work in chunks.
for ichunk in range(nchunk):
log.info('Working on chunk {:02d}/{:02d}'.format(ichunk+1,nchunk))
chunksuffix = '{:02d}'.format(ichunk)
# There's probably a smarter way to do this
nobjchunk = np.min((nobj,chunksize))
if (nchunk>1) and (ichunk==(nchunk-1)):
nobjchunk = np.max((nobjchunk,nobj%((nchunk-1)*chunksize)))
# Build and write out the simulated object catalog.
simcat = build_simcat(nobjchunk,brickname,brickwcs,metacat)
simcatfile = os.path.join(decals_sim_dir,'simcat-'+brickname+'-'+
lobjtype+'-'+chunksuffix+'.fits')
log.info('Writing {}'.format(simcatfile))
if os.path.isfile(simcatfile):
os.remove(simcatfile)
simcat.write(simcatfile)
# # Use Tractor to just process the blobs containing the simulated sources.
if args.test:
# Test code
simdecals = SimDecals(metacat=metacat, simcat=simcat, test=True)
ccdinfo = decals.ccds_touching_wcs(brickwcs)
sim = SimImage(simdecals, ccdinfo[19])
tim = sim.get_tractor_image(const2psf=True, radecpoly=targetrd)
#for ii, ccd in enumerate(ccdinfo):
# log.info('Working on CCD {}'.format(ii))
# sim = SimImage(simdecals,ccd)
# tim = sim.get_tractor_image(const2psf=True, radecpoly=targetrd, test=args.test)
else:
simdecals = SimDecals(metacat=metacat,simcat=simcat)
blobxy = zip(simcat['x'],simcat['y'])
run_brick(brickname, decals=simdecals, outdir=decals_sim_dir,
threads=8, zoom=args.zoom, wise=False, sdssInit=False,
forceAll=True, writePickles=False, do_calibs=False,
write_metrics=False, pixPsf=False, blobxy=blobxy,
early_coadds=False, stages=['writecat'])
(h,t) = os.path.split(fn)
(h,t) = os.path.split(h)
print 'Brick:', t
brickname = t
bricknames.append(brickname)
for brickname in bricknames:
scaledfns = glob('data/scaled/decals-dr2/*/%s/image-%s-*.fits' %
(brickname[:3], brickname))
for s in scaledfns:
print 'Deleting', s
os.unlink(s)
for brickname in bricknames:
brick = decals.get_brick_by_name(brickname)
print 'Got brick:', brick
bwcs = wcs_for_brick(brick)
for zoom in list(reversed(range(5, 13))):
print 'Zoom', zoom
xx,yy = tiles_touching_wcs(bwcs, zoom)
print len(xx), 'tiles touching WCS'
for x,y in zip(xx,yy):
v = 2
map_decals_dr2(req, v, zoom, x, y, savecache=True, forcecache=True,
ignoreCached=True,
hack_jpeg=True, drname='decals-dr2')
(h, t) = os.path.split(h)
print 'Brick:', t
brickname = t
bricknames.append(brickname)
for brickname in bricknames:
scaledfns = glob('data/scaled/decals-dr2/*/%s/image-%s-*.fits' %
(brickname[:3], brickname))
for s in scaledfns:
print 'Deleting', s
os.unlink(s)
for brickname in bricknames:
brick = decals.get_brick_by_name(brickname)
print 'Got brick:', brick
bwcs = wcs_for_brick(brick)
for zoom in list(reversed(range(5, 13))):
print 'Zoom', zoom
xx, yy = tiles_touching_wcs(bwcs, zoom)
print len(xx), 'tiles touching WCS'
for x, y in zip(xx, yy):
v = 2
map_decals_dr2(req,
v,
zoom,
x,
y,
savecache=True,
forcecache=True,
def main():
"""Main routine which parses the optional inputs."""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='DECaLS simulations.')
parser.add_argument('-n',
'--nobj',
type=long,
default=None,
metavar='',
help='number of objects to simulate (required input)')
parser.add_argument('-c',
'--chunksize',
type=long,
default=500,
metavar='',
help='divide NOBJ into CHUNKSIZE chunks')
parser.add_argument('-b',
'--brick',
type=str,
default='2428p117',
metavar='',
help='simulate objects in this brick')
parser.add_argument('-o',
'--objtype',
type=str,
default='STAR',
metavar='',
help='object type (STAR, ELG, LRG, QSO, LSB)')
parser.add_argument(
'-t',
'--threads',
type=int,
default=8,
metavar='',
help='number of threads to use when calling The Tractor')
parser.add_argument('-s',
'--seed',
type=long,
default=None,
metavar='',
help='random number seed')
parser.add_argument('-z',
'--zoom',
nargs=4,
type=int,
metavar='',
help='see runbrick.py; (default is 0 3600 0 3600)')
parser.add_argument('--rmag-range',
nargs=2,
type=float,
default=(18, 26),
metavar='',
help='r-band magnitude range')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='toggle on verbose output')
args = parser.parse_args()
if args.nobj is None:
parser.print_help()
sys.exit(1)
# Set the debugging level
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout)
log = logging.getLogger('__name__')
brickname = args.brick
objtype = args.objtype.upper()
lobjtype = objtype.lower()
if objtype == 'LRG':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
elif objtype == 'LSB':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
elif objtype == 'QSO':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
# Determine how many chunks we need
nobj = args.nobj
chunksize = args.chunksize
nchunk = long(np.ceil(nobj / chunksize))
log.info('Object type = {}'.format(objtype))
log.info('Number of objects = {}'.format(nobj))
log.info('Chunksize = {}'.format(chunksize))
log.info('Number of chunks = {}'.format(nchunk))
# Optionally zoom into a portion of the brick
decals = Decals()
brickinfo = decals.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(
), brickwcs.pixel_scale()
print(W, H, pixscale)
log.info('Brick = {}'.format(brickname))
if args.zoom is not None: # See also runbrick.stage_tims()
(x0, x1, y0, y1) = args.zoom
W = x1 - x0
H = y1 - y0
brickwcs = brickwcs.get_subimage(x0, y0, W, H)
log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
targetrd = np.array([
brickwcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
radec_center = brickwcs.radec_center()
log.info('RA, Dec center = {}'.format(radec_center))
log.info('Brick = {}'.format(brickname))
# Pack the input parameters into a meta-data table.
metacat = Table()
metacat['brickname'] = Column([brickname], dtype='S10')
metacat['objtype'] = Column([objtype], dtype='S10')
metacat['nobj'] = Column([args.nobj], dtype='i4')
metacat['chunksize'] = Column([args.chunksize], dtype='i2')
metacat['nchunk'] = Column([nchunk], dtype='i2')
#metacat['RA'] = Column([ra_range],dtype='f8')
#metacat['DEC'] = Column([dec_range],dtype='f8')
if args.zoom is None:
metacat['zoom'] = Column([[0, 3600, 0, 3600]], dtype='i4')
else:
metacat['zoom'] = Column([args.zoom], dtype='i4')
metacat['rmag_range'] = Column([args.rmag_range], dtype='f4')
# Deal with the random seed
if args.seed is not None:
log.info('Random seed = {}'.format(args.seed))
metacat['seed'] = Column([args.seed], dtype='i4')
rand = np.random.RandomState(args.seed)
seeds = rand.random_integers(0, 2**18, nchunk)
if 'DECALS_SIM_DIR' in os.environ:
decals_sim_dir = os.getenv('DECALS_SIM_DIR')
else:
decals_sim_dir = '.'
if not os.path.exists(os.path.join(decals_sim_dir, brickname)):
os.makedirs(os.path.join(decals_sim_dir, brickname))
metafile = os.path.join(decals_sim_dir, brickname,
'metacat-' + brickname + '-' + lobjtype + '.fits')
log.info('Writing {}'.format(metafile))
if os.path.isfile(metafile):
os.remove(metafile)
metacat.write(metafile)
# Work in chunks.
for ichunk in range(nchunk):
log.info('Working on chunk {:02d}/{:02d}'.format(ichunk + 1, nchunk))
chunksuffix = '{:02d}'.format(ichunk)
# There's probably a smarter way to do this
nobjchunk = np.min((nobj, chunksize))
if (nchunk > 1) and (ichunk == (nchunk - 1)):
nobjchunk = np.max((nobjchunk, nobj % ((nchunk - 1) * chunksize)))
# Build and write out the simulated object catalog.
simcat = build_simcat(nobjchunk, brickname, brickwcs, metacat,
seeds[ichunk])
simcatfile = os.path.join(
decals_sim_dir, brickname, 'simcat-' + brickname + '-' + lobjtype +
'-' + chunksuffix + '.fits')
log.info('Writing {}'.format(simcatfile))
if os.path.isfile(simcatfile):
os.remove(simcatfile)
simcat.write(simcatfile)
# # Use Tractor to just process the blobs containing the simulated sources.
simdecals = SimDecals(metacat=metacat, simcat=simcat)
blobxy = zip(simcat['x'], simcat['y'])
run_brick(brickname,
decals=simdecals,
outdir=os.path.join(decals_sim_dir, brickname),
threads=args.threads,
zoom=args.zoom,
wise=False,
sdssInit=False,
forceAll=True,
writePickles=False,
do_calibs=True,
write_metrics=False,
pixPsf=True,
blobxy=blobxy,
early_coadds=False,
stages=['writecat'],
splinesky=True)
log.info('Cleaning up...')
shutil.move(
os.path.join(decals_sim_dir, brickname, 'tractor', brickname[:3],
'tractor-' + brickname + '.fits'),
os.path.join(
decals_sim_dir, brickname, 'tractor-' + brickname + '-' +
lobjtype + '-' + chunksuffix + '.fits'))
shutil.move(
os.path.join(decals_sim_dir, brickname, 'coadd', brickname[:3],
brickname, 'decals-' + brickname + '-image.jpg'),
os.path.join(
decals_sim_dir, brickname, 'qa-' + brickname + '-' + lobjtype +
'-image-' + chunksuffix + '.jpg'))
shutil.move(
os.path.join(decals_sim_dir, brickname, 'coadd', brickname[:3],
brickname, 'decals-' + brickname + '-resid.jpg'),
os.path.join(
decals_sim_dir, brickname, 'qa-' + brickname + '-' + lobjtype +
'-resid-' + chunksuffix + '.jpg'))
shutil.rmtree(os.path.join(decals_sim_dir, brickname, 'coadd'))
shutil.rmtree(os.path.join(decals_sim_dir, brickname, 'tractor'))
#shutil.rmtree(os.path.join(decals_sim_dir,brickname,'images'))
#shutil.rmtree(os.path.join(decals_sim_dir,brickname,'metrics'))
# Write a log file
log.info('All done!')
def main():
import optparse
parser = optparse.OptionParser()
parser.add_option('--zoom', '-z', type=int, action='append', default=[],
help='Add zoom level; default 13')
parser.add_option('--threads', type=int, default=1, help='Number of threads')
parser.add_option('--y0', type=int, default=0, help='Start row')
parser.add_option('--y1', type=int, default=None, help='End row (non-inclusive)')
parser.add_option('--x0', type=int, default=None)
parser.add_option('--x1', type=int, default=None)
parser.add_option('-x', type=int)
parser.add_option('-y', type=int)
parser.add_option('--mindec', type=float, default=None, help='Minimum Dec to run')
parser.add_option('--maxdec', type=float, default=None, help='Maximum Dec to run')
parser.add_option('--minra', type=float, default=0., help='Minimum RA to run')
parser.add_option('--maxra', type=float, default=360., help='Maximum RA to run')
parser.add_option('--near', action='store_true', help='Only run tiles near bricks')
parser.add_option('--near-ccds', action='store_true', help='Only run tiles near CCDs')
parser.add_option('--queue', action='store_true', default=False,
help='Print qdo commands')
parser.add_option('--all', action='store_true', help='Render all tiles')
parser.add_option('--ignore', action='store_true', help='Ignore cached tile files',
default=False)
parser.add_option('--top', action='store_true', help='Top levels of the pyramid')
parser.add_option('--kind', default='image')
parser.add_option('--scale', action='store_true', help='Scale images?')
parser.add_option('--coadd', action='store_true', help='Create SDSS coadd images?')
parser.add_option('--grass', action='store_true', help='progress plot')
opt,args = parser.parse_args()
if len(opt.zoom) == 0:
opt.zoom = [13]
mp = multiproc(opt.threads)
if opt.kind == 'sdss':
if opt.maxdec is None:
opt.maxdec = 90
if opt.mindec is None:
opt.mindec = -25
elif opt.kind in ['halpha', 'unwise-neo1']:
if opt.maxdec is None:
opt.maxdec = 90
if opt.mindec is None:
opt.mindec = -90
else:
if opt.maxdec is None:
opt.maxdec = 40
if opt.mindec is None:
opt.mindec = -20
if opt.top:
top_levels(mp, opt)
sys.exit(0)
from legacypipe.common import Decals
decals = Decals()
if opt.near:
if opt.kind == 'sdss':
B = fits_table(os.path.join(settings.DATA_DIR, 'bricks-sdssco.fits'))
else:
B = decals.get_bricks()
print len(B), 'bricks'
if opt.scale:
opt.near_ccds = True
if opt.near_ccds:
if opt.kind == 'sdss':
C = fits_table(os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'),
columns=['rerun','ra','dec', 'run', 'camcol', 'field', 'score'])
C.cut(C.rerun == '301')
C.cut(C.score >= 0.6)
#C.delete_column('rerun')
# SDSS field size
radius = 1.01 * np.hypot(10., 14.)/2. / 60.
ccdsize = radius
print len(C), 'SDSS fields'
else:
C = decals.get_ccds()
print len(C), 'CCDs'
ccdsize = 0.2
if opt.x is not None:
opt.x0 = opt.x
opt.x1 = opt.x + 1
if opt.y is not None:
opt.y0 = opt.y
opt.y1 = opt.y + 1
if opt.coadd and opt.kind == 'sdss':
from legacypipe.common import wcs_for_brick
from map.views import trymakedirs
B = decals.get_bricks()
print len(B), 'bricks'
B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec))
print len(B), 'in Dec range'
B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra))
print len(B), 'in RA range'
if opt.queue:
# ~ square-degree tiles
# RA slices
rr = np.arange(opt.minra , opt.maxra +1)
dd = np.arange(opt.mindec, opt.maxdec+1)
for rlo,rhi in zip(rr, rr[1:]):
for dlo,dhi in zip(dd, dd[1:]):
print 'time python render-tiles.py --kind sdss --coadd --minra %f --maxra %f --mindec %f --maxdec %f' % (rlo, rhi, dlo, dhi)
sys.exit(0)
if opt.grass:
basedir = settings.DATA_DIR
codir = os.path.join(basedir, 'coadd', 'sdssco')
rr,dd = [],[]
exist = []
for i,b in enumerate(B):
print 'Brick', b.brickname,
fn = os.path.join(codir, b.brickname[:3], 'sdssco-%s-%s.fits' % (b.brickname, 'r'))
print '-->', fn,
if not os.path.exists(fn):
print
continue
print 'found'
rr.append(b.ra)
dd.append(b.dec)
exist.append(i)
exist = np.array(exist)
B.cut(exist)
B.writeto('bricks-sdssco-exist.fits')
import pylab as plt
plt.clf()
plt.plot(rr, dd, 'k.')
plt.title('SDSS coadd tiles')
plt.savefig('sdss.png')
sys.exit(0)
basedir = settings.DATA_DIR
codir = os.path.join(basedir, 'coadd', 'sdssco')
for b in B:
print 'Brick', b.brickname
wcs = wcs_for_brick(b, W=2400, H=2400, pixscale=0.396)
bands = 'gri'
dirnm = os.path.join(codir, b.brickname[:3])
fns = [os.path.join(dirnm, 'sdssco-%s-%s.fits' % (b.brickname, band))
for band in bands]
hdr = fitsio.FITSHDR()
hdr['SURVEY'] = 'SDSS'
wcs.add_to_header(hdr)
if all([os.path.exists(fn) for fn in fns]):
print 'Already exist'
continue
ims = map_sdss(req, 1, 0, 0, 0, get_images=True, wcs=wcs, ignoreCached=True,
forcescale=0)
if ims is None:
print 'No overlap'
continue
trymakedirs(os.path.join(dirnm, 'xxx'))
for fn,band,im in zip(fns,bands, ims):
fitsio.write(fn, im, header=hdr, clobber=True)
print 'Wrote', fn
# Also write scaled versions
dirnm = os.path.join(basedir, 'scaled', 'sdssco')
scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(brickname).3s', 'sdssco-%(brickname)s-%(band)s.fits')
for im,band in zip(ims,bands):
scalekwargs = dict(band=band, brick=b.brickid, brickname=b.brickname)
imwcs = wcs
for scale in range(1, 7):
print 'Writing scale level', scale
im,imwcs,sfn = get_scaled(scalepat, scalekwargs, scale, None,
wcs=imwcs, img=im, return_data=True)
sys.exit(0)
if opt.scale:
if opt.kind == 'sdss':
C.cut((C.dec >= opt.mindec) * (C.dec < opt.maxdec))
print len(C), 'in Dec range'
C.cut((C.ra >= opt.minra) * (C.ra < opt.maxra))
print len(C), 'in RA range'
from astrometry.sdss import AsTransWrapper, DR9
sdss = DR9(basedir=settings.SDSS_DIR)
sdss.saveUnzippedFiles(settings.SDSS_DIR)
sdss.setFitsioReadBZ2()
if settings.SDSS_PHOTOOBJS:
sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS,
resolve=settings.SDSS_RESOLVE)
basedir = settings.DATA_DIR
scaledir = 'sdss'
dirnm = os.path.join(basedir, 'scaled', scaledir)
for im in C:
from map.views import _read_sip_wcs
#if im.rerun != '301':
# continue
for band in 'gri':
scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i', 'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits')
tmpsuff = '.tmp%08i' % np.random.randint(100000000)
basefn = sdss.retrieve('frame', im.run, im.camcol, field=im.field,
band=band, rerun=im.rerun, tempsuffix=tmpsuff)
fnargs = dict(band=band, rerun=im.rerun, run=im.run,
camcol=im.camcol, field=im.field)
scaled = 1
fn = get_scaled(scalepat, fnargs, scaled, basefn,
read_base_wcs=read_astrans, read_wcs=_read_sip_wcs)
print 'get_scaled:', fn
return 0
else:
assert(False)
for zoom in opt.zoom:
N = 2**zoom
if opt.y1 is None:
y1 = N
else:
y1 = opt.y1
if opt.x0 is None:
opt.x0 = 0
x1 = opt.x1
if x1 is None:
x1 = N
# Find grid of Ra,Dec tile centers and select the ones near DECaLS bricks.
rr,dd = [],[]
yy = np.arange(opt.y0, y1)
xx = np.arange(opt.x0, x1)
if opt.grass:
import pylab as plt
tileexists = np.zeros((len(yy),len(xx)), bool)
basedir = settings.DATA_DIR
ver = tileversions[opt.kind][-1]
tiledir = os.path.join(basedir, 'tiles', opt.kind, '%i'%ver, '%i'%zoom)
for dirpath,dirnames,filenames in os.walk(tiledir):
# change walk order
dirnames.sort()
if len(filenames) == 0:
continue
print 'Dirpath', dirpath
#print 'Dirnames', dirnames
#print 'Filenames', filenames
# check for symlinks
if False:
fns = []
for fn in filenames:
fullfn = os.path.join(tiledir, dirpath, fn)
if os.path.isfile(fullfn) and not os.path.islink(fullfn):
fns.append(fn)
print len(fns), 'of', len(filenames), 'are files (not symlinks)'
filenames = fns
x = os.path.basename(dirpath)
x = int(x)
#print 'x', x
yy = [int(fn.replace('.jpg','')) for fn in filenames]
#print 'yy', yy
print len(yy), 'tiles'
for y in yy:
tileexists[y - opt.y0, x - opt.x0] = True
plt.clf()
plt.imshow(tileexists, interpolation='nearest', origin='upper',
vmin=0, vmax=1, cmap='gray')
fn = 'exist-%s-z%02i' % (opt.kind, zoom)
plt.savefig(fn+'.png')
fitsio.write(fn+'.fits', tileexists, clobber=True)
print 'Wrote', fn+'.png and', fn+'.fits'
continue
if not opt.all:
for y in yy:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, y)
r,d = wcs.get_center()
dd.append(d)
for x in xx:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, x, 0)
r,d = wcs.get_center()
rr.append(r)
dd = np.array(dd)
rr = np.array(rr)
if len(dd) > 1:
tilesize = max(np.abs(np.diff(dd)))
print 'Tile size:', tilesize
else:
if opt.near_ccds or opt.near:
try:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0+1)
r2,d2 = wcs.get_center()
except:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0-1)
r2,d2 = wcs.get_center()
tilesize = np.abs(dd[0] - d2)
print 'Tile size:', tilesize
else:
tilesize = 180.
I = np.flatnonzero((dd >= opt.mindec) * (dd <= opt.maxdec))
print 'Keeping', len(I), 'Dec points between', opt.mindec, 'and', opt.maxdec
dd = dd[I]
yy = yy[I]
I = np.flatnonzero((rr >= opt.minra) * (rr <= opt.maxra))
print 'Keeping', len(I), 'RA points between', opt.minra, 'and', opt.maxra
rr = rr[I]
xx = xx[I]
print len(rr), 'RA points x', len(dd), 'Dec points'
print 'x tile range:', xx.min(), xx.max(), 'y tile range:', yy.min(), yy.max()
for iy,y in enumerate(yy):
print
print 'Y row', y
if opt.near:
d = dd[iy]
I,J,dist = match_radec(rr, d+np.zeros_like(rr), B.ra, B.dec, 0.25 + tilesize, nearest=True)
if len(I) == 0:
print 'No matches to bricks'
continue
keep = np.zeros(len(rr), bool)
keep[I] = True
print 'Keeping', sum(keep), 'tiles in row', y, 'Dec', d
x = xx[keep]
elif opt.near_ccds:
d = dd[iy]
print 'RA range of tiles:', rr.min(), rr.max()
print 'Dec of tile row:', d
I,J,dist = match_radec(rr, d+np.zeros_like(rr), C.ra, C.dec, ccdsize + tilesize, nearest=True)
if len(I) == 0:
print 'No matches to CCDs'
continue
keep = np.zeros(len(rr), bool)
keep[I] = True
print 'Keeping', sum(keep), 'tiles in row', y, 'Dec', d
x = xx[keep]
else:
x = xx
if opt.queue:
cmd = 'python -u render-tiles.py --zoom %i --y0 %i --y1 %i --kind %s' % (zoom, y, y+1, opt.kind)
if opt.near_ccds:
cmd += ' --near-ccds'
if opt.all:
cmd += ' --all'
if opt.ignore:
cmd += ' --ignore'
print cmd
continue
# if opt.grass:
# for xi in x:
# basedir = settings.DATA_DIR
# ver = tileversions[opt.kind][-1]
# tilefn = os.path.join(basedir, 'tiles', opt.kind,
# '%i/%i/%i/%i.jpg' % (ver, zoom, xi, y))
# print 'Checking for', tilefn
# if os.path.exists(tilefn):
# print 'EXISTS'
# tileexists[yi-opt.y0, xi-opt.x0]
# continue
args = []
for xi in x:
args.append((opt.kind,zoom,xi,y, opt.ignore))
print 'Rendering', len(args), 'tiles in row y =', y
mp.map(_bounce_one_tile, args, chunksize=min(100, max(1, len(args)/opt.threads)))
print 'Rendered', len(args), 'tiles'
