def queue():
if False:
survey = LegacySurveyData()
ccds = survey.get_ccds()
bricks = survey.get_bricks()
print(len(bricks), 'bricks')
print(len(ccds), 'CCDs')
bricks.cut((bricks.dec >= -30) * (bricks.dec <= 30))
print(len(bricks), 'in Dec [-30, +30]')
I = survey.photometric_ccds(ccds)
ccds.cut(I)
print(len(ccds), 'pass photometric cut')
I, J, d = match_radec(bricks.ra,
bricks.dec,
ccds.ra,
ccds.dec,
0.5,
nearest=True)
print(len(I), 'bricks with CCDs nearby')
bricks.cut(I)
bricknames = bricks.brickname
else:
# DR7: use Martin's list of bricks w/ CCD coverage
f = open('nccds.dat')
bricknames = []
for line in f.readlines():
words = line.strip().split(' ')
brick = words[0]
nccd = int(words[1])
if nccd > 0:
bricknames.append(brick)
# qdo
bb = bricknames
while len(bb):
print(' '.join(bb[:100]))
bb = bb[100:]
return
mp = multiproc(16)
N = len(bricks)
args = [(brick, i, N, plots, {}) for i, brick in enumerate(bricks)]
mp.map(run_one_brick, args)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build-sample',
action='store_true',
help='Build the sample.')
parser.add_argument('--jpg-cutouts',
action='store_true',
help='Get jpg cutouts from the viewer.')
parser.add_argument('--ccd-cutouts',
action='store_true',
help='Get CCD cutouts of each galaxy.')
parser.add_argument('--runbrick',
action='store_true',
help='Run the pipeline.')
parser.add_argument('--build-webpage',
action='store_true',
help='(Re)build the web content.')
args = parser.parse_args()
# Top-level directory
key = 'LEGACY_SURVEY_LARGE_GALAXIES'
if key not in os.environ:
print('Required ${} environment variable not set'.format(key))
return 0
largedir = os.getenv(key)
samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
# --------------------------------------------------
# Build the sample of large galaxies based on the available imaging.
if args.build_sample:
# Read the parent catalog.
cat = read_rc3()
# Create a simple WCS object for each object and find all the CCDs
# touching that WCS footprint.
survey = LegacySurveyData(version='dr2') # hack!
allccds = survey.get_ccds()
keep = np.concatenate((survey.apply_blacklist(allccds),
survey.photometric_ccds(allccds)))
allccds.cut(keep)
ccdlist = []
outcat = []
for gal in cat:
galwcs = _simplewcs(gal)
ccds1 = allccds[ccds_touching_wcs(galwcs, allccds)]
ccds1 = ccds1[_uniqccds(ccds1)]
if len(
ccds1
) > 0 and 'g' in ccds1.filter and 'r' in ccds1.filter and 'z' in ccds1.filter:
print('Found {} CCDs for {}, D(25)={:.4f}'.format(
len(ccds1), gal['GALAXY'], gal['RADIUS']))
ccdsfile = os.path.join(
largedir, 'ccds',
'{}-ccds.fits'.format(gal['GALAXY'].strip().lower()))
print(' Writing {}'.format(ccdsfile))
if os.path.isfile(ccdsfile):
os.remove(ccdsfile)
ccds1.writeto(ccdsfile)
ccdlist.append(ccds1)
if len(outcat) == 0:
outcat = gal
else:
outcat = vstack((outcat, gal))
#if gal['GALAXY'] == 'MCG5-19-36':
# pdb.set_trace()
# Write out the final catalog.
samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
if os.path.isfile(samplefile):
os.remove(samplefile)
print('Writing {}'.format(samplefile))
outcat.write(samplefile)
print(outcat)
# Do we need to transfer any of the data to nyx?
_getfiles(merge_tables(ccdlist))
# --------------------------------------------------
# Get data, model, and residual cutouts from the legacysurvey viewer. Also
# get thumbnails that are lower resolution.
if args.jpg_cutouts:
thumbsize = 100
sample = fits.getdata(samplefile, 1)
for gal in sample:
size = np.ceil(10 * gal['RADIUS'] / PIXSCALE)
thumbpixscale = PIXSCALE * size / thumbsize
#imageurl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
# '&pixscale={:.3f}&size={:g}'.format(PIXSCALE, size)
#imagejpg = os.path.join(largedir, 'cutouts', gal['GALAXY'].strip().lower()+'-image.jpg')
#if os.path.isfile(imagejpg):
# os.remove(imagejpg)
#os.system('wget --continue -O {:s} "{:s}"' .format(imagejpg, imageurl))
thumburl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
'&pixscale={:.3f}&size={:g}'.format(thumbpixscale, thumbsize)
thumbjpg = os.path.join(
largedir, 'cutouts',
gal['GALAXY'].strip().lower() + '-image-thumb.jpg')
if os.path.isfile(thumbjpg):
os.remove(thumbjpg)
os.system('wget --continue -O {:s} "{:s}"'.format(
thumbjpg, thumburl))
# --------------------------------------------------
# (Re)build the webpage.
if args.build_webpage:
# index.html
html = open(os.path.join(largedir, 'index.html'), 'w')
html.write('<html><body>\n')
html.write('<h1>Sample of Large Galaxies</h1>\n')
html.write('<table border="2" width="30%">\n')
html.write('<tbody>\n')
sample = fits.getdata(samplefile, 1)
for gal in sample:
# Add coordinates and sizes here.
galaxy = gal['GALAXY'].strip().lower()
html.write('<tr>\n')
html.write('<td><a href="html/{}.html">{}</a></td>\n'.format(
galaxy, galaxy.upper()))
html.write(
'<td><a href="http://legacysurvey.org/viewer/?ra={:.6f}&dec={:.6f}" target="_blank"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'
.format(gal['RA'], gal['DEC'], galaxy, galaxy.upper()))
# html.write('<td><a href="html/{}.html"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'.format(galaxy, galaxy, galaxy.upper()))
html.write('</tr>\n')
html.write('</tbody>\n')
html.write('</table>\n')
html.write('</body></html>\n')
html.close()
sys.exit(1)
# individual galaxy pages
for gal in sample[:3]:
galaxy = gal['GALAXY'].strip().lower()
html = open(os.path.join(largedir, 'html/{}.html'.format(galaxy)),
'w')
html.write('<html><body>\n')
html.write(
'<a href=../cutouts/{}.jpg><img src=../cutouts/{}-image.jpg alt={} /></a>\n'
.format(galaxy, galaxy, galaxy, galaxy.upper()))
html.write('</body></html>\n')
html.close()
# --------------------------------------------------
# Get cutouts of all the CCDs for each galaxy.
if args.ccd_cutouts:
sample = fits.getdata(samplefile, 1)
for gal in sample[1:2]:
galaxy = gal['GALAXY'].strip().lower()
ccdsfile = os.path.join(largedir, 'ccds',
'{}-ccds.fits'.format(galaxy))
ccds = fits.getdata(ccdsfile)
pdb.set_trace()
# --------------------------------------------------
# Run the pipeline.
if args.runbrick:
sample = fits.getdata(samplefile, 1)
for gal in sample[1:2]:
galaxy = gal['GALAXY'].strip().lower()
diam = 10 * np.ceil(gal['RADIUS'] / PIXSCALE).astype(
'int16') # [pixels]
# Note: zoom is relative to the center of an imaginary brick with
# dimensions (0, 3600, 0, 3600).
survey = LegacySurveyData(version='dr2', output_dir=largedir)
run_brick(None,
survey,
radec=(gal['RA'], gal['DEC']),
blobxy=zip([diam / 2], [diam / 2]),
threads=1,
zoom=(1800 - diam / 2, 1800 + diam / 2, 1800 - diam / 2,
1800 + diam / 2),
wise=False,
forceAll=True,
writePickles=False,
do_calibs=False,
write_metrics=False,
pixPsf=True,
splinesky=True,
early_coadds=True,
stages=['writecat'],
ceres=False)
pdb.set_trace()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build-sample', action='store_true', help='Build the sample.')
parser.add_argument('--jpg-cutouts', action='store_true', help='Get jpg cutouts from the viewer.')
parser.add_argument('--ccd-cutouts', action='store_true', help='Get CCD cutouts of each galaxy.')
parser.add_argument('--runbrick', action='store_true', help='Run the pipeline.')
parser.add_argument('--build-webpage', action='store_true', help='(Re)build the web content.')
args = parser.parse_args()
# Top-level directory
key = 'LEGACY_SURVEY_LARGE_GALAXIES'
if key not in os.environ:
print('Required ${} environment variable not set'.format(key))
return 0
largedir = os.getenv(key)
samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
# --------------------------------------------------
# Build the sample of large galaxies based on the available imaging.
if args.build_sample:
# Read the parent catalog.
cat = read_rc3()
# Create a simple WCS object for each object and find all the CCDs
# touching that WCS footprint.
survey = LegacySurveyData(version='dr2') # hack!
allccds = survey.get_ccds()
keep = np.concatenate((survey.apply_blacklist(allccds),
survey.photometric_ccds(allccds)))
allccds.cut(keep)
ccdlist = []
outcat = []
for gal in cat:
galwcs = _simplewcs(gal)
ccds1 = allccds[ccds_touching_wcs(galwcs, allccds)]
ccds1 = ccds1[_uniqccds(ccds1)]
if len(ccds1) > 0 and 'g' in ccds1.filter and 'r' in ccds1.filter and 'z' in ccds1.filter:
print('Found {} CCDs for {}, D(25)={:.4f}'.format(
len(ccds1), gal['GALAXY'], gal['RADIUS']))
ccdsfile = os.path.join(largedir, 'ccds', '{}-ccds.fits'.format(gal['GALAXY'].strip().lower()))
print(' Writing {}'.format(ccdsfile))
if os.path.isfile(ccdsfile):
os.remove(ccdsfile)
ccds1.writeto(ccdsfile)
ccdlist.append(ccds1)
if len(outcat) == 0:
outcat = gal
else:
outcat = vstack((outcat, gal))
#if gal['GALAXY'] == 'MCG5-19-36':
# pdb.set_trace()
# Write out the final catalog.
samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
if os.path.isfile(samplefile):
os.remove(samplefile)
print('Writing {}'.format(samplefile))
outcat.write(samplefile)
print(outcat)
# Do we need to transfer any of the data to nyx?
_getfiles(merge_tables(ccdlist))
# --------------------------------------------------
# Get data, model, and residual cutouts from the legacysurvey viewer. Also
# get thumbnails that are lower resolution.
if args.jpg_cutouts:
thumbsize = 100
sample = fits.getdata(samplefile, 1)
for gal in sample:
size = np.ceil(10*gal['RADIUS']/PIXSCALE)
thumbpixscale = PIXSCALE*size/thumbsize
#imageurl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
# '&pixscale={:.3f}&size={:g}'.format(PIXSCALE, size)
#imagejpg = os.path.join(largedir, 'cutouts', gal['GALAXY'].strip().lower()+'-image.jpg')
#if os.path.isfile(imagejpg):
# os.remove(imagejpg)
#os.system('wget --continue -O {:s} "{:s}"' .format(imagejpg, imageurl))
thumburl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
'&pixscale={:.3f}&size={:g}'.format(thumbpixscale, thumbsize)
thumbjpg = os.path.join(largedir, 'cutouts', gal['GALAXY'].strip().lower()+'-image-thumb.jpg')
if os.path.isfile(thumbjpg):
os.remove(thumbjpg)
os.system('wget --continue -O {:s} "{:s}"' .format(thumbjpg, thumburl))
# --------------------------------------------------
# (Re)build the webpage.
if args.build_webpage:
# index.html
html = open(os.path.join(largedir, 'index.html'), 'w')
html.write('<html><body>\n')
html.write('<h1>Sample of Large Galaxies</h1>\n')
html.write('<table border="2" width="30%">\n')
html.write('<tbody>\n')
sample = fits.getdata(samplefile, 1)
for gal in sample:
# Add coordinates and sizes here.
galaxy = gal['GALAXY'].strip().lower()
html.write('<tr>\n')
html.write('<td><a href="html/{}.html">{}</a></td>\n'.format(galaxy, galaxy.upper()))
html.write('<td><a href="http://legacysurvey.org/viewer/?ra={:.6f}&dec={:.6f}" target="_blank"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'.format(gal['RA'], gal['DEC'], galaxy, galaxy.upper()))
# html.write('<td><a href="html/{}.html"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'.format(galaxy, galaxy, galaxy.upper()))
html.write('</tr>\n')
html.write('</tbody>\n')
html.write('</table>\n')
html.write('</body></html>\n')
html.close()
sys.exit(1)
# individual galaxy pages
for gal in sample[:3]:
galaxy = gal['GALAXY'].strip().lower()
html = open(os.path.join(largedir, 'html/{}.html'.format(galaxy)), 'w')
html.write('<html><body>\n')
html.write('<a href=../cutouts/{}.jpg><img src=../cutouts/{}-image.jpg alt={} /></a>\n'.format(galaxy, galaxy, galaxy, galaxy.upper()))
html.write('</body></html>\n')
html.close()
# --------------------------------------------------
# Get cutouts of all the CCDs for each galaxy.
if args.ccd_cutouts:
sample = fits.getdata(samplefile, 1)
for gal in sample[1:2]:
galaxy = gal['GALAXY'].strip().lower()
ccdsfile = os.path.join(largedir, 'ccds', '{}-ccds.fits'.format(galaxy))
ccds = fits.getdata(ccdsfile)
pdb.set_trace()
# --------------------------------------------------
# Run the pipeline.
if args.runbrick:
sample = fits.getdata(samplefile, 1)
for gal in sample[1:2]:
galaxy = gal['GALAXY'].strip().lower()
diam = 10*np.ceil(gal['RADIUS']/PIXSCALE).astype('int16') # [pixels]
# Note: zoom is relative to the center of an imaginary brick with
# dimensions (0, 3600, 0, 3600).
survey = LegacySurveyData(version='dr2', output_dir=largedir)
run_brick(None, survey, radec=(gal['RA'], gal['DEC']), blobxy=zip([diam/2], [diam/2]),
threads=1, zoom=(1800-diam/2, 1800+diam/2, 1800-diam/2, 1800+diam/2),
wise=False, forceAll=True, writePickles=False, do_calibs=False,
write_metrics=False, pixPsf=True, splinesky=True,
early_coadds=True, stages=['writecat'], ceres=False)
pdb.set_trace()
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).")
parser.add_argument('--calibs', action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper', type=int, default=None,
help='Batch N calibs per line')
parser.add_argument('--forced', action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb', action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--touching', action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--near', action='store_true',
help='Quick cut to only CCDs near selected bricks')
parser.add_argument('--check', action='store_true',
help='Check which calibrations actually need to run.')
parser.add_argument('--check-coadd', action='store_true',
help='Check which caoadds actually need to run.')
parser.add_argument('--out', help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command', action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--opt', help='With --command, extra options to add')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric or blacklist cuts')
parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table')
parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table')
parser.add_argument('--delete-sky', action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--delete-pvastrom', action='store_true',
help='Delete any existing PV WCS calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
parser.add_argument('--brickq', type=int, default=None,
help='Queue only bricks with the given "brickq" value [0 to 3]')
parser.add_argument('--brickq-deps', action='store_true', default=False,
help='Queue bricks directly using qdo API, setting brickq dependencies')
parser.add_argument('--queue', default='bricks',
help='With --brickq-deps, the QDO queue name to use')
opt = parser.parse_args()
survey = LegacySurveyData()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = survey.get_bricks()
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
if opt.ignore_cuts == False:
I = survey.photometric_ccds(T)
print(len(I), 'CCDs are photometric')
T.cut(I)
I = survey.apply_blacklist(T)
print(len(I), 'CCDs are not blacklisted')
T.cut(I)
print(len(T), 'CCDs remain')
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo,dhi = -25, 40
rlo,rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo,dhi = -90,90
rlo,rhi = 0, 360
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo,rhi = 240,245
dlo,dhi = 5, 12
elif opt.region == 'edrplus':
rlo,rhi = 235,248
dlo,dhi = 5, 15
elif opt.region == 'edr-south':
rlo,rhi = 240,245
dlo,dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo,rhi = 149.75, 150.75
dlo,dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo,rhi = 240,250
dlo,dhi = 10,15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo,rhi = 320., 360.
dlo,dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo,rhi = 240,360
dlo,dhi = 20,40
elif opt.region == 'north':
rlo,rhi = 120,240
dlo,dhi = 20,40
elif opt.region == 'northeast':
rlo,rhi = 0,120
dlo,dhi = 20,40
elif opt.region == 'southwest':
rlo,rhi = 240,360
dlo,dhi = -20,0
elif opt.region == 'south':
rlo,rhi = 120,240
dlo,dhi = -20,0
elif opt.region == 'southeast':
rlo,rhi = 0,120
dlo,dhi = -20,0
elif opt.region == 'southsoutheast':
rlo,rhi = 0,120
dlo,dhi = -20,-10
elif opt.region == 'midwest':
rlo,rhi = 240,360
dlo,dhi = 0,20
elif opt.region == 'middle':
rlo,rhi = 120,240
dlo,dhi = 0,20
elif opt.region == 'mideast':
rlo,rhi = 0,120
dlo,dhi = 0,20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo,rhi = 250,260
dlo,dhi = 30,35
elif opt.region == 'deep2f3':
rlo,rhi = 351.25, 353.75
dlo,dhi = 0, 0.5
elif opt.region == 'virgo':
rlo,rhi = 185,190
dlo,dhi = 10, 15
elif opt.region == 'virgo2':
rlo,rhi = 182,192
dlo,dhi = 8, 18
elif opt.region == 'lsb':
rlo,rhi = 147.2, 147.8
dlo,dhi = -0.4, 0.4
elif opt.region == 'eboss-elg':
# RA -45 to +45
# Dec -5 to +7
rlo,rhi = 315., 45.
dlo,dhi = -5., 7.
elif opt.region == 'eboss-ngc':
# NGC ELGs
# RA 115 to 175
# Dec 15 to 30
rlo,rhi = 115., 175.
dlo,dhi = 15., 30.
elif opt.region == 'mzls':
dlo,dhi = 30., 90.
elif opt.region == 'dr4-bootes':
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
#dlo,dhi = 34., 35.
#rlo,rhi = 209.5, 210.5
dlo,dhi = 33., 36.
rlo,rhi = 216.5, 219.5
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
log(len(B), 'bricks in range')
for name in B.get('brickname'):
print(name)
B.writeto('bricks-cut.fits')
I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs')
plt.clf()
plt.plot(B.ra, B.dec, 'b.')
plt.title('DR3 bricks')
plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
plt.savefig('bricks.png')
if opt.brickq is not None:
B.cut(B.brickq == opt.brickq)
log('Cut to', len(B), 'with brickq =', opt.brickq)
if opt.touching:
keep = np.zeros(len(T), bool)
for j in J:
keep[j] = True
T.cut(keep)
log('Cut to', len(T), 'CCDs near bricks')
# Aside -- how many near DR1=1 CCDs?
if False:
T2 = D.get_ccds()
log(len(T2), 'CCDs')
T2.cut(T2.dr1 == 1)
log(len(T2), 'CCDs marked DR1=1')
log(len(B), 'bricks in range')
I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B2 = B[keep]
log('Total of', len(B2), 'bricks near CCDs with DR1=1')
for band in 'grz':
Tb = T2[T2.filter == band]
log(len(Tb), 'in filter', band)
I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, survey.bricksize)
good = np.zeros(len(B2), np.uint8)
for i in I:
good[i] = 1
B2.set('has_' + band, good)
B2.writeto('survey-bricks-in-dr1.fits')
sys.exit(0)
# sort by dec decreasing
#B.cut(np.argsort(-B.dec))
# RA increasing
B.cut(np.argsort(B.ra))
for b in B:
if opt.check:
fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
if opt.check_coadd:
fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
print(b.brickname)
if opt.save_to_fits:
assert(opt.touching)
# Write cut tables to file
for tab,typ in zip([B,T],['bricks','ccds']):
fn='%s-%s-cut.fits' % (typ,opt.name)
if os.path.exists(fn):
os.remove(fn)
tab.writeto(fn)
print('Wrote %s' % fn)
# Write text files listing ccd and filename names
nm1,nm2= 'ccds-%s.txt'% opt.name,'filenames-%s.txt' % opt.name
if os.path.exists(nm1):
os.remove(nm1)
if os.path.exists(nm2):
os.remove(nm2)
f1,f2=open(nm1,'w'),open(nm2,'w')
fns= list(set(T.get('image_filename')))
for fn in fns:
f2.write('%s\n' % fn.strip())
for ti in T:
f1.write('%s\n' % ti.get('image_filename').strip())
f1.close()
f2.close()
print('Wrote *-names.txt')
if opt.brickq_deps:
import qdo
from legacypipe.survey import on_bricks_dependencies
#... find Queue...
q = qdo.connect(opt.queue, create_ok=True)
print('Connected to QDO queue', opt.queue, q)
brick_to_task = dict()
I = survey.photometric_ccds(T)
print(len(I), 'CCDs are photometric')
T.cut(I)
I = survey.apply_blacklist(T)
print(len(I), 'CCDs are not blacklisted')
T.cut(I)
print(len(T), 'CCDs remaining')
T.wra = T.ra + (T.ra > 180) * -360
wra = rlo - 360
plt.clf()
plt.plot(T.wra, T.dec, 'b.')
ax = [wra, rhi, dlo, dhi]
plt.axis(ax)
plt.title('CCDs')
plt.savefig('q-ccds.png')
B.wra = B.ra + (B.ra > 180) * -360
# this slight overestimate (for DECam images) is fine
radius = 0.3
Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius,
indexlist=True)
ikeep = []
for ib,(b,Iccd) in enumerate(zip(B, Iccds)):
if Iccd is None or len(Iccd) == 0:
print('No matched CCDs to brick', b.brickname)
continue
wcs = wcs_for_brick(b)
cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
print(len(cI), 'CCDs touching brick', b.brickname)
if len(cI) == 0:
continue
ikeep.append(ib)
B.cut(np.array(ikeep))
print('Cut to', len(B), 'bricks touched by CCDs')
for brickq in range(4):
I = np.flatnonzero(B.brickq == brickq)
print(len(I), 'bricks with brickq =', brickq)
J = np.flatnonzero(B.brickq < brickq)
preB = B[J]
reqs = []
if brickq > 0:
for b in B[I]:
# find brick dependencies
brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
# convert to task ids
taskdeps = [brick_to_task.get(b.brickname,None) for b in brickdeps]
# If we dropped a dependency brick from a previous brickq because
# of no overlapping CCDs, it won't appear in the brick_to_task map.
taskdeps = [t for t in taskdeps if t is not None]
reqs.append(taskdeps)
plt.clf()
plt.plot(B.wra, B.dec, '.', color='0.5')
plt.plot(B.wra[I], B.dec[I], 'b.')
plt.axis(ax)
plt.title('Bricks: brickq=%i' % brickq)
plt.savefig('q-bricks-%i.png' % brickq)
# submit to qdo queue
print('Queuing', len(B[I]), 'bricks')
if brickq == 0:
reqs = None
else:
assert(len(I) == len(reqs))
taskids = q.add_multiple(B.brickname[I], requires=reqs)
assert(len(taskids) == len(I))
print('Queued', len(taskids), 'bricks')
brick_to_task.update(dict(zip(B.brickname[I], taskids)))
if not (opt.calibs or opt.forced or opt.lsb):
sys.exit(0)
bands = 'grz'
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
if opt.touching:
allI = set()
for b in B:
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
if len(I) == 0:
continue
allI.update(I)
allI = list(allI)
allI.sort()
elif opt.near:
# Roughly brick radius + DECam image radius
radius = 0.35
allI,nil,nil = match_radec(T.ra, T.dec, B.ra, B.dec, radius, nearest=True)
else:
allI = np.arange(len(T))
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
for j,i in enumerate(allI):
expstr = '%08i' % T.expnum[i]
outfn = os.path.join('forced', expstr[:5], expstr,
'decam-%s-%s-forced.fits' %
(expstr, T.ccdname[i]))
imgfn = os.path.join(survey.survey_dir, 'images',
T.image_filename[i].strip())
if (not os.path.exists(imgfn) and
imgfn.endswith('.fz') and
os.path.exists(imgfn[:-3])):
imgfn = imgfn[:-3]
#f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
# (imgfn, T.image_hdu[i], outfn))
f.write('python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n' %
(T.expnum[i], T.ccdname[i], outfn))
f.close()
log('Wrote', opt.out)
sys.exit(0)
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
for j,i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
sys.exit(0)
log('Writing calibs to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if cmd is None:
cmd = ''
f.write(cmd + ' '.join(batch) + '\n')
cmd = None
if opt.command:
cmd = 'python legacypipe/run-calib.py '
if opt.opt is not None:
cmd += opt.opt + ' '
for j,i in enumerate(allI):
if opt.delete_sky or opt.delete_pvastrom:
log(j+1, 'of', len(allI))
im = survey.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
log(' deleting:', im.pvwcsfn)
os.unlink(im.pvwcsfn)
if opt.check:
log(j+1, 'of', len(allI))
im = survey.get_image_object(T[i])
if not im.run_calibs(im, just_check=True):
log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done')
continue
if opt.command:
s = '%i-%s' % (T.expnum[i], T.ccdname[i])
prefix = 'python legacypipe/run-calib.py ' + opt.opt
#('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
# (T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
prefix = ''
if j < 10:
print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i],
'filename', T.image_filename[i])
if not opt.nper:
f.write(prefix + s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, cmd)
batch = []
if opt.check:
f.flush()
if len(batch):
write_batch(f, batch, cmd)
f.close()
log('Wrote', opt.out)
return 0
def main(outfn='ccds-annotated.fits', ccds=None):
survey = LegacySurveyData(ccds=ccds)
if ccds is None:
ccds = survey.get_ccds()
# File from the "observing" svn repo:
# https://desi.lbl.gov/svn/decam/code/observing/trunk
tiles = fits_table('decam-tiles_obstatus.fits')
I = survey.photometric_ccds(ccds)
ccds.photometric = np.zeros(len(ccds), bool)
ccds.photometric[I] = True
I = survey.apply_blacklist(ccds)
ccds.blacklist_ok = np.zeros(len(ccds), bool)
ccds.blacklist_ok[I] = True
ccds.good_region = np.empty((len(ccds), 4), np.int16)
ccds.good_region[:,:] = -1
ccds.ra0 = np.zeros(len(ccds), np.float64)
ccds.dec0 = np.zeros(len(ccds), np.float64)
ccds.ra1 = np.zeros(len(ccds), np.float64)
ccds.dec1 = np.zeros(len(ccds), np.float64)
ccds.ra2 = np.zeros(len(ccds), np.float64)
ccds.dec2 = np.zeros(len(ccds), np.float64)
ccds.ra3 = np.zeros(len(ccds), np.float64)
ccds.dec3 = np.zeros(len(ccds), np.float64)
ccds.dra = np.zeros(len(ccds), np.float32)
ccds.ddec = np.zeros(len(ccds), np.float32)
ccds.ra_center = np.zeros(len(ccds), np.float64)
ccds.dec_center = np.zeros(len(ccds), np.float64)
ccds.sig1 = np.zeros(len(ccds), np.float32)
ccds.meansky = np.zeros(len(ccds), np.float32)
ccds.stdsky = np.zeros(len(ccds), np.float32)
ccds.maxsky = np.zeros(len(ccds), np.float32)
ccds.minsky = np.zeros(len(ccds), np.float32)
ccds.pixscale_mean = np.zeros(len(ccds), np.float32)
ccds.pixscale_std = np.zeros(len(ccds), np.float32)
ccds.pixscale_max = np.zeros(len(ccds), np.float32)
ccds.pixscale_min = np.zeros(len(ccds), np.float32)
ccds.psfnorm_mean = np.zeros(len(ccds), np.float32)
ccds.psfnorm_std = np.zeros(len(ccds), np.float32)
ccds.galnorm_mean = np.zeros(len(ccds), np.float32)
ccds.galnorm_std = np.zeros(len(ccds), np.float32)
gaussgalnorm = np.zeros(len(ccds), np.float32)
# 2nd moments
ccds.psf_mx2 = np.zeros(len(ccds), np.float32)
ccds.psf_my2 = np.zeros(len(ccds), np.float32)
ccds.psf_mxy = np.zeros(len(ccds), np.float32)
#
ccds.psf_a = np.zeros(len(ccds), np.float32)
ccds.psf_b = np.zeros(len(ccds), np.float32)
ccds.psf_theta = np.zeros(len(ccds), np.float32)
ccds.psf_ell = np.zeros(len(ccds), np.float32)
ccds.humidity = np.zeros(len(ccds), np.float32)
ccds.outtemp = np.zeros(len(ccds), np.float32)
ccds.tileid = np.zeros(len(ccds), np.int32)
ccds.tilepass = np.zeros(len(ccds), np.uint8)
ccds.tileebv = np.zeros(len(ccds), np.float32)
plvers = []
for iccd,ccd in enumerate(ccds):
im = survey.get_image_object(ccd)
print('Reading CCD %i of %i:' % (iccd+1, len(ccds)), im)
X = im.get_good_image_subregion()
for i,x in enumerate(X):
if x is not None:
ccds.good_region[iccd,i] = x
W,H = ccd.width, ccd.height
psf = None
wcs = None
sky = None
try:
tim = im.get_tractor_image(pixPsf=True, splinesky=True, subsky=False,
pixels=False, dq=False, invvar=False)
except:
import traceback
traceback.print_exc()
plvers.append('')
continue
if tim is None:
plvers.append('')
continue
psf = tim.psf
wcs = tim.wcs.wcs
sky = tim.sky
hdr = tim.primhdr
# print('Got PSF', psf)
# print('Got sky', type(sky))
# print('Got WCS', wcs)
ccds.humidity[iccd] = hdr.get('HUMIDITY')
ccds.outtemp[iccd] = hdr.get('OUTTEMP')
ccds.sig1[iccd] = tim.sig1
plvers.append(tim.plver)
# parse 'DECaLS_15150_r' to get tile number
obj = ccd.object.strip()
words = obj.split('_')
tile = None
if len(words) == 3 and words[0] == 'DECaLS':
try:
tileid = int(words[1])
tile = tiles[tileid - 1]
if tile.tileid != tileid:
I = np.flatnonzero(tile.tileid == tileid)
tile = tiles[I[0]]
except:
pass
if tile is not None:
ccds.tileid [iccd] = tile.tileid
ccds.tilepass[iccd] = tile.get('pass')
ccds.tileebv [iccd] = tile.ebv_med
# Instantiate PSF on a grid
S = 32
xx = np.linspace(1+S, W-S, 5)
yy = np.linspace(1+S, H-S, 5)
xx,yy = np.meshgrid(xx, yy)
psfnorms = []
galnorms = []
for x,y in zip(xx.ravel(), yy.ravel()):
p = im.psf_norm(tim, x=x, y=y)
g = im.galaxy_norm(tim, x=x, y=y)
psfnorms.append(p)
galnorms.append(g)
ccds.psfnorm_mean[iccd] = np.mean(psfnorms)
ccds.psfnorm_std [iccd] = np.std (psfnorms)
ccds.galnorm_mean[iccd] = np.mean(galnorms)
ccds.galnorm_std [iccd] = np.std (galnorms)
# PSF in center of field
cx,cy = (W+1)/2., (H+1)/2.
p = psf.getPointSourcePatch(cx, cy).patch
ph,pw = p.shape
px,py = np.meshgrid(np.arange(pw), np.arange(ph))
psum = np.sum(p)
# print('psum', psum)
p /= psum
# centroids
cenx = np.sum(p * px)
ceny = np.sum(p * py)
# print('cenx,ceny', cenx,ceny)
# second moments
x2 = np.sum(p * (px - cenx)**2)
y2 = np.sum(p * (py - ceny)**2)
xy = np.sum(p * (px - cenx)*(py - ceny))
# semi-major/minor axes and position angle
theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.)
theta = np.abs(theta) * np.sign(xy)
s = np.sqrt(((x2 - y2)/2.)**2 + xy**2)
a = np.sqrt((x2 + y2) / 2. + s)
b = np.sqrt((x2 + y2) / 2. - s)
ell = 1. - b/a
# print('PSF second moments', x2, y2, xy)
# print('PSF position angle', theta)
# print('PSF semi-axes', a, b)
# print('PSF ellipticity', ell)
ccds.psf_mx2[iccd] = x2
ccds.psf_my2[iccd] = y2
ccds.psf_mxy[iccd] = xy
ccds.psf_a[iccd] = a
ccds.psf_b[iccd] = b
ccds.psf_theta[iccd] = theta
ccds.psf_ell [iccd] = ell
print('Computing Gaussian approximate PSF quantities...')
# Galaxy norm using Gaussian approximation of PSF.
realpsf = tim.psf
tim.psf = im.read_psf_model(0, 0, gaussPsf=True,
psf_sigma=tim.psf_sigma)
gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy)
tim.psf = realpsf
# Sky -- evaluate on a grid (every ~10th pixel)
skygrid = sky.evaluateGrid(np.linspace(0, ccd.width-1, int(1+ccd.width/10)),
np.linspace(0, ccd.height-1, int(1+ccd.height/10)))
ccds.meansky[iccd] = np.mean(skygrid)
ccds.stdsky[iccd] = np.std(skygrid)
ccds.maxsky[iccd] = skygrid.max()
ccds.minsky[iccd] = skygrid.min()
# WCS
ccds.ra0[iccd],ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1)
ccds.ra1[iccd],ccds.dec1[iccd] = wcs.pixelxy2radec(1, H)
ccds.ra2[iccd],ccds.dec2[iccd] = wcs.pixelxy2radec(W, H)
ccds.ra3[iccd],ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1)
midx, midy = (W+1)/2., (H+1)/2.
rc,dc = wcs.pixelxy2radec(midx, midy)
ra,dec = wcs.pixelxy2radec([1,W,midx,midx], [midy,midy,1,H])
ccds.dra [iccd] = max(degrees_between(ra, dc+np.zeros_like(ra),
rc, dc))
ccds.ddec[iccd] = max(degrees_between(rc+np.zeros_like(dec), dec,
rc, dc))
ccds.ra_center [iccd] = rc
ccds.dec_center[iccd] = dc
# Compute scale change across the chip
# how many pixels to step
step = 10
xx = np.linspace(1+step, W-step, 5)
yy = np.linspace(1+step, H-step, 5)
xx,yy = np.meshgrid(xx, yy)
pixscale = []
for x,y in zip(xx.ravel(), yy.ravel()):
sx = [x-step, x-step, x+step, x+step, x-step]
sy = [y-step, y+step, y+step, y-step, y-step]
sr,sd = wcs.pixelxy2radec(sx, sy)
rc,dc = wcs.pixelxy2radec(x, y)
# project around a tiny little TAN WCS at (x,y), with 1" pixels
locwcs = Tan(rc, dc, 0., 0., 1./3600, 0., 0., 1./3600, 1., 1.)
ok,lx,ly = locwcs.radec2pixelxy(sr, sd)
#print('local x,y:', lx, ly)
A = polygon_area((lx, ly))
pixscale.append(np.sqrt(A / (2*step)**2))
# print('Pixel scales:', pixscale)
ccds.pixscale_mean[iccd] = np.mean(pixscale)
ccds.pixscale_min[iccd] = min(pixscale)
ccds.pixscale_max[iccd] = max(pixscale)
ccds.pixscale_std[iccd] = np.std(pixscale)
ccds.plver = np.array(plvers)
sfd = tractor.sfd.SFDMap()
allbands = 'ugrizY'
filts = ['%s %s' % ('DES', f) for f in allbands]
wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
ebv,ext = sfd.extinction(filts + wisebands, ccds.ra_center,
ccds.dec_center, get_ebv=True)
ext = ext.astype(np.float32)
ccds.ebv = ebv.astype(np.float32)
ccds.decam_extinction = ext[:,:len(allbands)]
ccds.wise_extinction = ext[:,len(allbands):]
# Depth
detsig1 = ccds.sig1 / ccds.psfnorm_mean
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.psfdepth = -2.5 * (np.log10(depth) - 9)
detsig1 = ccds.sig1 / ccds.galnorm_mean
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.galdepth = -2.5 * (np.log10(depth) - 9)
# Depth using Gaussian FWHM.
psf_sigma = ccds.fwhm / 2.35
gnorm = 1./(2. * np.sqrt(np.pi) * psf_sigma)
detsig1 = ccds.sig1 / gnorm
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9)
# Gaussian galaxy depth
detsig1 = ccds.sig1 / gaussgalnorm
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9)
ccds.writeto(outfn)
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(
description=
"This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
)
parser.add_argument('--calibs',
action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper',
type=int,
default=None,
help='Batch N calibs per line')
parser.add_argument('--forced',
action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb',
action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--touching',
action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--near',
action='store_true',
help='Quick cut to only CCDs near selected bricks')
parser.add_argument('--check',
action='store_true',
help='Check which calibrations actually need to run.')
parser.add_argument('--check-coadd',
action='store_true',
help='Check which caoadds actually need to run.')
parser.add_argument('--out',
help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command',
action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--opt', help='With --command, extra options to add')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--ignore_cuts',
action='store_true',
default=False,
help='no photometric or blacklist cuts')
parser.add_argument('--save_to_fits',
action='store_true',
default=False,
help='save cut brick,ccd to fits table')
parser.add_argument(
'--name',
action='store',
default='dr3',
help='save with this suffix, e.g. refers to ccds table')
parser.add_argument('--delete-sky',
action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--delete-pvastrom',
action='store_true',
help='Delete any existing PV WCS calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
parser.add_argument(
'--brickq',
type=int,
default=None,
help='Queue only bricks with the given "brickq" value [0 to 3]')
parser.add_argument(
'--brickq-deps',
action='store_true',
default=False,
help='Queue bricks directly using qdo API, setting brickq dependencies'
)
parser.add_argument('--queue',
default='bricks',
help='With --brickq-deps, the QDO queue name to use')
opt = parser.parse_args()
survey = LegacySurveyData()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = survey.get_bricks()
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
if opt.ignore_cuts == False:
I = survey.photometric_ccds(T)
print(len(I), 'CCDs are photometric')
T.cut(I)
I = survey.apply_blacklist(T)
print(len(I), 'CCDs are not blacklisted')
T.cut(I)
print(len(T), 'CCDs remain')
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo, dhi = -25, 40
rlo, rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(
test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo, dhi = -90, 90
rlo, rhi = 0, 360
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo, rhi = 240, 245
dlo, dhi = 5, 12
elif opt.region == 'edrplus':
rlo, rhi = 235, 248
dlo, dhi = 5, 15
elif opt.region == 'edr-south':
rlo, rhi = 240, 245
dlo, dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo, rhi = 149.75, 150.75
dlo, dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo, rhi = 240, 250
dlo, dhi = 10, 15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo, rhi = 320., 360.
dlo, dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo, rhi = 240, 360
dlo, dhi = 20, 40
elif opt.region == 'north':
rlo, rhi = 120, 240
dlo, dhi = 20, 40
elif opt.region == 'northeast':
rlo, rhi = 0, 120
dlo, dhi = 20, 40
elif opt.region == 'southwest':
rlo, rhi = 240, 360
dlo, dhi = -20, 0
elif opt.region == 'south':
rlo, rhi = 120, 240
dlo, dhi = -20, 0
elif opt.region == 'southeast':
rlo, rhi = 0, 120
dlo, dhi = -20, 0
elif opt.region == 'southsoutheast':
rlo, rhi = 0, 120
dlo, dhi = -20, -10
elif opt.region == 'midwest':
rlo, rhi = 240, 360
dlo, dhi = 0, 20
elif opt.region == 'middle':
rlo, rhi = 120, 240
dlo, dhi = 0, 20
elif opt.region == 'mideast':
rlo, rhi = 0, 120
dlo, dhi = 0, 20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo, rhi = 250, 260
dlo, dhi = 30, 35
elif opt.region == 'deep2f3':
rlo, rhi = 351.25, 353.75
dlo, dhi = 0, 0.5
elif opt.region == 'virgo':
rlo, rhi = 185, 190
dlo, dhi = 10, 15
elif opt.region == 'virgo2':
rlo, rhi = 182, 192
dlo, dhi = 8, 18
elif opt.region == 'lsb':
rlo, rhi = 147.2, 147.8
dlo, dhi = -0.4, 0.4
elif opt.region == 'eboss-sgc':
# generous boundaries to make sure get all relevant images
# RA -45 to +45
# Dec -5 to +7
rlo, rhi = 310., 50.
dlo, dhi = -6., 6.
elif opt.region == 'eboss-ngc':
# generous boundaries to make sure get all relevant images
# NGC ELGs
# RA 115 to 175
# Dec 15 to 30
rlo, rhi = 122., 177.
dlo, dhi = 12., 32.
elif opt.region == 'mzls':
dlo, dhi = 30., 90.
elif opt.region == 'dr4-bootes':
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
#dlo,dhi = 34., 35.
#rlo,rhi = 209.5, 210.5
dlo, dhi = 33., 36.
rlo, rhi = 216.5, 219.5
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(
np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
(B.dec <= dhi))
log(len(B), 'bricks in range')
#for name in B.get('brickname'):
#print(name)
B.writeto('bricks-cut.fits')
I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs')
plt.clf()
plt.plot(B.ra, B.dec, 'b.')
plt.title('DR3 bricks')
plt.axis([360, 0, np.min(B.dec) - 1, np.max(B.dec) + 1])
plt.savefig('bricks.png')
if opt.brickq is not None:
B.cut(B.brickq == opt.brickq)
log('Cut to', len(B), 'with brickq =', opt.brickq)
if opt.touching:
keep = np.zeros(len(T), bool)
for j in J:
keep[j] = True
T.cut(keep)
log('Cut to', len(T), 'CCDs near bricks')
# Aside -- how many near DR1=1 CCDs?
if False:
T2 = D.get_ccds()
log(len(T2), 'CCDs')
T2.cut(T2.dr1 == 1)
log(len(T2), 'CCDs marked DR1=1')
log(len(B), 'bricks in range')
I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B2 = B[keep]
log('Total of', len(B2), 'bricks near CCDs with DR1=1')
for band in 'grz':
Tb = T2[T2.filter == band]
log(len(Tb), 'in filter', band)
I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec,
survey.bricksize)
good = np.zeros(len(B2), np.uint8)
for i in I:
good[i] = 1
B2.set('has_' + band, good)
B2.writeto('survey-bricks-in-dr1.fits')
sys.exit(0)
# sort by dec decreasing
#B.cut(np.argsort(-B.dec))
# RA increasing
B.cut(np.argsort(B.ra))
for b in B:
if opt.check:
fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3],
b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
if opt.check_coadd:
fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (
b.brickname[:3], b.brickname, b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
#print(b.brickname)
if opt.save_to_fits:
assert (opt.touching)
# Write cut tables to file
for tab, typ in zip([B, T], ['bricks', 'ccds']):
fn = '%s-%s-cut.fits' % (typ, opt.region)
if os.path.exists(fn):
os.remove(fn)
tab.writeto(fn)
print('Wrote %s' % fn)
# Write text files listing ccd and filename names
nm1, nm2 = 'ccds-%s.txt' % opt.region, 'filenames-%s.txt' % opt.region
if os.path.exists(nm1):
os.remove(nm1)
if os.path.exists(nm2):
os.remove(nm2)
f1, f2 = open(nm1, 'w'), open(nm2, 'w')
fns = list(set(T.get('image_filename')))
for fn in fns:
f2.write('%s\n' % fn.strip())
for ti in T:
f1.write('%s\n' % ti.get('image_filename').strip())
f1.close()
f2.close()
print('Wrote *-names.txt')
if opt.brickq_deps:
import qdo
from legacypipe.survey import on_bricks_dependencies
#... find Queue...
q = qdo.connect(opt.queue, create_ok=True)
print('Connected to QDO queue', opt.queue, q)
brick_to_task = dict()
I = survey.photometric_ccds(T)
print(len(I), 'CCDs are photometric')
T.cut(I)
I = survey.apply_blacklist(T)
print(len(I), 'CCDs are not blacklisted')
T.cut(I)
print(len(T), 'CCDs remaining')
T.wra = T.ra + (T.ra > 180) * -360
wra = rlo - 360
plt.clf()
plt.plot(T.wra, T.dec, 'b.')
ax = [wra, rhi, dlo, dhi]
plt.axis(ax)
plt.title('CCDs')
plt.savefig('q-ccds.png')
B.wra = B.ra + (B.ra > 180) * -360
# this slight overestimate (for DECam images) is fine
radius = 0.3
Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius, indexlist=True)
ikeep = []
for ib, (b, Iccd) in enumerate(zip(B, Iccds)):
if Iccd is None or len(Iccd) == 0:
print('No matched CCDs to brick', b.brickname)
continue
wcs = wcs_for_brick(b)
cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
print(len(cI), 'CCDs touching brick', b.brickname)
if len(cI) == 0:
continue
ikeep.append(ib)
B.cut(np.array(ikeep))
print('Cut to', len(B), 'bricks touched by CCDs')
for brickq in range(4):
I = np.flatnonzero(B.brickq == brickq)
print(len(I), 'bricks with brickq =', brickq)
J = np.flatnonzero(B.brickq < brickq)
preB = B[J]
reqs = []
if brickq > 0:
for b in B[I]:
# find brick dependencies
brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
# convert to task ids
taskdeps = [
brick_to_task.get(b.brickname, None) for b in brickdeps
]
# If we dropped a dependency brick from a previous brickq because
# of no overlapping CCDs, it won't appear in the brick_to_task map.
taskdeps = [t for t in taskdeps if t is not None]
reqs.append(taskdeps)
plt.clf()
plt.plot(B.wra, B.dec, '.', color='0.5')
plt.plot(B.wra[I], B.dec[I], 'b.')
plt.axis(ax)
plt.title('Bricks: brickq=%i' % brickq)
plt.savefig('q-bricks-%i.png' % brickq)
# submit to qdo queue
print('Queuing', len(B[I]), 'bricks')
if brickq == 0:
reqs = None
else:
assert (len(I) == len(reqs))
taskids = q.add_multiple(B.brickname[I], requires=reqs)
assert (len(taskids) == len(I))
print('Queued', len(taskids), 'bricks')
brick_to_task.update(dict(zip(B.brickname[I], taskids)))
if not (opt.calibs or opt.forced or opt.lsb):
sys.exit(0)
bands = 'grz'
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
if opt.touching:
allI = set()
for b in B:
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickid,
'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
if len(I) == 0:
continue
allI.update(I)
allI = list(allI)
allI.sort()
elif opt.near:
# Roughly brick radius + DECam image radius
radius = 0.35
allI, nil, nil = match_radec(T.ra,
T.dec,
B.ra,
B.dec,
radius,
nearest=True)
else:
allI = np.arange(len(T))
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
expstr = '%08i' % T.expnum[i]
outfn = os.path.join(
'forced', expstr[:5], expstr,
'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i]))
imgfn = os.path.join(survey.survey_dir, 'images',
T.image_filename[i].strip())
if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
and os.path.exists(imgfn[:-3])):
imgfn = imgfn[:-3]
#f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
# (imgfn, T.image_hdu[i], outfn))
f.write(
'python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n'
% (T.expnum[i], T.ccdname[i], outfn))
f.close()
log('Wrote', opt.out)
sys.exit(0)
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write(
'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
% (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
sys.exit(0)
log('Writing calibs to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if cmd is None:
cmd = ''
f.write(cmd + ' '.join(batch) + '\n')
cmd = None
if opt.command:
cmd = 'python legacypipe/run-calib.py '
if opt.opt is not None:
cmd += opt.opt + ' '
for j, i in enumerate(allI):
if opt.delete_sky or opt.delete_pvastrom:
log(j + 1, 'of', len(allI))
im = survey.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
log(' deleting:', im.pvwcsfn)
os.unlink(im.pvwcsfn)
if opt.check:
log(j + 1, 'of', len(allI))
im = survey.get_image_object(T[i])
if not im.run_calibs(im, just_check=True):
log('Calibs for', im.expnum, im.ccdname, im.calname,
'already done')
continue
if opt.command:
s = '%i-%s' % (T.expnum[i], T.ccdname[i])
prefix = 'python legacypipe/run-calib.py ' + opt.opt
#('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
# (T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
prefix = ''
if j < 10:
print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname',
T.ccdname[i], 'filename', T.image_filename[i])
if not opt.nper:
f.write(prefix + s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, cmd)
batch = []
if opt.check:
f.flush()
if len(batch):
write_batch(f, batch, cmd)
f.close()
log('Wrote', opt.out)
return 0