def getbrickfiles(brickname=None):
survey = LegacySurveyData()
brickinfo = survey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
ccdinfo = survey.ccds_touching_wcs(brickwcs)
nccd = len(ccdinfo)
calibdir = survey.get_calib_dir()
imagedir = survey.survey_dir
# Construct image file names and the calibration file names.
expnum = ccdinfo.expnum
ccdname = ccdinfo.ccdname
psffiles = list()
skyfiles = list()
imagefiles = list()
for ccd in ccdinfo:
info = survey.get_image_object(ccd)
for attr in ['imgfn', 'dqfn', 'wtfn']:
fn = getattr(info, attr).replace(imagedir+'/', '')
#if '160108_073601' in fn:
# pdb.set_trace()
imagefiles.append(fn)
psffiles.append(info.psffn.replace(calibdir, 'calib'))
skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))
#for ii in range(nccd):
#exp = '{0:08d}'.format(expnum[ii])
#rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
#psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
#skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
#imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))
#print(np.array(imagefiles))
#print(np.array(psffiles))
#print(np.array(skyfiles))
return imagefiles, psffiles, skyfiles
def getbrickfiles(brickname=None):
survey = LegacySurveyData()
brickinfo = survey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
ccdinfo = survey.ccds_touching_wcs(brickwcs)
nccd = len(ccdinfo)
calibdir = survey.get_calib_dir()
imagedir = survey.survey_dir
# Construct image file names and the calibration file names.
expnum = ccdinfo.expnum
ccdname = ccdinfo.ccdname
psffiles = list()
skyfiles = list()
imagefiles = list()
for ccd in ccdinfo:
info = survey.get_image_object(ccd)
for attr in ['imgfn', 'dqfn', 'wtfn']:
fn = getattr(info, attr).replace(imagedir + '/', '')
#if '160108_073601' in fn:
# pdb.set_trace()
imagefiles.append(fn)
psffiles.append(info.psffn.replace(calibdir, 'calib'))
skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))
#for ii in range(nccd):
#exp = '{0:08d}'.format(expnum[ii])
#rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
#psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
#skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
#imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))
#print(np.array(imagefiles))
#print(np.array(psffiles))
#print(np.array(skyfiles))
return imagefiles, psffiles, skyfiles
def __init__(
self,
expnum=None,
ccdname=None,
ccdwcs=None,
pattern='/project/projectdirs/cosmo/work/ps1/cats/chunks-qz-star-v3/ps1-%(hp)05d.fits'
):
"""Read PS1 or gaia sources for an exposure number + CCD name or CCD WCS
Args:
expnum, ccdname: select catalogue with these
ccdwcs: or select catalogue with this
pattern: absolute path and wildcard for PS1 or Gaia catalogues
dr: /project/projectdirs/cosmo/work/
PS1: ${dr}/ps1/cats/chunks-qz-star-v3/ps1-%(hp)05d.fits
PS1-Gaia: ${dr}/gaia/chunks-ps1-gaia/chunk-%(hp)05d.fits
"""
assert ('ps1' in pattern or 'gaia' in pattern)
#assert(ps1_or_gaia in ['ps1','ps1_gaia'])
#if ps1_or_gaia == 'ps1':
# # PS1 "qz" directory
# # e.g. /project/projectdirs/cosmo/work/ps1/cats/chunks-qz-star-v2
# self.catdir= os.getenv('PS1CAT_DIR')
#elif ps1_or_gaia == 'ps1_gaia':
# # PS1-Gaia "qz" matches-only directory
# # e.g. /project/projectdirs/cosmo/work/gaia/chunks-ps1-gaia
# self.catdir= os.getenv('PS1_GAIA_MATCHES')
#fnpattern = os.path.join(self.catdir, prefix + '-%(hp)05d.fits')
super(ps1cat, self).__init__(pattern)
if ccdwcs is None:
from legacypipe.survey import LegacySurveyData
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
self.ccdwcs = im.get_wcs()
else:
self.ccdwcs = ccdwcs
def __init__(self, expnum=None, ccdname=None, ccdwcs=None):
"""Initialize the class with either the exposure number *and* CCD name, or
directly with the WCS of the CCD of interest.
"""
# GAIA and PS1 info, gaia for astrometry, ps1 for photometry
self.gaiadir = os.getenv('GAIACAT_DIR')
# PS1 only
self.ps1dir = os.getenv('PS1CAT_DIR') # PS1 only
if self.ps1dir is None:
raise ValueError('Need PS1CAT_DIR environment variable to be set.')
if self.gaiadir is None:
print(
'WARNING: GAIACAT_DIR environment variable not set: using Pan-STARRS1 for astrometry'
)
self.nside = 32
if ccdwcs is None:
from legacypipe.survey import LegacySurveyData
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
self.ccdwcs = im.get_wcs()
else:
self.ccdwcs = ccdwcs
def __init__(self, expnum=None, ccdname=None, ccdwcs=None):
"""Read PS1 or gaia sources for an exposure number + CCD name or CCD WCS
Args:
expnum, ccdname: select catalogue with these
ccdwcs: or select catalogue with this
"""
self.ps1catdir = os.getenv('PS1CAT_DIR')
if self.ps1catdir is None:
raise ValueError(
'You must have the PS1CAT_DIR environment variable set to point to healpixed PS1 catalogs'
)
fnpattern = os.path.join(self.ps1catdir, 'ps1-%(hp)05d.fits')
super(ps1cat, self).__init__(fnpattern)
if ccdwcs is None:
from legacypipe.survey import LegacySurveyData
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
self.ccdwcs = im.get_wcs()
else:
self.ccdwcs = ccdwcs
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).")
parser.add_argument('--calibs', action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper', type=int, default=None,
help='Batch N calibs per line')
parser.add_argument('--forced', action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb', action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--touching', action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--near', action='store_true',
help='Quick cut to only CCDs near selected bricks')
parser.add_argument('--check', action='store_true',
help='Check which calibrations actually need to run.')
parser.add_argument('--check-coadd', action='store_true',
help='Check which caoadds actually need to run.')
parser.add_argument('--out', help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command', action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--opt', help='With --command, extra options to add')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric or blacklist cuts')
parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table')
parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table')
parser.add_argument('--delete-sky', action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--delete-pvastrom', action='store_true',
help='Delete any existing PV WCS calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
parser.add_argument('--brickq', type=int, default=None,
help='Queue only bricks with the given "brickq" value [0 to 3]')
parser.add_argument('--brickq-deps', action='store_true', default=False,
help='Queue bricks directly using qdo API, setting brickq dependencies')
parser.add_argument('--queue', default='bricks',
help='With --brickq-deps, the QDO queue name to use')
opt = parser.parse_args()
survey = LegacySurveyData()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = survey.get_bricks()
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
if opt.ignore_cuts == False:
I = survey.photometric_ccds(T)
print(len(I), 'CCDs are photometric')
T.cut(I)
I = survey.apply_blacklist(T)
print(len(I), 'CCDs are not blacklisted')
T.cut(I)
print(len(T), 'CCDs remain')
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo,dhi = -25, 40
rlo,rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo,dhi = -90,90
rlo,rhi = 0, 360
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo,rhi = 240,245
dlo,dhi = 5, 12
elif opt.region == 'edrplus':
rlo,rhi = 235,248
dlo,dhi = 5, 15
elif opt.region == 'edr-south':
rlo,rhi = 240,245
dlo,dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo,rhi = 149.75, 150.75
dlo,dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo,rhi = 240,250
dlo,dhi = 10,15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo,rhi = 320., 360.
dlo,dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo,rhi = 240,360
dlo,dhi = 20,40
elif opt.region == 'north':
rlo,rhi = 120,240
dlo,dhi = 20,40
elif opt.region == 'northeast':
rlo,rhi = 0,120
dlo,dhi = 20,40
elif opt.region == 'southwest':
rlo,rhi = 240,360
dlo,dhi = -20,0
elif opt.region == 'south':
rlo,rhi = 120,240
dlo,dhi = -20,0
elif opt.region == 'southeast':
rlo,rhi = 0,120
dlo,dhi = -20,0
elif opt.region == 'southsoutheast':
rlo,rhi = 0,120
dlo,dhi = -20,-10
elif opt.region == 'midwest':
rlo,rhi = 240,360
dlo,dhi = 0,20
elif opt.region == 'middle':
rlo,rhi = 120,240
dlo,dhi = 0,20
elif opt.region == 'mideast':
rlo,rhi = 0,120
dlo,dhi = 0,20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo,rhi = 250,260
dlo,dhi = 30,35
elif opt.region == 'deep2f3':
rlo,rhi = 351.25, 353.75
dlo,dhi = 0, 0.5
elif opt.region == 'virgo':
rlo,rhi = 185,190
dlo,dhi = 10, 15
elif opt.region == 'virgo2':
rlo,rhi = 182,192
dlo,dhi = 8, 18
elif opt.region == 'lsb':
rlo,rhi = 147.2, 147.8
dlo,dhi = -0.4, 0.4
elif opt.region == 'eboss-elg':
# RA -45 to +45
# Dec -5 to +7
rlo,rhi = 315., 45.
dlo,dhi = -5., 7.
elif opt.region == 'eboss-ngc':
# NGC ELGs
# RA 115 to 175
# Dec 15 to 30
rlo,rhi = 115., 175.
dlo,dhi = 15., 30.
elif opt.region == 'mzls':
dlo,dhi = 30., 90.
elif opt.region == 'dr4-bootes':
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
#dlo,dhi = 34., 35.
#rlo,rhi = 209.5, 210.5
dlo,dhi = 33., 36.
rlo,rhi = 216.5, 219.5
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
log(len(B), 'bricks in range')
for name in B.get('brickname'):
print(name)
B.writeto('bricks-cut.fits')
I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs')
plt.clf()
plt.plot(B.ra, B.dec, 'b.')
plt.title('DR3 bricks')
plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
plt.savefig('bricks.png')
if opt.brickq is not None:
B.cut(B.brickq == opt.brickq)
log('Cut to', len(B), 'with brickq =', opt.brickq)
if opt.touching:
keep = np.zeros(len(T), bool)
for j in J:
keep[j] = True
T.cut(keep)
log('Cut to', len(T), 'CCDs near bricks')
# Aside -- how many near DR1=1 CCDs?
if False:
T2 = D.get_ccds()
log(len(T2), 'CCDs')
T2.cut(T2.dr1 == 1)
log(len(T2), 'CCDs marked DR1=1')
log(len(B), 'bricks in range')
I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B2 = B[keep]
log('Total of', len(B2), 'bricks near CCDs with DR1=1')
for band in 'grz':
Tb = T2[T2.filter == band]
log(len(Tb), 'in filter', band)
I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, survey.bricksize)
good = np.zeros(len(B2), np.uint8)
for i in I:
good[i] = 1
B2.set('has_' + band, good)
B2.writeto('survey-bricks-in-dr1.fits')
sys.exit(0)
# sort by dec decreasing
#B.cut(np.argsort(-B.dec))
# RA increasing
B.cut(np.argsort(B.ra))
for b in B:
if opt.check:
fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
if opt.check_coadd:
fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
print(b.brickname)
if opt.save_to_fits:
assert(opt.touching)
# Write cut tables to file
for tab,typ in zip([B,T],['bricks','ccds']):
fn='%s-%s-cut.fits' % (typ,opt.name)
if os.path.exists(fn):
os.remove(fn)
tab.writeto(fn)
print('Wrote %s' % fn)
# Write text files listing ccd and filename names
nm1,nm2= 'ccds-%s.txt'% opt.name,'filenames-%s.txt' % opt.name
if os.path.exists(nm1):
os.remove(nm1)
if os.path.exists(nm2):
os.remove(nm2)
f1,f2=open(nm1,'w'),open(nm2,'w')
fns= list(set(T.get('image_filename')))
for fn in fns:
f2.write('%s\n' % fn.strip())
for ti in T:
f1.write('%s\n' % ti.get('image_filename').strip())
f1.close()
f2.close()
print('Wrote *-names.txt')
if opt.brickq_deps:
import qdo
from legacypipe.survey import on_bricks_dependencies
#... find Queue...
q = qdo.connect(opt.queue, create_ok=True)
print('Connected to QDO queue', opt.queue, q)
brick_to_task = dict()
I = survey.photometric_ccds(T)
print(len(I), 'CCDs are photometric')
T.cut(I)
I = survey.apply_blacklist(T)
print(len(I), 'CCDs are not blacklisted')
T.cut(I)
print(len(T), 'CCDs remaining')
T.wra = T.ra + (T.ra > 180) * -360
wra = rlo - 360
plt.clf()
plt.plot(T.wra, T.dec, 'b.')
ax = [wra, rhi, dlo, dhi]
plt.axis(ax)
plt.title('CCDs')
plt.savefig('q-ccds.png')
B.wra = B.ra + (B.ra > 180) * -360
# this slight overestimate (for DECam images) is fine
radius = 0.3
Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius,
indexlist=True)
ikeep = []
for ib,(b,Iccd) in enumerate(zip(B, Iccds)):
if Iccd is None or len(Iccd) == 0:
print('No matched CCDs to brick', b.brickname)
continue
wcs = wcs_for_brick(b)
cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
print(len(cI), 'CCDs touching brick', b.brickname)
if len(cI) == 0:
continue
ikeep.append(ib)
B.cut(np.array(ikeep))
print('Cut to', len(B), 'bricks touched by CCDs')
for brickq in range(4):
I = np.flatnonzero(B.brickq == brickq)
print(len(I), 'bricks with brickq =', brickq)
J = np.flatnonzero(B.brickq < brickq)
preB = B[J]
reqs = []
if brickq > 0:
for b in B[I]:
# find brick dependencies
brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
# convert to task ids
taskdeps = [brick_to_task.get(b.brickname,None) for b in brickdeps]
# If we dropped a dependency brick from a previous brickq because
# of no overlapping CCDs, it won't appear in the brick_to_task map.
taskdeps = [t for t in taskdeps if t is not None]
reqs.append(taskdeps)
plt.clf()
plt.plot(B.wra, B.dec, '.', color='0.5')
plt.plot(B.wra[I], B.dec[I], 'b.')
plt.axis(ax)
plt.title('Bricks: brickq=%i' % brickq)
plt.savefig('q-bricks-%i.png' % brickq)
# submit to qdo queue
print('Queuing', len(B[I]), 'bricks')
if brickq == 0:
reqs = None
else:
assert(len(I) == len(reqs))
taskids = q.add_multiple(B.brickname[I], requires=reqs)
assert(len(taskids) == len(I))
print('Queued', len(taskids), 'bricks')
brick_to_task.update(dict(zip(B.brickname[I], taskids)))
if not (opt.calibs or opt.forced or opt.lsb):
sys.exit(0)
bands = 'grz'
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
if opt.touching:
allI = set()
for b in B:
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
if len(I) == 0:
continue
allI.update(I)
allI = list(allI)
allI.sort()
elif opt.near:
# Roughly brick radius + DECam image radius
radius = 0.35
allI,nil,nil = match_radec(T.ra, T.dec, B.ra, B.dec, radius, nearest=True)
else:
allI = np.arange(len(T))
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
for j,i in enumerate(allI):
expstr = '%08i' % T.expnum[i]
outfn = os.path.join('forced', expstr[:5], expstr,
'decam-%s-%s-forced.fits' %
(expstr, T.ccdname[i]))
imgfn = os.path.join(survey.survey_dir, 'images',
T.image_filename[i].strip())
if (not os.path.exists(imgfn) and
imgfn.endswith('.fz') and
os.path.exists(imgfn[:-3])):
imgfn = imgfn[:-3]
#f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
# (imgfn, T.image_hdu[i], outfn))
f.write('python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n' %
(T.expnum[i], T.ccdname[i], outfn))
f.close()
log('Wrote', opt.out)
sys.exit(0)
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
for j,i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
sys.exit(0)
log('Writing calibs to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if cmd is None:
cmd = ''
f.write(cmd + ' '.join(batch) + '\n')
cmd = None
if opt.command:
cmd = 'python legacypipe/run-calib.py '
if opt.opt is not None:
cmd += opt.opt + ' '
for j,i in enumerate(allI):
if opt.delete_sky or opt.delete_pvastrom:
log(j+1, 'of', len(allI))
im = survey.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
log(' deleting:', im.pvwcsfn)
os.unlink(im.pvwcsfn)
if opt.check:
log(j+1, 'of', len(allI))
im = survey.get_image_object(T[i])
if not im.run_calibs(im, just_check=True):
log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done')
continue
if opt.command:
s = '%i-%s' % (T.expnum[i], T.ccdname[i])
prefix = 'python legacypipe/run-calib.py ' + opt.opt
#('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
# (T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
prefix = ''
if j < 10:
print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i],
'filename', T.image_filename[i])
if not opt.nper:
f.write(prefix + s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, cmd)
batch = []
if opt.check:
f.flush()
if len(batch):
write_batch(f, batch, cmd)
f.close()
log('Wrote', opt.out)
return 0
def main():
survey = LegacySurveyData()
ccds = survey.get_ccds()
print(len(ccds), 'CCDs')
expnums = np.unique(ccds.expnum)
print(len(expnums), 'unique exposures')
for expnum in expnums:
expnumstr = '%08i' % expnum
skyoutfn = os.path.join('splinesky', expnumstr[:5], 'decam-%s.fits' % expnumstr)
psfoutfn = os.path.join('psfex', expnumstr[:5], 'decam-%s.fits' % expnumstr)
if os.path.exists(skyoutfn) and os.path.exists(psfoutfn):
print('Exposure', expnum, 'is done already')
continue
C = ccds[ccds.expnum == expnum]
print(len(C), 'CCDs in expnum', expnum)
psfex = []
psfhdrvals = []
splinesky = []
skyhdrvals = []
for ccd in C:
im = survey.get_image_object(ccd)
fn = im.splineskyfn
if os.path.exists(fn):
T = fits_table(fn)
splinesky.append(T)
# print(fn)
# T.about()
hdr = fitsio.read_header(fn)
skyhdrvals.append([hdr[k] for k in [
'SKY', 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
else:
print('File not found:', fn)
fn = im.psffn
if os.path.exists(fn):
T = fits_table(fn)
hdr = fitsio.read_header(fn, ext=1)
keys = ['LOADED', 'ACCEPTED', 'CHI2', 'POLNAXIS',
'POLNGRP', 'PSF_FWHM', 'PSF_SAMP', 'PSFNAXIS',
'PSFAXIS1', 'PSFAXIS2', 'PSFAXIS3',]
if hdr['POLNAXIS'] == 0:
# No polynomials. Fake it.
T.polgrp1 = np.array([0])
T.polgrp2 = np.array([0])
T.polname1 = np.array(['fake'])
T.polname2 = np.array(['fake'])
T.polzero1 = np.array([0])
T.polzero2 = np.array([0])
T.polscal1 = np.array([1])
T.polscal2 = np.array([1])
T.poldeg1 = np.array([0])
T.poldeg2 = np.array([0])
else:
keys.extend([
'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1',
'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2',
'POLDEG1'])
for k in keys:
T.set(k.lower(), np.array([hdr[k]]))
psfex.append(T)
#print(fn)
#T.about()
hdr = fitsio.read_header(fn)
psfhdrvals.append([hdr.get(k,'') for k in [
'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
else:
print('File not found:', fn)
if len(psfex):
padded = pad_arrays([p.psf_mask[0] for p in psfex])
cols = psfex[0].columns()
cols.remove('psf_mask')
T = merge_tables(psfex, columns=cols)
T.psf_mask = np.concatenate([[p] for p in padded])
T.legpipev = np.array([h[0] for h in psfhdrvals])
T.plver = np.array([h[1] for h in psfhdrvals])
T.expnum = np.array([h[2] for h in psfhdrvals])
T.ccdname = np.array([h[3] for h in psfhdrvals])
fn = psfoutfn
trymakedirs(fn, dir=True)
T.writeto(fn)
print('Wrote', fn)
if len(splinesky):
T = fits_table()
T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky])
T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky])
padded = pad_arrays([t.gridvals[0] for t in splinesky])
T.gridvals = np.concatenate([[p] for p in padded])
padded = pad_arrays([t.xgrid[0] for t in splinesky])
T.xgrid = np.concatenate([[p] for p in padded])
padded = pad_arrays([t.xgrid[0] for t in splinesky])
T.ygrid = np.concatenate([[p] for p in padded])
cols = splinesky[0].columns()
print('Columns:', cols)
for c in ['gridvals', 'xgrid', 'ygrid']:
cols.remove(c)
T.add_columns_from(merge_tables(splinesky, columns=cols))
T.skyclass = np.array([h[0] for h in skyhdrvals])
T.legpipev = np.array([h[1] for h in skyhdrvals])
T.plver = np.array([h[2] for h in skyhdrvals])
T.expnum = np.array([h[3] for h in skyhdrvals])
T.ccdname = np.array([h[4] for h in skyhdrvals])
fn = skyoutfn
trymakedirs(fn, dir=True)
T.writeto(fn)
print('Wrote', fn)
def main(survey=None, opt=None):
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
import pylab as plt
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.expnum)
filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
filename = opt.expnum
# Try parsing HDU number
try:
hdu = int(opt.ccdname)
ccdname = None
except:
hdu = -1
ccdname = opt.ccdname
if survey is None:
survey = LegacySurveyData()
catsurvey = survey
if opt.catalog_dir is not None:
catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir)
if filename is not None and hdu >= 0:
# FIXME -- try looking up in CCDs file?
# Read metadata from file
print('Warning: faking metadata from file contents')
T = exposure_metadata([filename], hdus=[hdu])
print('Metadata:')
T.about()
if not 'ccdzpt' in T.columns():
phdr = fitsio.read_header(filename)
T.ccdzpt = np.array([phdr['MAGZERO']])
print('WARNING: using header MAGZERO')
T.ccdraoff = np.array([0.])
T.ccddecoff = np.array([0.])
print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if hdu >= 0:
T.cut(T.image_hdu == hdu)
print(len(T), 'with HDU', hdu)
if filename is not None:
T.cut(np.array([f.strip() == filename for f in T.image_filename]))
print(len(T), 'with filename', filename)
if opt.camera is not None:
T.cut(T.camera == opt.camera)
print(len(T), 'with camera', opt.camera)
assert (len(T) == 1)
ccd = T[0]
im = survey.get_image_object(ccd)
if opt.do_calib:
#from legacypipe.survey import run_calibs
#kwa = dict(splinesky=True)
#run_calibs((im, kwa))
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
splinesky=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf)
print('Got tim:', tim)
print('Read image:', Time() - t0)
if opt.catfn in ['DR1', 'DR2', 'DR3', 'DR5', 'DR']:
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = catsurvey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) *
(yy >= -margin) * (yy <= (H + margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
for col in ['out_of_bounds', 'left_blob']:
if col in T.get_columns():
T.cut(T.get(col) == False)
print('Cut to', len(T), 'on', col)
if len(T):
TT.append(T)
if len(TT) == 0:
print('No sources to photometer.')
return 0
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
print('Total of', len(T), 'catalog sources')
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
surveydir = survey.get_survey_dir()
del survey
kwargs = {}
cols = T.get_columns()
if 'flux_r' in cols and not 'decam_flux_r' in cols:
kwargs.update(fluxPrefix='')
cat = read_fits_catalog(T, **kwargs)
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
print('Read catalog:', Time() - t0)
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
ps=ps)
t0 = Time()
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F)).astype(np.int32)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32)
F.ra = T.ra
F.dec = T.dec
ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
F.mask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1),
np.clip(np.round(F.x).astype(int), 0, w - 1)]
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, surveydir)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='EXPOSURE',
value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
comment='Name of this image'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2'
}
columns = F.get_columns()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
print('Getting model image...')
tr = Tractor([tim], cat)
mod = tr.getModelImage(tim)
fitsio.write(opt.save_model, mod, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
print('Finished forced phot:', Time() - t0)
return 0
pixscale = 0.262
cd = pixscale / 3600.
targetwcs = Tan(ra, dec, W / 2., H / 2., -cd, 0., 0., cd, float(W), float(H))
rr, dd = targetwcs.pixelxy2radec([1, W, W, 1, 1], [1, 1, H, H, 1])
targetrd = np.vstack((rr, dd)).T
ccds = survey.ccds_touching_wcs(targetwcs)
print(len(ccds), 'CCDs touching WCS')
print('MJDs', ccds.mjd_obs)
ccds.writeto('test-ccds.fits')
ccd1 = ccds[(ccds.expnum == expnum) * (ccds.ccdname == ccdname)]
print('CCD1:', ccd1)
im1 = survey.get_image_object(ccd1[0])
print('Im:', im1)
#wcs = im1.get_wcs()
#x0,x1,y0,y1,slc = im1.get_image_extent(wcs=wcs, radecpoly=targetrd)
tim1 = im1.get_tractor_image(radecpoly=targetrd,
pixPsf=True,
hybridPsf=True,
normalizePsf=True,
splinesky=True)
print('Tim', tim1)
##
tims = [tim1]
ccd1.ccd_x0 = np.array([tim.x0 for tim in tims]).astype(np.int16)
psf = psf.drawImage(scale=pixscale, nx=W + 1, ny=H + 1)
## psf = tractor.GaussianMixturePSF(1., 0., 0., v, v, 0.)
psf = tractor.psf.PixelizedPSF(psf.array)
return psf
if __name__ == '__main__':
print('Welcome to montelg src.')
survey = LegacySurveyData()
ccds = survey.get_annotated_ccds()
ccds = survey.cleanup_ccds_table(ccds)
im = survey.get_image_object(ccds[0])
bands = ['g', 'r', 'z']
red = dict(g=2.5, r=1., i=0.4, z=0.4)
os.environ[
'LEGACY_SURVEY_DIR'] = '/project/projectdirs/cosmo/data/legacysurvey/dr8/'
## Source.
ra, dec = 40., 10.
gre = 0.40 # [arcsec].
gmag = 23.0
gflux = 10.**(-0.4 * (gmag - 22.5)) # [Nanomaggies].
## gflux = exptime * 10.**((zpt - gmag) / 2.5) # [Total counts on the image].
def psf_residuals(expnum,
ccdname,
stampsize=35,
nstar=30,
magrange=(13, 17),
verbose=0,
splinesky=False):
# Set the debugging level.
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
pngprefix = 'qapsf-{}-{}'.format(expnum, ccdname)
# Gather all the info we need about this CCD.
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
band = ccd.filter
ps1band = dict(g=0, r=1, i=2, z=3, Y=4)
print('Band {}'.format(band))
#scales = dict(g=0.0066, r=0.01, z=0.025)
#vmin, vmax = np.arcsinh(-1), np.arcsinh(100)
#print(scales[band])
im = survey.get_image_object(ccd)
iminfo = im.get_image_info()
H, W = iminfo['dims']
wcs = im.get_wcs()
# Choose a uniformly selected subset of PS1 stars on this CCD.
ps1 = ps1cat(ccdwcs=wcs)
cat = ps1.get_stars(band=band, magrange=magrange)
rand = np.random.RandomState(seed=expnum * ccd.ccdnum)
these = rand.choice(len(cat) - 1, nstar, replace=False)
#these = rand.random_integers(0,len(cat)-1,nstar)
cat = cat[these]
cat = cat[np.argsort(cat.median[:, ps1band[band]])] # sort by magnitude
#print(cat.nmag_ok)
get_tim_kwargs = dict(pixPsf=True, splinesky=splinesky)
# Make a QAplot of the positions of all the stars.
tim = im.get_tractor_image(**get_tim_kwargs)
img = tim.getImage()
#img = tim.getImage()/scales[band]
fig = plt.figure(figsize=(5, 10))
ax = fig.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
#ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest',
# origin='lower',vmin=vmax,vmax=vmax)
ax.imshow(img, **tim.ima)
ax.axis('off')
ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.format(
band, expnum, ccdname, ccd.airmass, ccd.seeing))
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ax.text(xpos,
ypos,
'{:2d}'.format(istar + 1),
color='red',
horizontalalignment='left')
circ = plt.Circle((xpos, ypos), radius=30, color='g', fill=False, lw=1)
ax.add_patch(circ)
#radec = wcs.radec_bounds()
#ax.scatter(cat.ra,cat.dec)
#ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998])
#ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015])
#ax.set_xlabel('$RA\ (deg)$',fontsize=18)
#ax.set_ylabel('$Dec\ (deg)$',fontsize=18)
fig.savefig(pngprefix + '-ccd.png', bbox_inches='tight')
# Initialize the many-stamp QAplot
ncols = 3
nrows = np.ceil(nstar / ncols).astype('int')
inchperstamp = 2.0
fig = plt.figure(figsize=(inchperstamp * 3 * ncols, inchperstamp * nrows))
irow = 0
icol = 0
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
mag = ps1star.median[ps1band[band]] # r-band
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ix, iy = int(xpos), int(ypos)
# create a little tractor Image object around the star
slc = (slice(max(iy - stampsize, 0), min(iy + stampsize + 1, H)),
slice(max(ix - stampsize, 0), min(ix + stampsize + 1, W)))
# The PSF model 'const2Psf' is the one used in DR1: a 2-component
# Gaussian fit to PsfEx instantiated in the image center.
tim = im.get_tractor_image(slc=slc, **get_tim_kwargs)
stamp = tim.getImage()
ivarstamp = tim.getInvvar()
# Initialize a tractor PointSource from PS1 measurements
flux = NanoMaggies.magToNanomaggies(mag)
star = PointSource(RaDecPos(ra, dec), NanoMaggies(**{band: flux}))
# Fit just the source RA,Dec,flux.
tractor = Tractor([tim], [star])
tractor.freezeParam('images')
print('2-component MOG:', tim.psf)
tractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_mog = tractor.getModelImage(0)
chi2_mog = -2.0 * tractor.getLogLikelihood()
mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
# Now change the PSF model to a pixelized PSF model from PsfEx instantiated
# at this place in the image.
psf = PixelizedPsfEx(im.psffn)
tim.psf = psf.constantPsfAt(xpos, ypos)
#print('PSF model:', tim.psf)
#tractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_psfex = tractor.getModelImage(0)
chi2_psfex = -2.0 * tractor.getLogLikelihood()
mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
#mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95])
sig = np.std((stamp - model_psfex)[ivarstamp > 0])
mn, mx = [-2.0 * sig, 5 * sig]
# Generate a QAplot.
if (istar > 0) and (istar % (ncols) == 0):
irow = irow + 1
icol = 3 * istar - 3 * ncols * irow
#print(istar, irow, icol, icol+1, icol+2)
ax1 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol),
aspect='equal')
ax1.axis('off')
#ax1.imshow(stamp, **tim.ima)
ax1.imshow(stamp,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax1.text(0.1,
0.9,
'{:2d}'.format(istar + 1),
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax1.transAxes)
ax2 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 1),
aspect='equal')
ax2.axis('off')
#ax2.imshow(stamp-model_mog, **tim.ima)
ax2.imshow(stamp - model_mog,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax2.text(0.1,
0.9,
'MoG',
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax2.transAxes)
ax2.text(0.08,
0.08,
'{:.3f}'.format(mag_mog),
color='white',
horizontalalignment='left',
verticalalignment='bottom',
transform=ax2.transAxes)
#ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14)
#ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex))
ax3 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 2),
aspect='equal')
ax3.axis('off')
#ax3.imshow(stamp-model_psfex, **tim.ima)
ax3.imshow(stamp - model_psfex,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax3.text(0.1,
0.9,
'PSFEx',
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax3.transAxes)
ax3.text(0.08,
0.08,
'{:.3f}'.format(mag_psfex),
color='white',
horizontalalignment='left',
verticalalignment='bottom',
transform=ax3.transAxes)
if istar == (nstar - 1):
break
fig.savefig(pngprefix + '-stargrid.png', bbox_inches='tight')
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'--dr',
'--drdir',
dest='drdir',
default='/project/projectdirs/cosmo/data/legacysurvey/dr5',
help='Directory containing data release w/ tar-gzipped calibs')
parser.add_argument(
'-b',
'--brick',
help='Brick name to run; required unless --radec is given')
parser.add_argument(
'--radec',
nargs=2,
help='RA,Dec center for a custom location (not a brick)')
parser.add_argument('--pixscale',
type=float,
default=0.262,
help='Pixel scale of the output coadds (arcsec/pixel)')
parser.add_argument('-W',
'--width',
type=int,
default=3600,
help='Target image width, default %(default)i')
parser.add_argument('-H',
'--height',
type=int,
default=3600,
help='Target image height, default %(default)i')
parser.add_argument(
'--zoom',
type=int,
nargs=4,
help='Set target image extent (default "0 3600 0 3600")')
parser.add_argument('--no-psf',
dest='do_psf',
default=True,
action='store_false',
help='Do not extract PsfEx files')
parser.add_argument('--no-sky',
dest='do_sky',
default=True,
action='store_false',
help='Do not extract SplineSky files')
opt = parser.parse_args()
if opt.brick is None and opt.radec is None:
parser.print_help()
return -1
optdict = vars(opt)
drdir = opt.drdir
W = opt.width
H = opt.height
pixscale = opt.pixscale
target_extent = opt.zoom
do_psf = opt.do_psf
do_sky = opt.do_sky
#brickname = '1501p020'
custom = (opt.radec is not None)
#ra,dec = 216.03, 34.86
if custom:
ra, dec = opt.radec #27.30, -10.43
ra = float(ra)
dec = float(dec)
#W,H = 1000,1000
#W,H = 1500,1500
brickname = 'custom_%.3f_%.3f' % (ra, dec)
#do_sky = False
survey = LegacySurveyData()
if custom:
from legacypipe.survey import BrickDuck
# Custom brick; create a fake 'brick' object
brick = BrickDuck(ra, dec, brickname)
else:
brickname = opt.brick
brick = survey.get_brick_by_name(brickname)
# Get WCS object describing brick
targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
if target_extent is not None:
(x0, x1, y0, y1) = target_extent
W = x1 - x0
H = y1 - y0
targetwcs = targetwcs.get_subimage(x0, y0, W, H)
# Find CCDs
ccds = survey.ccds_touching_wcs(targetwcs, ccdrad=None)
if ccds is None:
raise NothingToDoError('No CCDs touching brick')
print(len(ccds), 'CCDs touching target WCS')
for ccd in ccds:
im = survey.get_image_object(ccd)
print('CCD', im)
expnum = '%08i' % im.expnum
if do_psf:
if os.path.exists(im.psffn) or os.path.exists(im.merged_psffn):
print('PSF file exists')
else:
print('Need PSF', im.psffn, im.merged_psffn)
tarfn = os.path.join(
drdir, 'calib', im.camera, 'psfex-merged',
'legacysurvey_dr5_calib_decam_psfex-merged_%s.tar.gz' %
expnum[:5])
print(tarfn)
if os.path.exists(tarfn):
outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum)
cmd = 'cd %s/%s/psfex-merged && tar xvzf %s %s' % (
survey.get_calib_dir(), im.camera, tarfn, outfn)
print(cmd)
os.system(cmd)
if do_sky:
if os.path.exists(im.splineskyfn) or os.path.exists(
im.merged_splineskyfn):
print('Sky file exists')
else:
print('Need sky', im.splineskyfn, im.merged_splineskyfn)
tarfn = os.path.join(
drdir, 'calib', im.camera, 'splinesky-merged',
'legacysurvey_dr5_calib_decam_splinesky-merged_%s.tar.gz' %
expnum[:5])
print(tarfn)
if os.path.exists(tarfn):
outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum)
cmd = 'cd %s/%s/splinesky-merged && tar xvzf %s %s' % (
survey.get_calib_dir(), im.camera, tarfn, outfn)
print(cmd)
os.system(cmd)
from legacyanalysis.ps1cat import ps1cat
from legacypipe.survey import LegacySurveyData
from tractor import Image, PointSource, PixPos, NanoMaggies, Tractor
ps = PlotSequence('rewcs')
expnum, ccdname = 431109, 'N14'
cat = ps1cat(expnum=expnum, ccdname=ccdname)
stars = cat.get_stars()
print len(stars), 'stars'
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum,ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
wcs = im.get_wcs()
tim = im.get_tractor_image(pixPsf=True, splinesky=True)
margin = 15
ok,stars.xx,stars.yy = wcs.radec2pixelxy(stars.ra, stars.dec)
stars.xx -= 1.
stars.yy -= 1.
W,H = wcs.get_width(), wcs.get_height()
stars.ix = np.round(stars.xx).astype(int)
stars.iy = np.round(stars.yy).astype(int)
stars.cut((stars.ix >= margin) * (stars.ix < (W-margin)) *
(stars.iy >= margin) * (stars.iy < (H-margin)))
plt.clf()
plt.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.95,
def rbmain():
from legacypipe.catalog import read_fits_catalog
from legacypipe.survey import LegacySurveyData, wcs_for_brick
from tractor.galaxy import DevGalaxy
from tractor import PointSource, Catalog
from tractor import GaussianMixturePSF
from legacypipe.survey import BrickDuck
from legacypipe.forced_photom import main as forced_main
from astrometry.util.file import trymakedirs
import shutil
ceres = 'ceres' in sys.argv
psfex = 'psfex' in sys.argv
for v in [
'UNWISE_COADDS_TIMERESOLVED_DIR', 'SKY_TEMPLATE_DIR',
'LARGEGALAXIES_CAT', 'GAIA_CAT_DIR', 'TYCHO2_KD_DIR'
]:
if v in os.environ:
del os.environ[v]
oldargs = sys.argv
sys.argv = [sys.argv[0]]
main()
sys.argv = oldargs
# Test create_kdtree and (reading CCD kd-tree)!
indir = os.path.join(os.path.dirname(__file__), 'testcase6')
with tempfile.TemporaryDirectory() as surveydir:
files = [
'calib', 'gaia', 'images', 'survey-bricks.fits.gz',
'tycho2.kd.fits'
]
for fn in files:
src = os.path.join(indir, fn)
dst = os.path.join(surveydir, fn)
#trymakedirs(dst, dir=True)
print('Copy', src, dst)
if os.path.isfile(src):
shutil.copy(src, dst)
else:
shutil.copytree(src, dst)
from legacypipe.create_kdtrees import create_kdtree
infn = os.path.join(indir, 'survey-ccds-1.fits.gz')
outfn = os.path.join(surveydir, 'survey-ccds-1.kd.fits')
create_kdtree(infn, outfn, False)
os.environ['TYCHO2_KD_DIR'] = surveydir
outdir = 'out-testcase6-kd'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--outdir', outdir
])
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0].strip() == 'PSF')
assert (T.type[1].strip() == 'PSF')
# There is a Tycho-2 star in the blob.
I = np.flatnonzero(T.ref_cat == 'T2')
assert (len(I) == 1)
assert (T.ref_id[I][0] == 1909016711)
cat = read_fits_catalog(T)
assert (len(cat) == 2)
assert (isinstance(cat[0], PointSource))
assert (isinstance(cat[1], PointSource))
cat, ivs = read_fits_catalog(T, invvars=True)
assert (len(cat) == 2)
assert (isinstance(cat[0], PointSource))
assert (isinstance(cat[1], PointSource))
cat2 = Catalog(*cat)
assert (len(ivs) == len(cat2.getParams()))
# test --fit-on-coadds
outdir = 'out-testcase6-coadds'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--fit-on-coadds', '--outdir', outdir
])
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0].strip() == 'PSF')
assert (T.type[1].strip() == 'PSF')
# There is a Tycho-2 star in the blob.
I = np.flatnonzero(T.ref_cat == 'T2')
assert (len(I) == 1)
assert (T.ref_id[I][0] == 1909016711)
del os.environ['TYCHO2_KD_DIR']
# test --skip-coadds
r = main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--outdir', outdir, '--skip-coadd'
])
assert (r == 0)
# test --skip
r = main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--outdir', outdir, '--skip'
])
assert (r == 0)
# NothingToDoError (neighbouring brick)
r = main(args=[
'--brick', '1102p240', '--zoom', '0', '100', '0', '100',
'--force-all', '--no-write', '--no-wise', '--no-gaia',
'--survey-dir', surveydir, '--outdir', outdir
])
assert (r == 0)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase9')
# Test for some get_tractor_image kwargs
survey = LegacySurveyData(surveydir)
fakebrick = BrickDuck(9.1228, 3.3975, 'quack')
wcs = wcs_for_brick(fakebrick, W=100, H=100)
ccds = survey.ccds_touching_wcs(wcs)
ccd = ccds[0]
im = survey.get_image_object(ccd)
H, W = wcs.shape
targetrd = np.array([
wcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
tim = im.get_tractor_image(radecpoly=targetrd)
assert (tim.getImage() is not None)
assert (tim.getInvError() is not None)
assert (tim.dq is not None)
tim2 = im.get_tractor_image(radecpoly=targetrd, pixels=False)
assert (np.all(tim2.getImage() == 0.))
tim4 = im.get_tractor_image(radecpoly=targetrd, invvar=False)
u = np.unique(tim4.inverr)
assert (len(u) == 1)
u = u[0]
target = tim4.zpscale / tim4.sig1
assert (np.abs(u / target - 1.) < 0.001)
tim3 = im.get_tractor_image(radecpoly=targetrd, invvar=False, dq=False)
assert (not hasattr(tim3, 'dq'))
tim5 = im.get_tractor_image(radecpoly=targetrd, gaussPsf=True)
print(tim5.getPsf())
assert (isinstance(tim5.getPsf(), GaussianMixturePSF))
surveydir = os.path.join(os.path.dirname(__file__), 'testcase12')
os.environ['TYCHO2_KD_DIR'] = surveydir
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
os.environ['UNWISE_MODEL_SKY_DIR'] = os.path.join(surveydir, 'images',
'unwise-mod')
#python legacypipe/runbrick.py --radec --width 100 --height 100 --outdir dup5b --survey-dir test/testcase12 --force-all --no-wise
unwdir = os.path.join(surveydir, 'images', 'unwise')
main(args=[
'--radec', '346.684', '12.791', '--width', '100', '--height', '100',
'--no-wise-ceres', '--unwise-dir', unwdir, '--survey-dir', surveydir,
'--outdir', 'out-testcase12', '--skip-coadd', '--force-all'
])
# --plots for stage_wise_forced
main(args=[
'--radec', '346.684', '12.791', '--width', '100', '--height', '100',
'--no-wise-ceres', '--unwise-dir', unwdir, '--survey-dir', surveydir,
'--outdir', 'out-testcase12', '--stage', 'wise_forced', '--plots'
])
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
del os.environ['TYCHO2_KD_DIR']
del os.environ['UNWISE_MODEL_SKY_DIR']
M = fitsio.read(
'out-testcase12/coadd/cus/custom-346684p12791/legacysurvey-custom-346684p12791-maskbits.fits.fz'
)
# Count masked & unmasked bits (the cluster splits this 100x100 field)
from collections import Counter
c = Counter(M.ravel())
from legacypipe.bits import MASKBITS
assert (c[0] >= 4000)
assert (c[MASKBITS['CLUSTER']] >= 4000)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase9')
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
os.environ['LARGEGALAXIES_CAT'] = os.path.join(surveydir,
'sga-sub.kd.fits')
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
surveydir, '--outdir', 'out-testcase9', '--skip', '--force-all',
'--ps', 'tc9-ps.fits', '--ps-t0',
str(int(time.time()))
])
# (omit --force-all --no-write... reading from pickles below!)
# Test with --apodize
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--old-calibs-ok', '--no-wise', '--force-all', '--no-write',
'--survey-dir', surveydir, '--outdir', 'out-testcase9-ap', '--apodize'
])
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
surveydir, '--outdir', 'out-testcase9', '--plots', '--stage', 'halos'
])
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
surveydir, '--outdir', 'out-testcase9-coadds', '--stage',
'image_coadds', '--blob-image'
])
T = fits_table(
'out-testcase9/tractor/cus/tractor-custom-009122p03397.fits')
assert (len(T) == 4)
# Gaia star becomes a DUP!
assert (np.sum([t == 'DUP' for t in T.type]) == 1)
# LSLGA galaxy exists!
Igal = np.flatnonzero([r == 'L3' for r in T.ref_cat])
assert (len(Igal) == 1)
assert (np.all(T.ref_id[Igal] > 0))
assert (T.type[Igal[0]] == 'SER')
# --brick and --zoom rather than --radec --width --height
main(args=[
'--survey-dir', surveydir, '--outdir', 'out-testcase9b', '--zoom',
'1950', '2050', '340', '440', '--brick', '0091p035', '--force-all'
])
# test forced phot??
shutil.copy('test/testcase9/survey-bricks.fits.gz', 'out-testcase9b')
forced_main(args=[
'--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
'out-testcase9b', '372546', 'N26', 'forced1.fits'
])
assert (os.path.exists('forced1.fits'))
_ = fits_table('forced1.fits')
# ... more tests...!
forced_main(args=[
'--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
'out-testcase9b', '--derivs', '--threads', '2', '--apphot', '372547',
'N26', 'forced2.fits'
])
assert (os.path.exists('forced2.fits'))
_ = fits_table('forced2.fits')
forced_main(args=[
'--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
'out-testcase9b', '--agn', '257266', 'S21', 'forced3.fits'
])
assert (os.path.exists('forced3.fits'))
_ = fits_table('forced3.fits')
if ceres:
forced_main(args=[
'--survey-dir', surveydir, '--catalog-dir', 'out-testcase9b',
'--derivs', '--threads', '2', '--apphot', '372546', 'N26',
'forced4.fits'
])
assert (os.path.exists('forced4.fits'))
_ = fits_table('forced4.fits')
# Test cache_dir
with tempfile.TemporaryDirectory() as cachedir, \
tempfile.TemporaryDirectory() as tempsurveydir:
files = []
for dirpath, _, filenames in os.walk(surveydir):
for fn in filenames:
path = os.path.join(dirpath, fn)
relpath = os.path.relpath(path, surveydir)
files.append(relpath)
# cache or no?
files.sort()
files_cache = files[::2]
files_nocache = files[1::2]
# Survey-ccds *must* be in nocache.
fn = 'survey-ccds-1.kd.fits'
if fn in files_cache:
files_cache.remove(fn)
files_nocache.append(fn)
for fn in files_cache:
src = os.path.join(surveydir, fn)
dst = os.path.join(cachedir, fn)
trymakedirs(dst, dir=True)
print('Copy', src, dst)
shutil.copy(src, dst)
for fn in files_nocache:
src = os.path.join(surveydir, fn)
dst = os.path.join(tempsurveydir, fn)
trymakedirs(dst, dir=True)
print('Copy', src, dst)
shutil.copy(src, dst)
main(args=[
'--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
'--no-wise', '--survey-dir', tempsurveydir, '--cache-dir',
cachedir, '--outdir', 'out-testcase9cache', '--force-all'
])
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
del os.environ['LARGEGALAXIES_CAT']
# if ceres:
# surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
# main(args=['--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
# '--no-wise', '--force-all', '--no-write', '--ceres',
# '--survey-dir', surveydir,
# '--outdir', 'out-testcase3-ceres',
# '--no-depth-cut'])
# MzLS + BASS data
# python legacypipe/runbrick.py --run north --brick 1773p595 --zoom 1300 1500 700 900 --survey-dir dr9-north -s coadds
# fitscopy coadd/177/1773p595/legacysurvey-1773p595-ccds.fits"[#row<3 || #row==12]" cx.fits
# python legacyanalysis/create_testcase.py cx.fits test/mzlsbass2 1773p595 --survey-dir dr9-north/ --fpack
surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass2')
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir2, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir2,
'--outdir', 'out-mzlsbass2'
])
T = fits_table('out-mzlsbass2/tractor/177/tractor-1773p595.fits')
assert (np.sum(T.ref_cat == 'G2') == 3)
assert (np.sum(T.ref_id > 0) == 3)
# Test --max-blobsize, --checkpoint, --bail-out
outdir = 'out-mzlsbass2b'
chk = 'checkpoint-mzb2b.p'
if os.path.exists(chk):
os.unlink(chk)
main(args=[
'--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
'--no-wise', '--force-all', '--stage', 'fitblobs', '--write-stage',
'srcs', '--survey-dir', surveydir2, '--outdir', outdir, '--checkpoint',
chk, '--nblobs', '3'
])
# err... --max-blobsize does not result in bailed-out blobs masked,
# because it treats large blobs as *completed*...
#'--max-blobsize', '3000',
outdir = 'out-mzlsbass2c'
main(args=[
'--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
'--no-wise', '--force-all', '--survey-dir', surveydir2, '--outdir',
outdir, '--bail-out', '--checkpoint', chk, '--no-write'
])
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
M = fitsio.read(
os.path.join(outdir, 'coadd', '177', '1773p595',
'legacysurvey-1773p595-maskbits.fits.fz'))
assert (np.sum((M & MASKBITS['BAILOUT']) > 0) >= 1000)
# Test RexGalaxy
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6-rex'
the_args = [
'--brick',
'1102p240',
'--zoom',
'500',
'600',
'650',
'750',
'--force-all',
'--no-write',
'--no-wise',
'--skip-calibs',
#'--rex', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
]
print('python legacypipe/runbrick.py', ' '.join(the_args))
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
main(args=the_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0].strip() == 'PSF')
cmd = (
'(cd %s && sha256sum -c %s)' %
(outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
print(cmd)
rtn = os.system(cmd)
assert (rtn == 0)
# Test with a Tycho-2 star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir,
'--outdir', outdir
])
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0].strip() == 'PSF')
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
# Test that we can run splinesky calib if required...
from legacypipe.decam import DecamImage
DecamImage.splinesky_boxsize = 128
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
survey = LegacySurveyData(surveydir)
# get brick by id
brickid = 473357
brick = survey.get_brick(brickid)
assert (brick.brickname == '1867p255')
assert (brick.brickid == brickid)
outdir = 'out-testcase4'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
fn = os.path.join(surveydir, 'calib', 'sky-single', 'decam', 'CP',
'V4.8.2', 'CP20170315', 'c4d_170316_062107_ooi_z_ls9',
'c4d_170316_062107_ooi_z_ls9-N2-splinesky.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images',
'unwise-tr'), '--blob-image', '--no-hybrid-psf',
'--survey-dir', surveydir, '--outdir', outdir, '-v', '--no-wise-ceres'
])
print('Checking for calib file', fn)
assert (os.path.exists(fn))
# Test with blob-masking when creating sky calib.
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--blob-mask-dir',
surveydir, '--survey-dir', surveydir, '--stage', 'image_coadds',
'--outdir', 'out-testcase4b', '--plots'
])
print('Checking for calib file', fn)
assert (os.path.exists(fn))
if ceres:
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images', 'unwise-tr'), '--survey-dir',
surveydir, '--outdir', outdir
])
if psfex:
# Check that we can regenerate PsfEx files if necessary.
fn = os.path.join(surveydir, 'calib', 'psfex', 'decam', 'CP', 'V4.8.2',
'CP20170315',
'c4d_170316_062107_ooi_z_ls9-psfex.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image',
'--survey-dir', surveydir, '--outdir', outdir, '-v'
])
print('After generating PsfEx calib:')
os.system('find %s' % (os.path.join(surveydir, 'calib')))
# Wrap-around, hybrid PSF
surveydir = os.path.join(os.path.dirname(__file__), 'testcase8')
outdir = 'out-testcase8'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick',
'1209p050',
'--zoom',
'720',
'1095',
'3220',
'3500',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
])
# Test with a Tycho-2 star + another saturated star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase7')
outdir = 'out-testcase7'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick',
'1102p240',
'--zoom',
'250',
'350',
'1550',
'1650',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
])
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 4)
# Check skipping blobs outside the brick's unique area.
# (this now doesn't detect any sources at all, reasonably)
# surveydir = os.path.join(os.path.dirname(__file__), 'testcase5')
# outdir = 'out-testcase5'
#
# fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits')
# if os.path.exists(fn):
# os.unlink(fn)
#
# main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150',
# '--force-all', '--no-write', '--coadd-bw',
# '--survey-dir', surveydir,
# '--early-coadds',
# '--outdir', outdir] + extra_args)
#
# assert(os.path.exists(fn))
# T = fits_table(fn)
# assert(len(T) == 1)
# Custom RA,Dec; blob ra,dec.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4b'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
# Catalog written with one entry (--blobradec)
fn = os.path.join(outdir, 'tractor', 'cus',
'tractor-custom-186743p25461.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--radec', '186.743965', '25.461788', '--width', '250', '--height',
'250', '--force-all', '--no-write', '--no-wise', '--blobradec',
'186.740369', '25.453855', '--survey-dir', surveydir, '--outdir',
outdir
])
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 1)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
outdir = 'out-testcase3'
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
os.environ['GAIA_CAT_VER'] = '2'
checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle')
if os.path.exists(checkpoint_fn):
os.unlink(checkpoint_fn)
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
])
# Read catalog into Tractor sources to test read_fits_catalog
survey = LegacySurveyData(survey_dir=outdir)
fn = survey.find_file('tractor', brick='2447p120')
print('Checking', fn)
T = fits_table(fn)
cat = read_fits_catalog(T)
print('Read catalog:', cat)
assert (len(cat) == 2)
src = cat[1]
print('Source0', src)
from tractor.sersic import SersicGalaxy
assert (type(src) in [DevGalaxy, SersicGalaxy])
assert (np.abs(src.pos.ra - 244.77973) < 0.00001)
assert (np.abs(src.pos.dec - 12.07234) < 0.00002)
src = cat[0]
print('Source1', src)
assert (type(src) == PointSource)
assert (np.abs(src.pos.ra - 244.77828) < 0.00001)
assert (np.abs(src.pos.dec - 12.07250) < 0.00001)
# Check that we can run again, using that checkpoint file.
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
])
# Assert...... something?
# Test --checkpoint without --threads
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1'
])
# From Kaylan's Bootes pre-DR4 run
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3')
# main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass3'] + extra_args)
# With plots!
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', 'out-testcase3', '--plots', '--nblobs', '1'
])
def main(survey=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if survey is None:
survey = LegacySurveyData()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(np.array([f.strip() == opt.filename for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert(len(T) == 1)
ccd = T[0]
im = survey.get_image_object(ccd)
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True,
constant_invvar=opt.constant_invvar)
print('Got tim:', tim)
print('Read image:', Time()-t0)
if opt.catfn in ['DR1', 'DR2', 'DR3']:
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=survey)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = survey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
if len(T):
TT.append(T)
if len(TT) == 0:
print('No sources to photometer.')
return 0
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
surveydir = survey.get_survey_dir()
del survey
cat = read_fits_catalog(T)
# print('Got cat:', cat)
print('Read catalog:', Time()-t0)
print('Forced photom...')
opti = None
forced_kwargs = {}
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
#forced_kwargs.update(verbose=True)
for src in cat:
# Limit sizes of huge models
from tractor.galaxy import ProfileGalaxy
if isinstance(src, ProfileGalaxy):
px,py = tim.wcs.positionToPixel(src.getPosition())
h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval)
MAXHALF = 128
if h > MAXHALF:
print('halfsize', h,'for',src,'-> setting to',MAXHALF)
src.halfsize = MAXHALF
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
disable_galaxy_cache()
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32)
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
if opt.forced:
if opt.plots is None:
forced_kwargs.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True, fitstats=True,
shared_params=False, priors=False, **forced_kwargs)
if opt.plots:
(data,mod,ie,chi,roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([src.getBrightness().getFlux(tim.band)
for src in cat]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2 .astype(np.float32)
print('Forced photom:', Time()-t0)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap.astype(np.float32)
ap = 1./(np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap.astype(np.float32)
print('Aperture photom:', Time()-t0)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, surveydir)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE',
value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
comment='Name of this image'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'}
columns = F.get_columns()
for i,col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
print('Getting model image...')
mod = tr.getModelImage(tim)
fitsio.write(opt.save_model, mod, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
print('Finished forced phot:', Time()-t0)
return 0
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--catalog', help='Catalog to render')
parser.add_argument('--brick-coadd', help='Produce a coadd of the images overlapping the given brickname.')
parser.add_argument('--ccds', help='Use this table of CCDs')
parser.add_argument('--brick-wcs', help='File containing a WCS header describing the coadd WCS to render.')
parser.add_argument('--brick-wcs-ext', type=int, help='FITS file extension containing a WCS header describing the coadd WCS to render.')
parser.add_argument('--outlier-mask-brick', help='Comma-separated list of bricknames from which outlier masks should be read.')
parser.add_argument('--out', help='Filename pattern ("BAND" will be replaced by band name) of output images.')
parser.add_argument('--resid', help='Filename pattern ("BAND" will be replaced by band name) of residual images.')
parser.add_argument('--jpeg', help='Write RGB image to this filename')
parser.add_argument('--resid-jpeg', help='Write RGB residual image to this filename')
opt = parser.parse_args()
if opt.catalog is None:
print('Need catalog!')
return -1
cat = fits_table(opt.catalog)
if opt.ccds is None:
if opt.brick_coadd is None:
print('Need brick catalog!')
return -1
brickname = opt.brick_coadd
survey = LegacySurveyData()
if opt.brick_wcs is None:
print('FIXME')
return -1
else:
wcs = Tan(opt.brick_wcs, opt.brick_wcs_ext)
tcat = read_fits_catalog(cat)
if opt.ccds:
ccdfn = opt.ccds
else:
ccdfn = survey.find_file('ccds-table', brick=brickname)
print('Reading', ccdfn)
ccds = fits_table(ccdfn)
H,W = wcs.shape
targetrd = np.array([wcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
tims = []
for ccd in ccds:
im = survey.get_image_object(ccd)
#slc = slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1)
#tim = im.get_tractor_image(slc=slc)
tim = im.get_tractor_image(radecpoly=targetrd)
print('Read', tim)
tims.append(tim)
if opt.outlier_mask_brick is not None:
bricks = opt.outlier_mask_brick.split(',')
for b in bricks:
print('Reading outlier mask for brick', b,
':', survey.find_file('outliers_mask', brick=b, output=False))
ok = read_outlier_mask_file(survey, tims, b,
subimage=True, output=False)
tr = Tractor(tims, tcat)
mods = list(tr.getModelImages())
bands = 'grz'
def write_model(band, cowimg=None, cowmod=None, **kwargs):
if cowmod is None:
print('No model for', band)
return
outfn = opt.out.replace('BAND', band)
fitsio.write(outfn, cowmod, clobber=True)
print('Wrote model for', band, 'to', outfn)
if opt.resid:
outfn = opt.resid.replace('BAND', band)
fitsio.write(outfn, cowimg - cowmod, clobber=True)
print('Wrote resid for', band, 'to', outfn)
C = make_coadds(tims, bands, wcs, mods=mods,
callback=write_model)
if opt.jpeg:
from legacypipe.survey import get_rgb
import pylab as plt
plt.imsave(opt.jpeg, get_rgb(C.comods, bands), origin='lower')
if opt.resid_jpeg:
from legacypipe.survey import get_rgb
import pylab as plt
plt.imsave(opt.resid_jpeg,
get_rgb([im-mod for im,mod in zip(C.coimgs, C.comods)], bands),
origin='lower')
def main(outfn='ccds-annotated.fits', ccds=None):
survey = LegacySurveyData(ccds=ccds)
if ccds is None:
ccds = survey.get_ccds()
# File from the "observing" svn repo:
# https://desi.lbl.gov/svn/decam/code/observing/trunk
tiles = fits_table('decam-tiles_obstatus.fits')
I = survey.photometric_ccds(ccds)
ccds.photometric = np.zeros(len(ccds), bool)
ccds.photometric[I] = True
I = survey.apply_blacklist(ccds)
ccds.blacklist_ok = np.zeros(len(ccds), bool)
ccds.blacklist_ok[I] = True
ccds.good_region = np.empty((len(ccds), 4), np.int16)
ccds.good_region[:,:] = -1
ccds.ra0 = np.zeros(len(ccds), np.float64)
ccds.dec0 = np.zeros(len(ccds), np.float64)
ccds.ra1 = np.zeros(len(ccds), np.float64)
ccds.dec1 = np.zeros(len(ccds), np.float64)
ccds.ra2 = np.zeros(len(ccds), np.float64)
ccds.dec2 = np.zeros(len(ccds), np.float64)
ccds.ra3 = np.zeros(len(ccds), np.float64)
ccds.dec3 = np.zeros(len(ccds), np.float64)
ccds.dra = np.zeros(len(ccds), np.float32)
ccds.ddec = np.zeros(len(ccds), np.float32)
ccds.ra_center = np.zeros(len(ccds), np.float64)
ccds.dec_center = np.zeros(len(ccds), np.float64)
ccds.sig1 = np.zeros(len(ccds), np.float32)
ccds.meansky = np.zeros(len(ccds), np.float32)
ccds.stdsky = np.zeros(len(ccds), np.float32)
ccds.maxsky = np.zeros(len(ccds), np.float32)
ccds.minsky = np.zeros(len(ccds), np.float32)
ccds.pixscale_mean = np.zeros(len(ccds), np.float32)
ccds.pixscale_std = np.zeros(len(ccds), np.float32)
ccds.pixscale_max = np.zeros(len(ccds), np.float32)
ccds.pixscale_min = np.zeros(len(ccds), np.float32)
ccds.psfnorm_mean = np.zeros(len(ccds), np.float32)
ccds.psfnorm_std = np.zeros(len(ccds), np.float32)
ccds.galnorm_mean = np.zeros(len(ccds), np.float32)
ccds.galnorm_std = np.zeros(len(ccds), np.float32)
gaussgalnorm = np.zeros(len(ccds), np.float32)
# 2nd moments
ccds.psf_mx2 = np.zeros(len(ccds), np.float32)
ccds.psf_my2 = np.zeros(len(ccds), np.float32)
ccds.psf_mxy = np.zeros(len(ccds), np.float32)
#
ccds.psf_a = np.zeros(len(ccds), np.float32)
ccds.psf_b = np.zeros(len(ccds), np.float32)
ccds.psf_theta = np.zeros(len(ccds), np.float32)
ccds.psf_ell = np.zeros(len(ccds), np.float32)
ccds.humidity = np.zeros(len(ccds), np.float32)
ccds.outtemp = np.zeros(len(ccds), np.float32)
ccds.tileid = np.zeros(len(ccds), np.int32)
ccds.tilepass = np.zeros(len(ccds), np.uint8)
ccds.tileebv = np.zeros(len(ccds), np.float32)
plvers = []
for iccd,ccd in enumerate(ccds):
im = survey.get_image_object(ccd)
print('Reading CCD %i of %i:' % (iccd+1, len(ccds)), im)
X = im.get_good_image_subregion()
for i,x in enumerate(X):
if x is not None:
ccds.good_region[iccd,i] = x
W,H = ccd.width, ccd.height
psf = None
wcs = None
sky = None
try:
tim = im.get_tractor_image(pixPsf=True, splinesky=True, subsky=False,
pixels=False, dq=False, invvar=False)
except:
import traceback
traceback.print_exc()
plvers.append('')
continue
if tim is None:
plvers.append('')
continue
psf = tim.psf
wcs = tim.wcs.wcs
sky = tim.sky
hdr = tim.primhdr
# print('Got PSF', psf)
# print('Got sky', type(sky))
# print('Got WCS', wcs)
ccds.humidity[iccd] = hdr.get('HUMIDITY')
ccds.outtemp[iccd] = hdr.get('OUTTEMP')
ccds.sig1[iccd] = tim.sig1
plvers.append(tim.plver)
# parse 'DECaLS_15150_r' to get tile number
obj = ccd.object.strip()
words = obj.split('_')
tile = None
if len(words) == 3 and words[0] == 'DECaLS':
try:
tileid = int(words[1])
tile = tiles[tileid - 1]
if tile.tileid != tileid:
I = np.flatnonzero(tile.tileid == tileid)
tile = tiles[I[0]]
except:
pass
if tile is not None:
ccds.tileid [iccd] = tile.tileid
ccds.tilepass[iccd] = tile.get('pass')
ccds.tileebv [iccd] = tile.ebv_med
# Instantiate PSF on a grid
S = 32
xx = np.linspace(1+S, W-S, 5)
yy = np.linspace(1+S, H-S, 5)
xx,yy = np.meshgrid(xx, yy)
psfnorms = []
galnorms = []
for x,y in zip(xx.ravel(), yy.ravel()):
p = im.psf_norm(tim, x=x, y=y)
g = im.galaxy_norm(tim, x=x, y=y)
psfnorms.append(p)
galnorms.append(g)
ccds.psfnorm_mean[iccd] = np.mean(psfnorms)
ccds.psfnorm_std [iccd] = np.std (psfnorms)
ccds.galnorm_mean[iccd] = np.mean(galnorms)
ccds.galnorm_std [iccd] = np.std (galnorms)
# PSF in center of field
cx,cy = (W+1)/2., (H+1)/2.
p = psf.getPointSourcePatch(cx, cy).patch
ph,pw = p.shape
px,py = np.meshgrid(np.arange(pw), np.arange(ph))
psum = np.sum(p)
# print('psum', psum)
p /= psum
# centroids
cenx = np.sum(p * px)
ceny = np.sum(p * py)
# print('cenx,ceny', cenx,ceny)
# second moments
x2 = np.sum(p * (px - cenx)**2)
y2 = np.sum(p * (py - ceny)**2)
xy = np.sum(p * (px - cenx)*(py - ceny))
# semi-major/minor axes and position angle
theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.)
theta = np.abs(theta) * np.sign(xy)
s = np.sqrt(((x2 - y2)/2.)**2 + xy**2)
a = np.sqrt((x2 + y2) / 2. + s)
b = np.sqrt((x2 + y2) / 2. - s)
ell = 1. - b/a
# print('PSF second moments', x2, y2, xy)
# print('PSF position angle', theta)
# print('PSF semi-axes', a, b)
# print('PSF ellipticity', ell)
ccds.psf_mx2[iccd] = x2
ccds.psf_my2[iccd] = y2
ccds.psf_mxy[iccd] = xy
ccds.psf_a[iccd] = a
ccds.psf_b[iccd] = b
ccds.psf_theta[iccd] = theta
ccds.psf_ell [iccd] = ell
print('Computing Gaussian approximate PSF quantities...')
# Galaxy norm using Gaussian approximation of PSF.
realpsf = tim.psf
tim.psf = im.read_psf_model(0, 0, gaussPsf=True,
psf_sigma=tim.psf_sigma)
gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy)
tim.psf = realpsf
# Sky -- evaluate on a grid (every ~10th pixel)
skygrid = sky.evaluateGrid(np.linspace(0, ccd.width-1, int(1+ccd.width/10)),
np.linspace(0, ccd.height-1, int(1+ccd.height/10)))
ccds.meansky[iccd] = np.mean(skygrid)
ccds.stdsky[iccd] = np.std(skygrid)
ccds.maxsky[iccd] = skygrid.max()
ccds.minsky[iccd] = skygrid.min()
# WCS
ccds.ra0[iccd],ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1)
ccds.ra1[iccd],ccds.dec1[iccd] = wcs.pixelxy2radec(1, H)
ccds.ra2[iccd],ccds.dec2[iccd] = wcs.pixelxy2radec(W, H)
ccds.ra3[iccd],ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1)
midx, midy = (W+1)/2., (H+1)/2.
rc,dc = wcs.pixelxy2radec(midx, midy)
ra,dec = wcs.pixelxy2radec([1,W,midx,midx], [midy,midy,1,H])
ccds.dra [iccd] = max(degrees_between(ra, dc+np.zeros_like(ra),
rc, dc))
ccds.ddec[iccd] = max(degrees_between(rc+np.zeros_like(dec), dec,
rc, dc))
ccds.ra_center [iccd] = rc
ccds.dec_center[iccd] = dc
# Compute scale change across the chip
# how many pixels to step
step = 10
xx = np.linspace(1+step, W-step, 5)
yy = np.linspace(1+step, H-step, 5)
xx,yy = np.meshgrid(xx, yy)
pixscale = []
for x,y in zip(xx.ravel(), yy.ravel()):
sx = [x-step, x-step, x+step, x+step, x-step]
sy = [y-step, y+step, y+step, y-step, y-step]
sr,sd = wcs.pixelxy2radec(sx, sy)
rc,dc = wcs.pixelxy2radec(x, y)
# project around a tiny little TAN WCS at (x,y), with 1" pixels
locwcs = Tan(rc, dc, 0., 0., 1./3600, 0., 0., 1./3600, 1., 1.)
ok,lx,ly = locwcs.radec2pixelxy(sr, sd)
#print('local x,y:', lx, ly)
A = polygon_area((lx, ly))
pixscale.append(np.sqrt(A / (2*step)**2))
# print('Pixel scales:', pixscale)
ccds.pixscale_mean[iccd] = np.mean(pixscale)
ccds.pixscale_min[iccd] = min(pixscale)
ccds.pixscale_max[iccd] = max(pixscale)
ccds.pixscale_std[iccd] = np.std(pixscale)
ccds.plver = np.array(plvers)
sfd = tractor.sfd.SFDMap()
allbands = 'ugrizY'
filts = ['%s %s' % ('DES', f) for f in allbands]
wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
ebv,ext = sfd.extinction(filts + wisebands, ccds.ra_center,
ccds.dec_center, get_ebv=True)
ext = ext.astype(np.float32)
ccds.ebv = ebv.astype(np.float32)
ccds.decam_extinction = ext[:,:len(allbands)]
ccds.wise_extinction = ext[:,len(allbands):]
# Depth
detsig1 = ccds.sig1 / ccds.psfnorm_mean
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.psfdepth = -2.5 * (np.log10(depth) - 9)
detsig1 = ccds.sig1 / ccds.galnorm_mean
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.galdepth = -2.5 * (np.log10(depth) - 9)
# Depth using Gaussian FWHM.
psf_sigma = ccds.fwhm / 2.35
gnorm = 1./(2. * np.sqrt(np.pi) * psf_sigma)
detsig1 = ccds.sig1 / gnorm
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9)
# Gaussian galaxy depth
detsig1 = ccds.sig1 / gaussgalnorm
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9)
ccds.writeto(outfn)
def main():
import argparse
parser = argparse.ArgumentParser(
description='This script creates small self-contained data sets that '
'are useful for test cases of the pipeline codes.')
parser.add_argument('ccds', help='CCDs table describing region to grab')
parser.add_argument('outdir', help='Output directory name')
parser.add_argument('brick', help='Brick containing these images')
parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image')
parser.add_argument('--fpack', action='store_true', default=False)
parser.add_argument('--pad', action='store_true', default=False,
help='Keep original image size, but zero out pixels outside ROI')
args = parser.parse_args()
C = fits_table(args.ccds)
print(len(C), 'CCDs in', args.ccds)
C.camera = np.array([c.strip() for c in C.camera])
survey = LegacySurveyData()
bricks = survey.get_bricks_readonly()
outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])]
assert(len(outbricks) == 1)
outsurvey = LegacySurveyData(survey_dir = args.outdir)
trymakedirs(args.outdir)
outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz'))
targetwcs = wcs_for_brick(outbricks[0])
H,W = targetwcs.shape
tycho = fits_table(os.path.join(survey.get_survey_dir(), 'tycho2.fits.gz'))
print('Read', len(tycho), 'Tycho-2 stars')
ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec)
margin = 100
tycho.cut(ok * (tx > -margin) * (tx < W+margin) *
(ty > -margin) * (ty < H+margin))
print('Cut to', len(tycho), 'Tycho-2 stars within brick')
del ok,tx,ty
tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz'))
outccds = C.copy()
for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1',
'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1',
'plver', 'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver',
'psfplver' ]:
outccds.delete_column(c)
outccds.image_hdu[:] = 1
# Convert to list to avoid truncating filenames
outccds.image_filename = [fn for fn in outccds.image_filename]
for iccd,ccd in enumerate(C):
decam = (ccd.camera.strip() == 'decam')
bok = (ccd.camera.strip() == '90prime')
im = survey.get_image_object(ccd)
print('Got', im)
slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1))
tim = im.get_tractor_image(slc, pixPsf=True, splinesky=True,
subsky=False, nanomaggies=False)
print('Tim:', tim.shape)
psf = tim.getPsf()
print('PSF:', psf)
psfex = psf.psfex
print('PsfEx:', psfex)
outim = outsurvey.get_image_object(ccd)
print('Output image:', outim)
print('Image filename:', outim.imgfn)
trymakedirs(outim.imgfn, dir=True)
imgdata = tim.getImage()
dqdata = tim.dq
if decam:
# DECam-specific code remaps the DQ codes... re-read raw
print('Reading data quality from', im.dqfn, 'hdu', im.hdu)
dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice)
ivdata = tim.getInvvar()
if args.pad:
# Create zero image of full size, copy in data.
fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype)
fullsize[slc] = imgdata
imgdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype)
fullsize[slc] = dqdata
dqdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype)
fullsize[slc] = ivdata
ivdata = fullsize
else:
# Adjust the header WCS by x0,y0
crpix1 = tim.hdr['CRPIX1']
crpix2 = tim.hdr['CRPIX2']
tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0
tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0
# Add image extension to filename
# fitsio doesn't compress .fz by default, so drop .fz suffix
outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits')
# if bok:
# outim.whtfn = outim.whtfn .replace('.wht.fits', '-%s.wht.fits' % im.ccdname)
# if not args.fpack:
# outim.whtfn = outim.whtfn .replace('.fits.fz', '.fits')
# else:
if True:
outim.wtfn = outim.wtfn .replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.wtfn = outim.wtfn .replace('.fits.fz', '.fits')
if outim.dqfn is not None:
outim.dqfn = outim.dqfn .replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.dqfn = outim.dqfn .replace('.fits.fz', '.fits')
if bok:
outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname)
ccdfn = outim.imgfn
ccdfn = ccdfn.replace(outsurvey.get_image_dir(),'')
if ccdfn.startswith('/'):
ccdfn = ccdfn[1:]
outccds.image_filename[iccd] = ccdfn
print('Changed output filenames to:')
print(outim.imgfn)
print(outim.dqfn)
ofn = outim.imgfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fitsio.write(ofn, None, header=tim.primhdr, clobber=True)
fitsio.write(ofn, imgdata, header=tim.hdr, extname=ccd.ccdname)
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
h,w = tim.shape
if not args.pad:
outccds.width[iccd] = w
outccds.height[iccd] = h
outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0
outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0
wcs = Tan(*[float(x) for x in
[ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2,
ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]])
if args.pad:
subwcs = wcs
else:
subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h)
outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center()
#if not bok:
if True:
print('Weight filename:', outim.wtfn)
wfn = outim.wtfn
# else:
# print('Weight filename:', outim.whtfn)
# wfn = outim.whtfn
trymakedirs(wfn, dir=True)
ofn = wfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fitsio.write(ofn, None, header=tim.primhdr, clobber=True)
fitsio.write(ofn, ivdata, header=tim.hdr, extname=ccd.ccdname)
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
if outim.dqfn is not None:
print('DQ filename', outim.dqfn)
trymakedirs(outim.dqfn, dir=True)
ofn = outim.dqfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fitsio.write(ofn, None, header=tim.primhdr, clobber=True)
fitsio.write(ofn, dqdata, header=tim.hdr, extname=ccd.ccdname)
if args.fpack:
cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
print('PSF filename:', outim.psffn)
trymakedirs(outim.psffn, dir=True)
psfex.writeto(outim.psffn)
if not bok:
print('Sky filename:', outim.splineskyfn)
sky = tim.getSky()
print('Sky:', sky)
trymakedirs(outim.splineskyfn, dir=True)
sky.write_fits(outim.splineskyfn)
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
# WISE
if args.wise is not None:
from wise.forcedphot import unwise_tiles_touching_wcs
from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs,
get_unwise_tractor_image, get_unwise_tile_dir)
# Read WCS...
print('Reading TAN wcs header from', args.wise)
targetwcs = Tan(args.wise)
tiles = unwise_tiles_touching_wcs(targetwcs)
print('Cut to', len(tiles), 'unWISE tiles')
H,W = targetwcs.shape
r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]),
np.array([H/2, H/2, 1, H ]))
roiradec = [r[0], r[1], d[2], d[3]]
unwise_dir = os.environ['UNWISE_COADDS_DIR']
wise_out = os.path.join(args.outdir, 'images', 'unwise')
print('Will write WISE outputs to', wise_out)
unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR']
wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr')
print('Will write WISE time-resolved outputs to', wise_tr_out)
W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_neo1-atlas.fits'))
print('Read', len(W), 'time-resolved WISE coadd tiles')
W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id]))
print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth')
# Write the time-resolved index subset.
W.writeto(os.path.join(wise_tr_out, 'time_resolved_neo1-atlas.fits'))
# this ought to be enough for anyone =)
Nepochs = 5
wisedata = []
# full depth
for band in [1,2,3,4]:
wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band))
# time-resolved
for band in [1,2]:
# W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2)
bitmask = (1 << (band-1))
for e in range(Nepochs):
# Which tiles have images for this epoch?
I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask)
if len(I) == 0:
continue
print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I])
edir = os.path.join(unwise_tr_dir, 'e%03i' % e)
eoutdir = os.path.join(wise_tr_out, 'e%03i' % e)
wisedata.append((edir, eoutdir, tiles.coadd_id[I], band))
wrote_masks = set()
for indir, outdir, tiles, band in wisedata:
for tile in tiles:
wanyband = 'w'
tim = get_unwise_tractor_image(indir, tile, band,
bandname=wanyband, roiradecbox=roiradec)
print('Got unWISE tim', tim)
print(tim.shape)
thisdir = get_unwise_tile_dir(outdir, tile)
print('Directory for this WISE tile:', thisdir)
base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band))
print('Base filename:', base)
masked = True
mu = 'm' if masked else 'u'
imfn = base + 'img-%s.fits' % mu
ivfn = base + 'invvar-%s.fits.gz' % mu
nifn = base + 'n-%s.fits.gz' % mu
nufn = base + 'n-u.fits.gz'
#print('WISE image header:', tim.hdr)
# Adjust the header WCS by x0,y0
wcs = tim.wcs.wcs
tim.hdr['CRPIX1'] = wcs.crpix[0]
tim.hdr['CRPIX2'] = wcs.crpix[1]
H,W = tim.shape
tim.hdr['IMAGEW'] = W
tim.hdr['IMAGEH'] = H
print('WCS:', wcs)
print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2'])
trymakedirs(imfn, dir=True)
fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True)
print('Wrote', imfn)
fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True)
print('Wrote', ivfn)
fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True)
print('Wrote', nifn)
fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True)
print('Wrote', nufn)
if not (indir,tile) in wrote_masks:
print('Looking for mask file for', indir, tile)
# record that we tried this dir/tile combo
wrote_masks.add((indir,tile))
for idir in indir.split(':'):
tdir = get_unwise_tile_dir(idir, tile)
maskfn = 'unwise-%s-msk.fits.gz' % tile
fn = os.path.join(tdir, maskfn)
print('Mask file:', fn)
if os.path.exists(fn):
print('Reading', fn)
(x0,x1,y0,y1) = tim.roi
roislice = (slice(y0,y1), slice(x0,x1))
F = fitsio.FITS(fn)[0]
hdr = F.read_header()
M = F[roislice]
outfn = os.path.join(thisdir, maskfn)
fitsio.write(outfn, M, header=tim.hdr, clobber=True)
print('Wrote', outfn)
break
outC = outsurvey.get_ccds_readonly()
for iccd,ccd in enumerate(outC):
outim = outsurvey.get_image_object(ccd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True, splinesky=True)
print('Got output tim:', otim)
def main():
import argparse
parser = argparse.ArgumentParser(
description='This script creates small self-contained data sets that '
'are useful for test cases of the pipeline codes.')
parser.add_argument('ccds', help='CCDs table describing region to grab')
parser.add_argument('outdir', help='Output directory name')
parser.add_argument('brick', help='Brick containing these images')
parser.add_argument('--survey-dir', type=str, default=None)
parser.add_argument('--cache-dir', type=str, default=None,
help='Directory to search for cached files')
parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image')
parser.add_argument('--wise-wcs-hdu', help='For WISE outputs, the HDU to read the WCS from in the file given by --wise.', type=int, default=0)
parser.add_argument('--fpack', action='store_true', default=False)
parser.add_argument('--gzip', action='store_true', default=False)
parser.add_argument('--pad', action='store_true', default=False,
help='Keep original image size, but zero out pixels outside ROI')
args = parser.parse_args()
v = 'SKY_TEMPLATE_DIR'
if v in os.environ:
del os.environ[v]
C = fits_table(args.ccds)
print(len(C), 'CCDs in', args.ccds)
C.camera = np.array([c.strip() for c in C.camera])
survey = LegacySurveyData(cache_dir=args.cache_dir, survey_dir=args.survey_dir)
if ',' in args.brick:
ra,dec = args.brick.split(',')
ra = float(ra)
dec = float(dec)
fakebricks = fits_table()
fakebricks.brickname = np.array([('custom-%06i%s%05i' %
(int(1000*ra), 'm' if dec < 0 else 'p',
int(1000*np.abs(dec))))])
fakebricks.ra = np.array([ra])
fakebricks.dec = np.array([dec])
bricks = fakebricks
outbricks = bricks
else:
bricks = survey.get_bricks_readonly()
outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])]
assert(len(outbricks) == 1)
outsurvey = LegacySurveyData(survey_dir = args.outdir)
trymakedirs(args.outdir)
outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz'))
targetwcs = wcs_for_brick(outbricks[0])
H,W = targetwcs.shape
tycho2fn = survey.find_file('tycho2')
kd = tree_open(tycho2fn, 'stars')
radius = 1.
rc,dc = targetwcs.radec_center()
I = tree_search_radec(kd, rc, dc, radius)
print(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc,dc))
# Read only the rows within range.
tycho = fits_table(tycho2fn, rows=I)
del kd
print('Read', len(tycho), 'Tycho-2 stars')
ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec)
#margin = 100
#tycho.cut(ok * (tx > -margin) * (tx < W+margin) *
# (ty > -margin) * (ty < H+margin))
print('Cut to', len(tycho), 'Tycho-2 stars within brick')
del ok,tx,ty
#tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz'))
f,tfn = tempfile.mkstemp(suffix='.fits')
os.close(f)
tycho.writeto(tfn)
outfn = os.path.join(args.outdir, 'tycho2.kd.fits')
cmd = 'startree -i %s -o %s -P -k -n stars -T' % (tfn, outfn)
print(cmd)
rtn = os.system(cmd)
assert(rtn == 0)
os.unlink(tfn)
from legacypipe.gaiacat import GaiaCatalog
gcat = GaiaCatalog()
# from ps1cat.py:
wcs = targetwcs
step=100.
margin=10.
# Grid the CCD in pixel space
W,H = wcs.get_width(), wcs.get_height()
xx,yy = np.meshgrid(
np.linspace(1-margin, W+margin, 2+int((W+2*margin)/step)),
np.linspace(1-margin, H+margin, 2+int((H+2*margin)/step)))
# Convert to RA,Dec and then to unique healpixes
ra,dec = wcs.pixelxy2radec(xx.ravel(), yy.ravel())
healpixes = set()
for r,d in zip(ra,dec):
healpixes.add(gcat.healpix_for_radec(r, d))
for hp in healpixes:
hpcat = gcat.get_healpix_catalog(hp)
ok,xx,yy = wcs.radec2pixelxy(hpcat.ra, hpcat.dec)
onccd = np.flatnonzero((xx >= 1.-margin) * (xx <= W+margin) *
(yy >= 1.-margin) * (yy <= H+margin))
hpcat.cut(onccd)
if len(hpcat):
outfn = os.path.join(args.outdir, 'gaia', 'chunk-%05d.fits' % hp)
trymakedirs(os.path.join(args.outdir, 'gaia'))
hpcat.writeto(outfn)
outccds = C.copy()
cols = outccds.get_columns()
for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1',
'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1',
'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver',
'psfplver' ]:
if c in cols:
outccds.delete_column(c)
outccds.image_hdu[:] = 1
# Convert to list to avoid truncating filenames
outccds.image_filename = [fn for fn in outccds.image_filename]
for iccd,ccd in enumerate(C):
decam = (ccd.camera.strip() == 'decam')
bok = (ccd.camera.strip() == '90prime')
im = survey.get_image_object(ccd)
print('Got', im)
if survey.cache_dir is not None:
im.check_for_cached_files(survey)
slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1))
psfkwargs = dict(pixPsf=True, gaussPsf=False, hybridPsf=False,
normalizePsf=False)
tim = im.get_tractor_image(slc, pixPsf=True,
subsky=False, nanomaggies=False,
no_remap_invvar=True, old_calibs_ok=True)
print('Tim:', tim.shape)
psfrow = psfhdr = None
if args.pad:
psf = im.read_psf_model(0, 0, w=im.width, h=im.height, **psfkwargs)
psfex = psf.psfex
else:
psf = tim.getPsf()
psfex = psf.psfex
# Did the PSF model come from a merged file?
for fn in [im.merged_psffn, im.psffn] + im.old_merged_psffns:
if not os.path.exists(fn):
continue
T = fits_table(fn)
I, = np.nonzero((T.expnum == im.expnum) *
np.array([c.strip() == im.ccdname for c in T.ccdname]))
if len(I) != 1:
continue
psfrow = T[I]
x0 = ccd.ccd_x0
y0 = ccd.ccd_y0
psfrow.polzero1[0] -= x0
psfrow.polzero2[0] -= y0
#psfhdr = fitsio.read_header(im.merged_psffn)
break
psfex.fwhm = tim.psf_fwhm
#### HACK
#psfrow = None
assert(psfrow is not None)
if psfrow is not None:
print('PSF row:', psfrow)
#else:
# print('PSF:', psf)
# print('PsfEx:', psfex)
skyrow = skyhdr = None
if args.pad:
primhdr = fitsio.read_header(im.imgfn)
imghdr = fitsio.read_header(im.imgfn, hdu=im.hdu)
sky = im.read_sky_model(splinesky=True, primhdr=primhdr, imghdr=imghdr)
#skyhdr = fitsio.read_header(im.splineskyfn)
#msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)
else:
sky = tim.getSky()
# Did the sky model come from a merged file?
#msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)
print('merged skyfn:', im.merged_skyfn)
print('single skyfn:', im.skyfn)
print('old merged skyfns:', im.old_merged_skyfns)
for fn in [im.merged_skyfn, im.skyfn] + im.old_merged_skyfns:
if not os.path.exists(fn):
continue
T = fits_table(fn)
I, = np.nonzero((T.expnum == im.expnum) *
np.array([c.strip() == im.ccdname for c in T.ccdname]))
skyrow = T[I]
skyrow.x0[0] = ccd.ccd_x0
skyrow.y0[0] = ccd.ccd_y0
# s_med = skyrow.sky_med[0]
# s_john = skyrow.sky_john[0]
# skyhdr = fitsio.read_header(fn)
assert(skyrow is not None)
### HACK
#skyrow = None
if skyrow is not None:
print('Sky row:', skyrow)
else:
print('Sky:', sky)
# Output filename format:
fn = ccd.image_filename.strip()
ccd.image_filename = os.path.join(os.path.dirname(fn),
'%s.%s.fits' % (os.path.basename(fn).split('.')[0], ccd.ccdname.strip()))
outim = outsurvey.get_image_object(ccd)
print('Output image:', outim)
print('Image filename:', outim.imgfn)
trymakedirs(outim.imgfn, dir=True)
imgdata = tim.getImage()
ivdata = tim.getInvvar()
# Since we remap DQ codes (always with Mosaic and Bok, sometimes with DECam),
# re-read from the FITS file rather than using tim.dq.
print('Reading data quality from', im.dqfn, 'hdu', im.hdu)
dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice)
print('Tim shape:', tim.shape, 'Slice', tim.slice)
print('image shape:', imgdata.shape, 'iv', ivdata.shape, 'DQ', dqdata.shape)
from collections import Counter
dqvals = Counter(dqdata.ravel())
print('DQ pixel counts:')
for k,n in dqvals.most_common():
print(' 0x%x' % k, ':', n)
if args.pad:
# Create zero image of full size, copy in data.
fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype)
fullsize[slc] = imgdata
imgdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype)
fullsize[slc] = dqdata
dqdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype)
fullsize[slc] = ivdata
ivdata = fullsize
else:
# Adjust the header WCS by x0,y0
crpix1 = tim.hdr['CRPIX1']
crpix2 = tim.hdr['CRPIX2']
tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0
tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0
# Add image extension to filename
# fitsio doesn't compress .fz by default, so drop .fz suffix
#outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.imgfn = outim.imgfn.replace('.fits', '.fits.gz')
#outim.wtfn = outim.wtfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.wtfn = outim.wtfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.wtfn = outim.wtfn.replace('.fits', '.fits.gz')
if outim.dqfn is not None:
#outim.dqfn = outim.dqfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.dqfn = outim.dqfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.dqfn = outim.dqfn.replace('.fits', '.fits.gz')
if bok:
outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname)
ccdfn = outim.imgfn
ccdfn = ccdfn.replace(outsurvey.get_image_dir(), '')
if ccdfn.startswith('/'):
ccdfn = ccdfn[1:]
outccds.image_filename[iccd] = ccdfn
print('Changed output filenames to:')
print(outim.imgfn)
print(outim.dqfn)
ofn = outim.imgfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(imgdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
h,w = tim.shape
if not args.pad:
outccds.width[iccd] = w
outccds.height[iccd] = h
outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0
outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0
wcs = Tan(*[float(x) for x in
[ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2,
ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]])
if args.pad:
subwcs = wcs
else:
subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h)
outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center()
print('Weight filename:', outim.wtfn)
wfn = outim.wtfn
trymakedirs(wfn, dir=True)
ofn = wfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(ivdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
if outim.dqfn is not None:
print('DQ filename', outim.dqfn)
trymakedirs(outim.dqfn, dir=True)
ofn = outim.dqfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(dqdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
psfout = outim.psffn
#if psfrow:
# psfout = outim.merged_psffn
print('PSF output filename:', psfout)
trymakedirs(psfout, dir=True)
if psfrow:
psfrow.writeto(psfout, primhdr=psfhdr)
else:
print('Writing PsfEx:', psfout)
psfex.writeto(psfout)
# update header
F = fitsio.FITS(psfout, 'rw')
F[0].write_keys([dict(name='EXPNUM', value=ccd.expnum),
dict(name='PLVER', value=psf.plver),
dict(name='PROCDATE', value=psf.procdate),
dict(name='PLPROCID', value=psf.plprocid),])
F.close()
skyout = outim.skyfn
#if skyrow:
# skyout = outim.merged_splineskyfn
print('Sky output filename:', skyout)
trymakedirs(skyout, dir=True)
if skyrow is not None:
skyrow.writeto(skyout, primhdr=skyhdr)
else:
primhdr = fitsio.FITSHDR()
primhdr['PLVER'] = sky.plver
primhdr['PLPROCID'] = sky.plprocid
primhdr['PROCDATE'] = sky.procdate
primhdr['EXPNUM'] = ccd.expnum
primhdr['IMGDSUM'] = sky.datasum
primhdr['S_MED'] = s_med
primhdr['S_JOHN'] = s_john
sky.write_fits(skyout, primhdr=primhdr)
# HACK -- check result immediately.
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
outsurvey.ccds = None
outC = outsurvey.get_ccds_readonly()
occd = outC[iccd]
outim = outsurvey.get_image_object(occd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True,
hybridPsf=True, old_calibs_ok=True)
print('Got output tim:', otim)
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
# WISE
if args.wise is not None:
from wise.forcedphot import unwise_tiles_touching_wcs
from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs,
get_unwise_tractor_image, get_unwise_tile_dir)
# Read WCS...
print('Reading TAN wcs header from', args.wise, 'HDU', args.wise_wcs_hdu)
targetwcs = Tan(args.wise, args.wise_wcs_hdu)
tiles = unwise_tiles_touching_wcs(targetwcs)
print('Cut to', len(tiles), 'unWISE tiles')
H,W = targetwcs.shape
r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]),
np.array([H/2, H/2, 1, H ]))
roiradec = [r[0], r[1], d[2], d[3]]
unwise_dir = os.environ['UNWISE_COADDS_DIR']
wise_out = os.path.join(args.outdir, 'images', 'unwise')
print('Will write WISE outputs to', wise_out)
unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR']
wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr')
print('Will write WISE time-resolved outputs to', wise_tr_out)
trymakedirs(wise_tr_out)
W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_atlas.fits'))
print('Read', len(W), 'time-resolved WISE coadd tiles')
W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id]))
print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth')
# Write the time-resolved index subset.
W.writeto(os.path.join(wise_tr_out, 'time_resolved_atlas.fits'))
# this ought to be enough for anyone =)
_,Nepochs = W.epoch_bitmask.shape
print('N epochs in time-resolved atlas:', Nepochs)
wisedata = []
# full depth
for band in [1,2,3,4]:
wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band, True))
# time-resolved
for band in [1,2]:
# W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2)
bitmask = (1 << (band-1))
for e in range(Nepochs):
# Which tiles have images for this epoch?
I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask)
if len(I) == 0:
continue
print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I])
edir = os.path.join(unwise_tr_dir, 'e%03i' % e)
eoutdir = os.path.join(wise_tr_out, 'e%03i' % e)
wisedata.append((edir, eoutdir, tiles.coadd_id[I], band, False))
wrote_masks = set()
model_dir = os.environ.get('UNWISE_MODEL_SKY_DIR')
if model_dir is not None:
model_dir_out = os.path.join(args.outdir, 'images', 'unwise-mod')
trymakedirs(model_dir_out)
for indir, outdir, tiles, band, fulldepth in wisedata:
for tile in tiles:
wanyband = 'w'
tim = get_unwise_tractor_image(indir, tile, band,
bandname=wanyband, roiradecbox=roiradec)
print('Got unWISE tim', tim)
print(tim.shape)
if model_dir is not None and fulldepth and band in [1,2]:
print('ROI', tim.roi)
#0387p575.1.mod.fits
fn = '%s.%i.mod.fits' % (tile, band)
print('Filename', fn)
F = fitsio.FITS(os.path.join(model_dir, fn))
x0,x1,y0,y1 = tim.roi
slc = slice(y0,y1),slice(x0,x1)
phdr = F[0].read_header()
outfn = os.path.join(model_dir_out, fn)
for e,extname in [(1,'MODEL'), (2,'SKY')]:
pix = F[e][slc]
hdr = F[e].read_header()
crpix1 = hdr['CRPIX1']
crpix2 = hdr['CRPIX2']
hdr['CRPIX1'] -= x0
hdr['CRPIX2'] -= y0
#print('mod', mod)
#print('Model', mod.shape)
if e == 1:
fitsio.write(outfn, None, clobber=True, header=phdr)
fitsio.write(outfn, pix, header=hdr, extname=extname)
print('Wrote', outfn)
thisdir = get_unwise_tile_dir(outdir, tile)
print('Directory for this WISE tile:', thisdir)
base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band))
print('Base filename:', base)
masked = True
mu = 'm' if masked else 'u'
imfn = base + 'img-%s.fits' % mu
ivfn = base + 'invvar-%s.fits.gz' % mu
nifn = base + 'n-%s.fits.gz' % mu
nufn = base + 'n-u.fits.gz'
#print('WISE image header:', tim.hdr)
# Adjust the header WCS by x0,y0
wcs = tim.wcs.wcs
tim.hdr['CRPIX1'] = wcs.crpix[0]
tim.hdr['CRPIX2'] = wcs.crpix[1]
H,W = tim.shape
tim.hdr['IMAGEW'] = W
tim.hdr['IMAGEH'] = H
print('WCS:', wcs)
print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2'])
trymakedirs(imfn, dir=True)
fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True)
print('Wrote', imfn)
fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True)
print('Wrote', ivfn)
fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True)
print('Wrote', nifn)
fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True)
print('Wrote', nufn)
if not (indir,tile) in wrote_masks:
print('Looking for mask file for', indir, tile)
# record that we tried this dir/tile combo
wrote_masks.add((indir,tile))
for idir in indir.split(':'):
tdir = get_unwise_tile_dir(idir, tile)
maskfn = 'unwise-%s-msk.fits.gz' % tile
fn = os.path.join(tdir, maskfn)
print('Mask file:', fn)
if os.path.exists(fn):
print('Reading', fn)
(x0,x1,y0,y1) = tim.roi
roislice = (slice(y0,y1), slice(x0,x1))
F = fitsio.FITS(fn)[0]
hdr = F.read_header()
M = F[roislice]
outfn = os.path.join(thisdir, maskfn)
fitsio.write(outfn, M, header=tim.hdr, clobber=True)
print('Wrote', outfn)
break
outC = outsurvey.get_ccds_readonly()
for iccd,ccd in enumerate(outC):
outim = outsurvey.get_image_object(ccd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True,
hybridPsf=True, old_calibs_ok=True)
print('Got output tim:', otim)
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--force',
action='store_true',
help='Run calib processes even if files already exist?')
parser.add_argument('--survey-dir', help='Override LEGACY_SURVEY_DIR')
parser.add_argument(
'--expnum',
type=str,
help='Cut to a single or set of exposures; comma-separated list')
parser.add_argument('--extname',
'--ccdname',
help='Cut to a single extension/CCD name')
parser.add_argument('--no-psf',
dest='psfex',
action='store_false',
help='Do not compute PsfEx calibs')
parser.add_argument('--no-sky',
dest='sky',
action='store_false',
help='Do not compute sky models')
parser.add_argument('--run-se',
action='store_true',
help='Run SourceExtractor')
parser.add_argument('--no-splinesky',
dest='splinesky',
default=True,
action='store_false',
help='Use constant, not splinesky')
parser.add_argument('--threads',
type=int,
help='Run multi-threaded',
default=None)
parser.add_argument('--continue',
dest='cont',
default=False,
action='store_true',
help='Continue even if one file fails?')
parser.add_argument('--plot-base',
help='Make plots with this base filename')
parser.add_argument(
'--blob-mask-dir',
type=str,
default=None,
help=
'The base directory to search for blob masks during sky model construction'
)
parser.add_argument('-v',
'--verbose',
dest='verbose',
action='count',
default=0,
help='Make more verbose')
parser.add_argument('args', nargs=argparse.REMAINDER)
opt = parser.parse_args()
import logging
if opt.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
survey = LegacySurveyData(survey_dir=opt.survey_dir)
T = None
if len(opt.args) == 0:
if opt.expnum is not None:
expnums = set([int(e) for e in opt.expnum.split(',')])
T = merge_tables([
survey.find_ccds(expnum=e, ccdname=opt.extname)
for e in expnums
])
print('Cut to', len(T), 'with expnum in', expnums, 'and extname',
opt.extname)
opt.args = range(len(T))
else:
parser.print_help()
return 0
ps = None
if opt.plot_base is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plot_base)
survey_blob_mask = None
if opt.blob_mask_dir is not None:
survey_blob_mask = LegacySurveyData(opt.blob_mask_dir)
args = []
for a in opt.args:
# Check for "expnum-ccdname" format.
if '-' in str(a):
words = a.split('-')
assert (len(words) == 2)
expnum = int(words[0])
ccdname = words[1]
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
if len(T) != 1:
print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname',
ccdname, ':', I)
print('WARNING: skipping this expnum,ccdname')
continue
t = T[0]
else:
i = int(a)
print('Index', i)
t = T[i]
im = survey.get_image_object(t)
print('Running', im.name)
kwargs = dict(psfex=opt.psfex,
sky=opt.sky,
ps=ps,
survey=survey,
survey_blob_mask=survey_blob_mask)
if opt.force:
kwargs.update(force=True)
if opt.run_se:
kwargs.update(se=True)
if opt.splinesky:
kwargs.update(splinesky=True)
if opt.cont:
kwargs.update(noraise=True)
if opt.threads:
args.append((im, kwargs))
else:
run_calibs((im, kwargs))
if opt.threads:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
mp.map(time_run_calibs, args)
return 0
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--force',
action='store_true',
help='Run calib processes even if files already exist?')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument(
'--expnum',
type=str,
help='Cut to a single or set of exposures; comma-separated list')
parser.add_argument('--extname',
'--ccdname',
help='Cut to a single extension/CCD name')
parser.add_argument('--no-psf',
dest='psfex',
action='store_false',
help='Do not compute PsfEx calibs')
parser.add_argument('--no-sky',
dest='sky',
action='store_false',
help='Do not compute sky models')
parser.add_argument('--run-se',
action='store_true',
help='Run SourceExtractor')
parser.add_argument('--splinesky',
action='store_true',
help='Spline sky, not constant')
parser.add_argument('--threads',
type=int,
help='Run multi-threaded',
default=None)
parser.add_argument('--continue',
dest='cont',
default=False,
action='store_true',
help='Continue even if one file fails?')
parser.add_argument('args', nargs=argparse.REMAINDER)
opt = parser.parse_args()
survey = LegacySurveyData()
if opt.ccds is not None:
T = fits_table(opt.ccds)
T = survey.cleanup_ccds_table(T)
print('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
#print len(T), 'CCDs'
if len(opt.args) == 0:
if opt.expnum is not None:
expnums = set([int(e) for e in opt.expnum.split(',')])
T.cut(np.array([e in expnums for e in T.expnum]))
print('Cut to', len(T), 'with expnum in', expnums)
if opt.extname is not None:
T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname]))
print('Cut to', len(T), 'with extname =', opt.extname)
opt.args = range(len(T))
args = []
for a in opt.args:
# Check for "expnum-ccdname" format.
if '-' in str(a):
words = a.split('-')
assert (len(words) == 2)
expnum = int(words[0])
ccdname = words[1]
I = np.flatnonzero((T.expnum == expnum) * (T.ccdname == ccdname))
if len(I) != 1:
print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname',
ccdname, ':', I)
print('WARNING: skipping this expnum,ccdname')
continue
assert (len(I) == 1)
t = T[I[0]]
else:
i = int(a)
print('Index', i)
t = T[i]
#print('CCDnmatch', t.ccdnmatch)
#if t.ccdnmatch < 20 and not opt.force:
# print('Skipping ccdnmatch = %i' % t.ccdnmatch)
# continue
im = survey.get_image_object(t)
print('Running', im.name)
kwargs = dict(psfex=opt.psfex, sky=opt.sky)
if opt.force:
kwargs.update(force=True)
if opt.run_se:
kwargs.update(se=True)
if opt.splinesky:
kwargs.update(splinesky=True)
if opt.cont:
kwargs.update(noraise=True)
if opt.threads:
args.append((im, kwargs))
else:
run_calibs((im, kwargs))
if opt.threads:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
mp.map(run_calibs, args)
return 0
def main():
fn = '/global/cscratch1/sd/dstn/c4d_190730_024955_ori/c4d_190730_024955_ori.52.fits'
survey_dir = '/global/cscratch1/sd/dstn/subtractor-survey-dir'
imagedir = os.path.join(survey_dir, 'images')
trymakedirs(imagedir)
calibdir = os.path.join(survey_dir, 'calib')
psfexdir = os.path.join(calibdir, 'decam', 'psfex-merged')
trymakedirs(psfexdir)
skydir = os.path.join(calibdir, 'decam', 'splinesky-merged')
trymakedirs(skydir)
basename = os.path.basename(fn)
basename = basename.replace('.fits', '')
# Output filenames for legacyzpts calibration/zeropoint files
f, photfn = tempfile.mkstemp()
os.close(f)
surveyfn = os.path.join(survey_dir, 'survey-ccds-%s.fits.gz' % basename)
annfn = os.path.join(survey_dir, 'annotated-%s.fits' % basename)
mp = multiproc()
survey = LegacySurveyData(survey_dir)
# Use the subclass above to handle DECam images!
survey.image_typemap.update(decam=GoldsteinDecamImage)
# Grab the exposure number and CCD name
hdr = fitsio.read_header(fn)
expnum = hdr['EXPNUM']
ccdname = hdr['EXTNAME'].strip()
print('Exposure', expnum, 'CCD', ccdname)
import logging
lvl = logging.INFO
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
if not os.path.exists(surveyfn):
# Run calibrations and zeropoints
runit(fn,
photfn,
surveyfn,
annfn,
mp,
survey=survey,
camera='decam',
debug=False,
choose_ccd=ccdname,
splinesky=True,
calibdir=calibdir,
measureclass=GoldsteinDecamMeasurer)
# Find catalog sources touching this CCD
ccds = survey.find_ccds(expnum=expnum, ccdname=ccdname)
assert (len(ccds) == 1)
ccd = ccds[0]
print('Got CCD', ccd)
# Create Tractor image
im = survey.get_image_object(ccd)
print('Got image:', im)
# Look at this sub-image, or the whole chip?
#zoomslice=None
zoomslice = (slice(0, 1000), slice(0, 1000))
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
splinesky=True,
hybridPsf=True,
normalizePsf=True,
old_calibs_ok=True)
print('Got tim:', tim)
# Read catalog files touching this CCD
catsurvey = LegacySurveyData(
'/global/project/projectdirs/cosmo/work/legacysurvey/dr8/south')
T = get_catalog_in_wcs(tim.subwcs, catsurvey)
print('Got', len(T), 'DR8 catalog sources within CCD')
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
from legacypipe.survey import radec_at_mjd
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra, dec = radec_at_mjd(T.ra[I], T.dec[I], T.ref_epoch[I].astype(float),
T.pmra[I], T.pmdec[I], T.parallax[I],
tim.time.toMjd())
T.ra[I] = ra
T.dec[I] = dec
# Create Tractor Source objects from the catalog
cat = read_fits_catalog(T, bands=tim.band)
print('Created', len(cat), 'source objects')
# Render model image!
tr = Tractor([tim], cat)
mod = tr.getModelImage(0)
# plots
ima = dict(interpolation='nearest',
origin='lower',
vmin=-2 * tim.sig1,
vmax=10 * tim.sig1,
cmap='gray')
plt.clf()
plt.imshow(tim.getImage(), **ima)
plt.title('Image')
plt.savefig('img.jpg')
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model')
plt.savefig('mod.jpg')
plt.clf()
plt.imshow(tim.getImage() - mod, **ima)
plt.title('Residual')
plt.savefig('res.jpg')
print(len(I1), 'oki images')
print(len(I2), 'non-oki images')
ccds.cut(
np.hstack(
zip(I1[np.random.permutation(len(I1))],
I2[np.random.permutation(len(I2))])))
#ccds.cut(np.random.permutation(len(ccds)))
allsigs1 = []
allsigs2 = []
names = []
offsets = [1, 2, 3, 5, 10, 20, 40]
for ccd in ccds[:16]:
im = survey.get_image_object(ccd)
tim1 = im.get_tractor_image(gaussPsf=True, splinesky=True, dq=False)
tim2 = im.get_tractor_image(gaussPsf=True,
splinesky=True,
dq=False,
nanomaggies=False,
subsky=False)
tim = tim2
name = tim.name
if 'oki' in im.imgfn:
name += ' oki'
elif 'ooi' in im.imgfn:
name += ' ooi'
def main():
survey = LegacySurveyData()
ccds = survey.get_ccds()
print(len(ccds), 'CCDs')
expnums = np.unique(ccds.expnum)
print(len(expnums), 'unique exposures')
for expnum in expnums:
expnumstr = '%08i' % expnum
skyoutfn = os.path.join('splinesky', expnumstr[:5], 'decam-%s.fits' % expnumstr)
psfoutfn = os.path.join('psfex', expnumstr[:5], 'decam-%s.fits' % expnumstr)
if os.path.exists(skyoutfn) and os.path.exists(psfoutfn):
print('Exposure', expnum, 'is done already')
continue
C = ccds[ccds.expnum == expnum]
print(len(C), 'CCDs in expnum', expnum)
psfex = []
psfhdrvals = []
splinesky = []
skyhdrvals = []
for ccd in C:
im = survey.get_image_object(ccd)
fn = im.splineskyfn
if os.path.exists(fn):
print('Reading', fn)
T = None
try:
T = fits_table(fn)
except KeyboardInterrupt:
raise
except:
print('Failed to read file', fn, ':', sys.exc_info()[1])
if T is not None:
splinesky.append(T)
# print(fn)
# T.about()
hdr = fitsio.read_header(fn)
skyhdrvals.append([hdr[k] for k in [
'SKY', 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
else:
print('File not found:', fn)
fn = im.psffn
if os.path.exists(fn):
T = fits_table(fn)
hdr = fitsio.read_header(fn, ext=1)
keys = ['LOADED', 'ACCEPTED', 'CHI2', 'POLNAXIS',
'POLNGRP', 'PSF_FWHM', 'PSF_SAMP', 'PSFNAXIS',
'PSFAXIS1', 'PSFAXIS2', 'PSFAXIS3',]
if hdr['POLNAXIS'] == 0:
# No polynomials. Fake it.
T.polgrp1 = np.array([0])
T.polgrp2 = np.array([0])
T.polname1 = np.array(['fake'])
T.polname2 = np.array(['fake'])
T.polzero1 = np.array([0])
T.polzero2 = np.array([0])
T.polscal1 = np.array([1])
T.polscal2 = np.array([1])
T.poldeg1 = np.array([0])
else:
keys.extend([
'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1',
'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2',
'POLDEG1'])
for k in keys:
try:
v = hdr[k]
except:
print('Did not find key', k, 'in', fn)
sys.exit(-1)
T.set(k.lower(), np.array([hdr[k]]))
psfex.append(T)
#print(fn)
#T.about()
hdr = fitsio.read_header(fn)
psfhdrvals.append([hdr.get(k,'') for k in [
'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
else:
print('File not found:', fn)
if len(psfex):
padded = pad_arrays([p.psf_mask[0] for p in psfex])
cols = psfex[0].columns()
cols.remove('psf_mask')
T = merge_tables(psfex, columns=cols)
T.psf_mask = np.concatenate([[p] for p in padded])
T.legpipev = np.array([h[0] for h in psfhdrvals])
T.plver = np.array([h[1] for h in psfhdrvals])
T.expnum = np.array([h[2] for h in psfhdrvals])
T.ccdname = np.array([h[3] for h in psfhdrvals])
fn = psfoutfn
trymakedirs(fn, dir=True)
T.writeto(fn)
print('Wrote', fn)
if len(splinesky):
T = fits_table()
T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky])
T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky])
padded = pad_arrays([t.gridvals[0] for t in splinesky])
T.gridvals = np.concatenate([[p] for p in padded])
padded = pad_arrays([t.xgrid[0] for t in splinesky])
T.xgrid = np.concatenate([[p] for p in padded])
padded = pad_arrays([t.xgrid[0] for t in splinesky])
T.ygrid = np.concatenate([[p] for p in padded])
cols = splinesky[0].columns()
print('Columns:', cols)
for c in ['gridvals', 'xgrid', 'ygrid']:
cols.remove(c)
T.add_columns_from(merge_tables(splinesky, columns=cols))
T.skyclass = np.array([h[0] for h in skyhdrvals])
T.legpipev = np.array([h[1] for h in skyhdrvals])
T.plver = np.array([h[2] for h in skyhdrvals])
T.expnum = np.array([h[3] for h in skyhdrvals])
T.ccdname = np.array([h[4] for h in skyhdrvals])
fn = skyoutfn
trymakedirs(fn, dir=True)
T.writeto(fn)
print('Wrote', fn)
def main(survey=None, opt=None, args=None):
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if args is None:
args = sys.argv[1:]
print('forced_photom.py', ' '.join(args))
if opt is None:
parser = get_parser()
opt = parser.parse_args(args)
import logging
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
t0 = Time()
if survey is None:
survey = LegacySurveyData(survey_dir=opt.survey_dir,
cache_dir=opt.cache_dir,
output_dir=opt.out_dir)
if opt.skip:
if opt.out is not None:
outfn = opt.out
else:
outfn = survey.find_file('forced',
output=True,
camera=opt.camera,
expnum=opt.expnum)
if os.path.exists(outfn):
print('Ouput file exists:', outfn)
return 0
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
return -1
if opt.out is None and opt.out_dir is None:
print('Must supply either --out or --out-dir')
return -1
if opt.expnum is None and opt.out is None:
print('If no --expnum is given, must supply --out filename')
return -1
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Cache CCDs files before the find_ccds call...
# Copy required files into the cache?
if opt.pre_cache:
def copy_files_to_cache(fns):
for fn in fns:
cachefn = fn.replace(survey.survey_dir, survey.cache_dir)
if not cachefn.startswith(survey.cache_dir):
print('Skipping', fn)
continue
outdir = os.path.dirname(cachefn)
trymakedirs(outdir)
print('Copy', fn)
print(' to', cachefn)
shutil.copyfile(fn, cachefn)
assert (survey.cache_dir is not None)
fnset = set()
fn = survey.find_file('bricks')
fnset.add(fn)
fns = survey.find_file('ccd-kds')
fnset.update(fns)
copy_files_to_cache(fnset)
# Read metadata from survey-ccds.fits table
ccds = survey.find_ccds(camera=opt.camera,
expnum=opt.expnum,
ccdname=opt.ccdname)
print(len(ccds), 'with camera', opt.camera, 'and expnum', opt.expnum,
'and ccdname', opt.ccdname)
# sort CCDs
ccds.cut(np.lexsort((ccds.ccdname, ccds.expnum, ccds.camera)))
# If there is only one catalog survey_dir, we pass it to get_catalog_in_wcs
# as the northern survey.
catsurvey_north = survey
catsurvey_south = None
if opt.catalog_dir_north is not None:
assert (opt.catalog_dir_south is not None)
assert (opt.catalog_resolve_dec_ngc is not None)
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
elif opt.catalog_dir is not None:
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)
# Copy required CCD & calib files into the cache?
if opt.pre_cache:
assert (survey.cache_dir is not None)
fnset = set()
for ccd in ccds:
im = survey.get_image_object(ccd)
for key in im.get_cacheable_filename_variables():
fn = getattr(im, key)
if fn is None or not (os.path.exists(fn)):
continue
fnset.add(fn)
copy_files_to_cache(fnset)
args = []
for ccd in ccds:
args.append((survey, catsurvey_north, catsurvey_south,
opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))
if opt.threads:
from astrometry.util.multiproc import multiproc
from astrometry.util.timingpool import TimingPool, TimingPoolMeas
pool = TimingPool(opt.threads)
poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(poolmeas)
mp = multiproc(None, pool=pool)
tm = Time()
FF = mp.map(bounce_one_ccd, args)
print('Multi-processing forced-phot:', Time() - tm)
del mp
Time.measurements.remove(poolmeas)
del poolmeas
pool.close()
pool.join()
del pool
else:
FF = map(bounce_one_ccd, args)
FF = [F for F in FF if F is not None]
if len(FF) == 0:
print('No photometry results to write.')
return 0
# Keep only the first header
_, version_hdr, _, _ = FF[0]
# unpack results
outlier_masks = [m for _, _, m, _ in FF]
outlier_hdrs = [h for _, _, _, h in FF]
FF = [F for F, _, _, _ in FF]
F = merge_tables(FF)
if len(ccds):
version_hdr.delete('CPHDU')
version_hdr.delete('CCDNAME')
from legacypipe.utils import add_bits
from legacypipe.bits import DQ_BITS
add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
for i, ap in enumerate(apertures_arcsec):
version_hdr.add_record(
dict(name='APRAD%i' % i,
value=ap,
comment='(optical) Aperture radius, in arcsec'))
unitmap = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2',
'apflux': 'nanomaggy',
'apflux_ivar': '1/nanomaggy^2',
'psfdepth': '1/nanomaggy^2',
'galdepth': '1/nanomaggy^2',
'sky': 'nanomaggy/arcsec^2',
'psfsize': 'arcsec',
'fwhm': 'pixels',
'ccdrarms': 'arcsec',
'ccddecrms': 'arcsec',
'ra': 'deg',
'dec': 'deg',
'skyrms': 'counts/sec',
'dra': 'arcsec',
'ddec': 'arcsec',
'dra_ivar': '1/arcsec^2',
'ddec_ivar': '1/arcsec^2'
}
columns = F.get_columns()
order = [
'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'fwhm', 'ccd_cuts',
'airmass', 'sky', 'skyrms', 'psfdepth', 'galdepth', 'ccdzpt',
'ccdrarms', 'ccddecrms', 'ccdphrms', 'ra', 'dec', 'flux', 'flux_ivar',
'fracflux', 'rchisq', 'fracmasked', 'fracin', 'apflux', 'apflux_ivar',
'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar'
]
columns = [c for c in order if c in columns]
units = [unitmap.get(c, '') for c in columns]
if opt.out is not None:
outdir = os.path.dirname(opt.out)
if len(outdir):
trymakedirs(outdir)
tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.out))
fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
F.writeto(tmpfn, units=units, append=True, columns=columns)
os.rename(tmpfn, opt.out)
print('Wrote', opt.out)
else:
with survey.write_output('forced',
camera=opt.camera,
expnum=opt.expnum) as out:
F.writeto(None,
fits_object=out.fits,
primheader=version_hdr,
units=units,
columns=columns)
print('Wrote', out.real_fn)
if opt.outlier_mask is not None:
# Add outlier bit meanings to the primary header
version_hdr.add_record(
dict(name='COMMENT', value='Outlier mask bit meanings'))
version_hdr.add_record(
dict(name='OUTL_POS',
value=1,
comment='Outlier mask bit for Positive outlier'))
version_hdr.add_record(
dict(name='OUTL_NEG',
value=2,
comment='Outlier mask bit for Negative outlier'))
if opt.outlier_mask == 'default':
outdir = os.path.join(opt.out_dir, 'outlier-masks')
camexp = set(zip(ccds.camera, ccds.expnum))
for c, e in camexp:
I = np.flatnonzero((ccds.camera == c) * (ccds.expnum == e))
ccd = ccds[I[0]]
imfn = ccd.image_filename.strip()
outfn = os.path.join(outdir, imfn.replace('.fits',
'-outlier.fits'))
trymakedirs(outfn, dir=True)
tempfn = outfn.replace('.fits', '-tmp.fits')
with fitsio.FITS(tempfn, 'rw', clobber=True) as fits:
fits.write(None, header=version_hdr)
for i in I:
mask = outlier_masks[i]
_, _, _, meth, tile = survey.get_compression_args(
'outliers_mask', shape=mask.shape)
fits.write(mask,
header=outlier_hdrs[i],
extname=ccds.ccdname[i],
compress=meth,
tile_dims=tile)
os.rename(tempfn, outfn)
print('Wrote', outfn)
elif opt.outlier_mask is not None:
with fitsio.FITS(opt.outlier_mask, 'rw', clobber=True) as F:
F.write(None, header=version_hdr)
for i, (hdr, mask) in enumerate(zip(outlier_hdrs, outlier_masks)):
_, _, _, meth, tile = survey.get_compression_args(
'outliers_mask', shape=mask.shape)
F.write(mask,
header=hdr,
extname=ccds.ccdname[i],
compress=meth,
tile_dims=tile)
print('Wrote', opt.outlier_mask)
tnow = Time()
print('Total:', tnow - t0)
return 0
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--force', action='store_true',
help='Run calib processes even if files already exist?')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--expnum', type=int, help='Cut to a single exposure')
parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name')
parser.add_argument('--no-psf', dest='psfex', action='store_false',
help='Do not compute PsfEx calibs')
parser.add_argument('--no-sky', dest='sky', action='store_false',
help='Do not compute sky models')
parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor')
parser.add_argument('--splinesky', action='store_true', help='Spline sky, not constant')
parser.add_argument('--threads', type=int, help='Run multi-threaded', default=None)
parser.add_argument('args',nargs=argparse.REMAINDER)
opt = parser.parse_args()
survey = LegacySurveyData()
if opt.ccds is not None:
T = fits_table(opt.ccds)
print('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
#print len(T), 'CCDs'
if len(opt.args) == 0:
if opt.expnum is not None:
T.cut(T.expnum == opt.expnum)
print('Cut to', len(T), 'with expnum =', opt.expnum)
if opt.extname is not None:
T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname]))
print('Cut to', len(T), 'with extname =', opt.extname)
opt.args = range(len(T))
args = []
for a in opt.args:
# Check for "expnum-ccdname" format.
if '-' in str(a):
words = a.split('-')
assert(len(words) == 2)
expnum = int(words[0])
ccdname = words[1]
I = np.flatnonzero((T.expnum == expnum) * (T.ccdname == ccdname))
if len(I) != 1:
print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname', ccdname, ':', I)
assert(len(I) == 1)
t = T[I[0]]
else:
i = int(a)
print('Index', i)
t = T[i]
print('CCDnmatch', t.ccdnmatch)
if t.ccdnmatch < 20 and not opt.force:
print('Skipping ccdnmatch = %i' % t.ccdnmatch)
continue
im = survey.get_image_object(t)
print('Running', im.calname)
kwargs = dict(psfex=opt.psfex, sky=opt.sky)
if opt.force:
kwargs.update(force=True)
if opt.run_se:
kwargs.update(se=True)
if opt.splinesky:
kwargs.update(splinesky=True)
if opt.threads:
args.append((im, kwargs))
else:
run_calibs((im, kwargs))
if opt.threads:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
mp.map(run_calibs, args)
return 0
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(
description=
"This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
)
parser.add_argument('--calibs',
action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper',
type=int,
default=None,
help='Batch N calibs per line')
parser.add_argument(
'--byexp',
action='store_true',
default=False,
help='Run one whole exposure per job (not one CCD per job)')
parser.add_argument('--forced',
action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb',
action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--stage', help='Stage image files to given directory')
parser.add_argument('--touching',
action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--near',
action='store_true',
help='Quick cut to only CCDs near selected bricks')
parser.add_argument('--check-coadd',
action='store_true',
help='Check which coadds actually need to run.')
parser.add_argument('--out',
help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command',
action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--opt', help='With --command, extra options to add')
parser.add_argument('--maxra', type=float, help='Maximum RA to run')
parser.add_argument('--minra', type=float, help='Minimum RA to run')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--ignore_cuts',
action='store_true',
default=False,
help='no photometric cuts')
parser.add_argument('--save_to_fits',
action='store_true',
default=False,
help='save cut brick,ccd to fits table')
parser.add_argument(
'--name',
action='store',
default='dr3',
help='save with this suffix, e.g. refers to ccds table')
parser.add_argument('--delete-sky',
action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
parser.add_argument('--nccds',
action='store_true',
default=False,
help='Prints number of CCDs per brick')
parser.add_argument('--bands',
default='g,r,z',
help='Set bands to keep: comma-separated list.')
opt = parser.parse_args()
want_ccds = (opt.calibs or opt.forced or opt.lsb)
want_bricks = not want_ccds
survey = LegacySurveyData()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = survey.get_bricks()
log('Bricks Dec range:', B.dec.min(), B.dec.max())
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
if opt.ignore_cuts == False:
log('Applying CCD cuts...')
if 'ccd_cuts' in T.columns():
T.cut(T.ccd_cuts == 0)
log(len(T), 'CCDs survive cuts')
bands = opt.bands.split(',')
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
log('CCDs Dec range:', T.dec.min(), T.dec.max())
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo, dhi = -25, 40
rlo, rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(
test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo, dhi = -90, 90
rlo, rhi = 0, 360
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo, rhi = 240, 245
dlo, dhi = 5, 12
elif opt.region == 'dr8-decam':
rlo, rhi = 0, 360
dlo, dhi = -70, 40
log('DR8-DECam region')
elif opt.region == 'edrplus':
rlo, rhi = 235, 248
dlo, dhi = 5, 15
elif opt.region == 'edr-south':
rlo, rhi = 240, 245
dlo, dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo, rhi = 149.75, 150.75
dlo, dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo, rhi = 240, 250
dlo, dhi = 10, 15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo, rhi = 320., 360.
dlo, dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo, rhi = 240, 360
dlo, dhi = 20, 40
elif opt.region == 'north':
rlo, rhi = 120, 240
dlo, dhi = 20, 40
elif opt.region == 'northeast':
rlo, rhi = 0, 120
dlo, dhi = 20, 40
elif opt.region == 'southwest':
rlo, rhi = 240, 360
dlo, dhi = -20, 0
elif opt.region == 'south':
rlo, rhi = 120, 240
dlo, dhi = -20, 0
elif opt.region == 'southeast':
rlo, rhi = 0, 120
dlo, dhi = -20, 0
elif opt.region == 'southsoutheast':
rlo, rhi = 0, 120
dlo, dhi = -20, -10
elif opt.region == 'midwest':
rlo, rhi = 240, 360
dlo, dhi = 0, 20
elif opt.region == 'middle':
rlo, rhi = 120, 240
dlo, dhi = 0, 20
elif opt.region == 'mideast':
rlo, rhi = 0, 120
dlo, dhi = 0, 20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo, rhi = 250, 260
dlo, dhi = 30, 35
elif opt.region == 'deep2f2':
rlo, rhi = 251.4, 254.4
dlo, dhi = 34.6, 35.3
elif opt.region == 'deep2f3':
rlo, rhi = 351.25, 353.75
dlo, dhi = 0, 0.5
elif opt.region == 'deep3':
rlo, rhi = 214, 216
dlo, dhi = 52.25, 53.25
elif opt.region == 'virgo':
rlo, rhi = 185, 190
dlo, dhi = 10, 15
elif opt.region == 'virgo2':
rlo, rhi = 182, 192
dlo, dhi = 8, 18
elif opt.region == 'coma':
# van Dokkum et al Coma cluster ultra-diffuse galaxies: 3x3 field centered on Coma cluster
rc, dc = 195., 28.
dd = 1.5
cosdec = np.cos(np.deg2rad(dc))
rlo, rhi = rc - dd / cosdec, rc + dd / cosdec
dlo, dhi = dc - dd, dc + dd
elif opt.region == 'lsb':
rlo, rhi = 147.2, 147.8
dlo, dhi = -0.4, 0.4
elif opt.region == 'eboss-sgc':
# generous boundaries to make sure get all relevant images
# RA -45 to +45
# Dec -5 to +7
rlo, rhi = 310., 50.
dlo, dhi = -6., 6.
elif opt.region == 'eboss-ngc':
# generous boundaries to make sure get all relevant images
# NGC ELGs
# RA 115 to 175
# Dec 15 to 30
# rlo,rhi = 122., 177.
# dlo,dhi = 12., 32.
rlo, rhi = 126., 168.
dlo, dhi = 18., 33.
elif opt.region == 'mzls':
dlo, dhi = -10., 90. # -10: pull in Stripe 82 data too
elif opt.region == 'dr4-bootes':
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
#dlo,dhi = 34., 35.
#rlo,rhi = 209.5, 210.5
dlo, dhi = 33., 36.
rlo, rhi = 216.5, 219.5
elif opt.region == 'des-sn-x3':
#rlo,rhi = 36., 37.
#dlo,dhi = -5., -4.
rlo, rhi = 36., 36.5
dlo, dhi = -4.5, -4.
elif opt.region == 'ngc2632':
# open cluster
rlo, rhi = 129.0, 131.0
dlo, dhi = 19.0, 20.5
elif opt.region == 'dr8sky':
rlo, rhi = 35.0, 37.0
dlo, dhi = -3.0, -1.0
# ADM DR8 test regions, see, e.g.:
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR8#Testregions
elif opt.region == 'dr8-test-s82':
rlo, rhi = 0, 45
dlo, dhi = -1.25, 1.25
elif opt.region == 'dr8-test-hsc-sgc':
rlo, rhi = 30, 40
dlo, dhi = -6.5, -1.25
elif opt.region == 'dr8-test-hsc-ngc':
rlo, rhi = 177.5, 182.5
dlo, dhi = -1, 1
elif opt.region == 'dr8-test-edr':
rlo, rhi = 240, 245
dlo, dhi = 5, 12
elif opt.region == 'dr8-test-hsc-north':
rlo, rhi = 240, 250
dlo, dhi = 42, 45
elif opt.region == 'dr8-test-deep2-egs':
rlo, rhi = 213, 216.5
dlo, dhi = 52, 54
elif opt.region == 'dr8-test-overlap':
rlo, rhi = 132, 140.5
dlo, dhi = 31.5, 35
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if opt.minra is not None:
rlo = opt.minra
if opt.maxra is not None:
rhi = opt.maxra
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(
np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
(B.dec <= dhi))
log(len(B), 'bricks in range; cut Dec range', B.dec.min(), B.dec.max())
#for name in B.get('brickname'):
# print(name)
#B.writeto('bricks-cut.fits')
bricksize = 0.25
# A bit more than 0.25-degree brick radius + Bok image radius ~ 0.57
search_radius = 1.05 * np.sqrt(2.) * (bricksize +
(0.455 * 4096 / 3600.)) / 2.
log(len(T), 'CCDs')
log(len(B), 'Bricks')
I, J, d = match_radec(B.ra,
B.dec,
T.ra,
T.dec,
search_radius,
nearest=True)
B.cut(I)
log('Cut to', len(B), 'bricks near CCDs')
log('Bricks Dec range:', B.dec.min(), B.dec.max())
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# plt.title('DR3 bricks')
# plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
# plt.savefig('bricks.png')
if opt.touching:
I, J, d = match_radec(T.ra,
T.dec,
B.ra,
B.dec,
search_radius,
nearest=True)
# list the ones that will be cut
# drop = np.ones(len(T))
# drop[I] = False
# for i in np.flatnonzero(drop):
# from astrometry.util.starutil_numpy import degrees_between
# dists = degrees_between(B.ra, B.dec, T.ra[i], T.dec[i])
# mindist = min(dists)
# print('Dropping:', T.ra[i], T.dec[i], 'min dist', mindist, 'search_radius', search_radius)
T.cut(I)
log('Cut to', len(T), 'CCDs near bricks')
# sort by RA increasing
B.cut(np.argsort(B.ra))
if opt.save_to_fits:
assert (opt.touching)
# Write cut tables to file
for tab, typ in zip([B, T], ['bricks', 'ccds']):
fn = '%s-%s-cut.fits' % (typ, opt.region)
if os.path.exists(fn):
os.remove(fn)
tab.writeto(fn)
log('Wrote %s' % fn)
# Write text files listing ccd and filename names
# nm1,nm2= 'ccds-%s.txt'% opt.region,'filenames-%s.txt' % opt.region
# if os.path.exists(nm1):
# os.remove(nm1)
# if os.path.exists(nm2):
# os.remove(nm2)
# f1,f2=open(nm1,'w'),open(nm2,'w')
# fns= list(set(T.get('image_filename')))
# for fn in fns:
# f2.write('%s\n' % fn.strip())
# for ti in T:
# f1.write('%s\n' % ti.get('image_filename').strip())
# f1.close()
# f2.close()
# log('Wrote *-names.txt')
if opt.touching:
if want_bricks:
# Shortcut the list of bricks that are definitely touching CCDs --
# a brick-ccd pair within this radius must be touching.
closest_radius = 0.95 * (bricksize + 0.262 * 2048 / 3600.) / 2.
J1, nil, nil = match_radec(B.ra,
B.dec,
T.ra,
T.dec,
closest_radius,
nearest=True)
log(len(J1), 'of', len(B), 'bricks definitely touch CCDs')
tocheck = np.ones(len(B), bool)
tocheck[J1] = False
J2 = []
for j in np.flatnonzero(tocheck):
b = B[j]
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickname)
if len(I) == 0:
continue
J2.append(j)
J = np.hstack((J1, J2))
J = np.sort(J).astype(int)
B.cut(J)
log('Cut to', len(B), 'bricks touching CCDs')
else:
J = []
allI = set()
for j, b in enumerate(B):
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickname)
if len(I) == 0:
continue
allI.update(I)
J.append(j)
allI = list(allI)
allI.sort()
B.cut(np.array(J))
log('Cut to', len(B), 'bricks touching CCDs')
elif opt.near:
# Find CCDs near bricks
allI, nil, nil = match_radec(T.ra,
T.dec,
B.ra,
B.dec,
search_radius,
nearest=True)
# Find bricks near CCDs
J, nil, nil = match_radec(B.ra,
B.dec,
T.ra,
T.dec,
search_radius,
nearest=True)
B.cut(J)
log('Cut to', len(B), 'bricks near CCDs')
else:
allI = np.arange(len(T))
if opt.byexp:
nil, eI = np.unique(T.expnum[allI], return_index=True)
allI = allI[eI]
print('Cut to', len(allI), 'expnums')
if opt.nccds:
from queue import Queue
from threading import Thread
log('Checking number of CCDs per brick')
def worker():
while True:
i = q.get()
if i is None:
break
b = B[i]
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(b.brickname, len(I))
q.task_done()
q = Queue()
num_threads = 24
threads = []
for i in range(num_threads):
t = Thread(target=worker)
t.start()
threads.append(t)
for i in range(len(B)):
q.put(i)
q.join()
for i in range(num_threads):
q.put(None)
for t in threads:
t.join()
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
if want_bricks:
# Print the list of bricks and exit.
for b in B:
print(b.brickname)
if opt.save_to_fits:
B.writeto('bricks-%s-touching.fits' % opt.region)
if not want_ccds:
sys.exit(0)
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.stage is not None:
cmd_pat = 'rsync -LRarv %s %s'
fns = set()
for iccd in allI:
im = survey.get_image_object(T[iccd])
fns.update([
im.imgfn, im.wtfn, im.dqfn, im.psffn, im.merged_psffn,
im.merged_splineskyfn, im.splineskyfn
])
for i, fn in enumerate(fns):
print('File', i + 1, 'of', len(fns), ':', fn)
if not os.path.exists(fn):
print('No such file:', fn)
continue
base = survey.get_survey_dir()
if base.endswith('/'):
base = base[:-1]
rel = os.path.relpath(fn, base)
dest = os.path.join(opt.stage, rel)
print('Dest:', dest)
if os.path.exists(dest):
print('Exists:', dest)
continue
cmd = cmd_pat % ('%s/./%s' % (base, rel), opt.stage)
print(cmd)
rtn = os.system(cmd)
assert (rtn == 0)
sys.exit(0)
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
expstr = '%08i' % T.expnum[i]
imgfn = os.path.join(survey.survey_dir, 'images',
T.image_filename[i].strip())
if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
and os.path.exists(imgfn[:-3])):
imgfn = imgfn[:-3]
outfn = os.path.join(
expstr[:5], expstr, 'forced-%s-%s-%s.fits' %
(T.camera[i].strip(), expstr, T.ccdname[i]))
f.write(
'python legacypipe/forced_photom.py --apphot --derivs --catalog-dir /project/projectdirs/cosmo/data/legacysurvey/dr7/ %i %s forced/%s\n'
% (T.expnum[i], T.ccdname[i], outfn))
f.close()
log('Wrote', opt.out)
fn = 'forced-ccds.fits'
T[allI].writeto(fn)
print('Wrote', fn)
sys.exit(0)
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write(
'python legacyanalysis/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
% (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
sys.exit(0)
log('Writing calibs to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if cmd is None:
cmd = ''
f.write(cmd + ' '.join(batch) + '\n')
cmd = None
if opt.command:
cmd = 'python legacypipe/run-calib.py '
if opt.opt is not None:
cmd += opt.opt + ' '
for j, i in enumerate(allI):
if opt.delete_sky:
log(j + 1, 'of', len(allI))
im = survey.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.command:
if opt.byexp:
s = '--expnum %i' % (T.expnum[i])
else:
s = '%i-%s' % (T.expnum[i], T.ccdname[i])
prefix = 'python legacypipe/run-calib.py '
if opt.opt is not None:
prefix = prefix + opt.opt
#('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
# (T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
prefix = ''
if j < 10:
print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname',
T.ccdname[i], 'filename', T.image_filename[i])
if not opt.nper:
f.write(prefix + s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, cmd)
batch = []
if len(batch):
write_batch(f, batch, cmd)
f.close()
log('Wrote', opt.out)
return 0
def psf_residuals(expnum,ccdname,stampsize=35,nstar=30,
magrange=(13,17),verbose=0, splinesky=False):
# Set the debugging level.
if verbose==0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)
pngprefix = 'qapsf-{}-{}'.format(expnum,ccdname)
# Gather all the info we need about this CCD.
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum,ccdname=ccdname)[0]
band = ccd.filter
ps1band = dict(g=0,r=1,i=2,z=3,Y=4)
print('Band {}'.format(band))
#scales = dict(g=0.0066, r=0.01, z=0.025)
#vmin, vmax = np.arcsinh(-1), np.arcsinh(100)
#print(scales[band])
im = survey.get_image_object(ccd)
iminfo = im.get_image_info()
H,W = iminfo['dims']
wcs = im.get_wcs()
# Choose a uniformly selected subset of PS1 stars on this CCD.
ps1 = ps1cat(ccdwcs=wcs)
cat = ps1.get_stars(band=band,magrange=magrange)
rand = np.random.RandomState(seed=expnum*ccd.ccdnum)
these = rand.choice(len(cat)-1,nstar,replace=False)
#these = rand.random_integers(0,len(cat)-1,nstar)
cat = cat[these]
cat = cat[np.argsort(cat.median[:,ps1band[band]])] # sort by magnitude
#print(cat.nmag_ok)
get_tim_kwargs = dict(pixPsf=True, splinesky=splinesky)
# Make a QAplot of the positions of all the stars.
tim = im.get_tractor_image(**get_tim_kwargs)
img = tim.getImage()
#img = tim.getImage()/scales[band]
fig = plt.figure(figsize=(5,10))
ax = fig.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
#ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest',
# origin='lower',vmin=vmax,vmax=vmax)
ax.imshow(img, **tim.ima)
ax.axis('off')
ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.
format(band,expnum,ccdname,ccd.airmass,ccd.seeing))
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ax.text(xpos,ypos,'{:2d}'.format(istar+1),color='red',
horizontalalignment='left')
circ = plt.Circle((xpos,ypos),radius=30,color='g',fill=False,lw=1)
ax.add_patch(circ)
#radec = wcs.radec_bounds()
#ax.scatter(cat.ra,cat.dec)
#ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998])
#ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015])
#ax.set_xlabel('$RA\ (deg)$',fontsize=18)
#ax.set_ylabel('$Dec\ (deg)$',fontsize=18)
fig.savefig(pngprefix+'-ccd.png',bbox_inches='tight')
# Initialize the many-stamp QAplot
ncols = 3
nrows = np.ceil(nstar/ncols).astype('int')
inchperstamp = 2.0
fig = plt.figure(figsize=(inchperstamp*3*ncols,inchperstamp*nrows))
irow = 0
icol = 0
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
mag = ps1star.median[ps1band[band]] # r-band
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ix,iy = int(xpos), int(ypos)
# create a little tractor Image object around the star
slc = (slice(max(iy-stampsize, 0), min(iy+stampsize+1, H)),
slice(max(ix-stampsize, 0), min(ix+stampsize+1, W)))
# The PSF model 'const2Psf' is the one used in DR1: a 2-component
# Gaussian fit to PsfEx instantiated in the image center.
tim = im.get_tractor_image(slc=slc, **get_tim_kwargs)
stamp = tim.getImage()
ivarstamp = tim.getInvvar()
# Initialize a tractor PointSource from PS1 measurements
flux = NanoMaggies.magToNanomaggies(mag)
star = PointSource(RaDecPos(ra,dec), NanoMaggies(**{band: flux}))
# Fit just the source RA,Dec,flux.
tractor = Tractor([tim], [star])
tractor.freezeParam('images')
print('2-component MOG:', tim.psf)
tractor.printThawedParams()
for step in range(50):
dlnp,X,alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_mog = tractor.getModelImage(0)
chi2_mog = -2.0*tractor.getLogLikelihood()
mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
# Now change the PSF model to a pixelized PSF model from PsfEx instantiated
# at this place in the image.
psf = PixelizedPsfEx(im.psffn)
tim.psf = psf.constantPsfAt(xpos, ypos)
#print('PSF model:', tim.psf)
#tractor.printThawedParams()
for step in range(50):
dlnp,X,alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_psfex = tractor.getModelImage(0)
chi2_psfex = -2.0*tractor.getLogLikelihood()
mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
#mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95])
sig = np.std((stamp-model_psfex)[ivarstamp>0])
mn, mx = [-2.0*sig,5*sig]
# Generate a QAplot.
if (istar>0) and (istar%(ncols)==0):
irow = irow+1
icol = 3*istar - 3*ncols*irow
#print(istar, irow, icol, icol+1, icol+2)
ax1 = plt.subplot2grid((nrows,3*ncols), (irow,icol), aspect='equal')
ax1.axis('off')
#ax1.imshow(stamp, **tim.ima)
ax1.imshow(stamp,cmap='gray',interpolation='nearest',
origin='lower',vmin=mn,vmax=mx)
ax1.text(0.1,0.9,'{:2d}'.format(istar+1),color='white',
horizontalalignment='left',verticalalignment='top',
transform=ax1.transAxes)
ax2 = plt.subplot2grid((nrows,3*ncols), (irow,icol+1), aspect='equal')
ax2.axis('off')
#ax2.imshow(stamp-model_mog, **tim.ima)
ax2.imshow(stamp-model_mog,cmap='gray',interpolation='nearest',
origin='lower',vmin=mn,vmax=mx)
ax2.text(0.1,0.9,'MoG',color='white',
horizontalalignment='left',verticalalignment='top',
transform=ax2.transAxes)
ax2.text(0.08,0.08,'{:.3f}'.format(mag_mog),color='white',
horizontalalignment='left',verticalalignment='bottom',
transform=ax2.transAxes)
#ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14)
#ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex))
ax3 = plt.subplot2grid((nrows,3*ncols), (irow,icol+2), aspect='equal')
ax3.axis('off')
#ax3.imshow(stamp-model_psfex, **tim.ima)
ax3.imshow(stamp-model_psfex,cmap='gray',interpolation='nearest',
origin='lower',vmin=mn,vmax=mx)
ax3.text(0.1,0.9,'PSFEx',color='white',
horizontalalignment='left',verticalalignment='top',
transform=ax3.transAxes)
ax3.text(0.08,0.08,'{:.3f}'.format(mag_psfex),color='white',
horizontalalignment='left',verticalalignment='bottom',
transform=ax3.transAxes)
if istar==(nstar-1):
break
fig.savefig(pngprefix+'-stargrid.png',bbox_inches='tight')
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'--expnum',
type=str,
help='Run specified exposure numbers (can be comma-separated list')
parser.add_argument(
'--all-found',
action='store_true',
default=False,
help='Only write output if all required input files are found')
parser.add_argument('--ccds',
help='Set ccds.fits file to load, default is all')
parser.add_argument('--continue',
dest='con',
help='Continue even if one exposure is bad',
action='store_true',
default=False)
parser.add_argument('--outdir',
help='Output directory, default %(default)s',
default='calib')
opt = parser.parse_args()
survey = LegacySurveyData()
if opt.ccds:
ccds = fits_table(opt.ccds)
ccds = survey.cleanup_ccds_table(ccds)
survey.ccds = ccds
if opt.expnum is not None:
expnums = [(None, int(x, 10)) for x in opt.expnum.split(',')]
else:
ccds = survey.get_ccds()
expnums = set(zip(ccds.camera, ccds.expnum))
print(len(expnums), 'unique camera+expnums')
for i, (camera, expnum) in enumerate(expnums):
print()
print('Exposure', i + 1, 'of', len(expnums), ':', camera, 'expnum',
expnum)
if camera is None:
C = survey.find_ccds(expnum=expnum)
print(len(C), 'CCDs with expnum', expnum)
camera = C.camera[0]
print('Set camera to', camera)
C = survey.find_ccds(expnum=expnum, camera=camera)
print(len(C), 'CCDs with expnum', expnum, 'and camera', camera)
im0 = survey.get_image_object(C[0])
skyoutfn = im0.merged_skyfn
psfoutfn = im0.merged_psffn
print('Checking for', skyoutfn)
print('Checking for', psfoutfn)
if os.path.exists(skyoutfn) and os.path.exists(psfoutfn):
print('Exposure', expnum, 'is done already')
continue
if not os.path.exists(skyoutfn):
try:
merge_splinesky(survey, expnum, C, skyoutfn, opt)
except:
if not opt.con:
raise
import traceback
traceback.print_exc()
print('Exposure failed:', expnum, '. Continuing...')
if not os.path.exists(psfoutfn):
try:
merge_psfex(survey, expnum, C, psfoutfn, opt)
except:
if not opt.con:
raise
import traceback
traceback.print_exc()
print('Exposure failed:', expnum, '. Continuing...')
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(
description=
"This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
)
parser.add_argument('--calibs',
action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper',
type=int,
default=None,
help='Batch N calibs per line')
parser.add_argument('--forced',
action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb',
action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--touching',
action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--near',
action='store_true',
help='Quick cut to only CCDs near selected bricks')
parser.add_argument('--check',
action='store_true',
help='Check which calibrations actually need to run.')
parser.add_argument('--check-coadd',
action='store_true',
help='Check which caoadds actually need to run.')
parser.add_argument('--out',
help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command',
action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--opt', help='With --command, extra options to add')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--ignore_cuts',
action='store_true',
default=False,
help='no photometric or blacklist cuts')
parser.add_argument('--save_to_fits',
action='store_true',
default=False,
help='save cut brick,ccd to fits table')
parser.add_argument(
'--name',
action='store',
default='dr3',
help='save with this suffix, e.g. refers to ccds table')
parser.add_argument('--delete-sky',
action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--delete-pvastrom',
action='store_true',
help='Delete any existing PV WCS calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
parser.add_argument(
'--brickq',
type=int,
default=None,
help='Queue only bricks with the given "brickq" value [0 to 3]')
parser.add_argument(
'--brickq-deps',
action='store_true',
default=False,
help='Queue bricks directly using qdo API, setting brickq dependencies'
)
parser.add_argument('--queue',
default='bricks',
help='With --brickq-deps, the QDO queue name to use')
opt = parser.parse_args()
survey = LegacySurveyData()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = survey.get_bricks()
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
if opt.ignore_cuts == False:
I = survey.photometric_ccds(T)
print(len(I), 'CCDs are photometric')
T.cut(I)
I = survey.apply_blacklist(T)
print(len(I), 'CCDs are not blacklisted')
T.cut(I)
print(len(T), 'CCDs remain')
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo, dhi = -25, 40
rlo, rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(
test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo, dhi = -90, 90
rlo, rhi = 0, 360
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo, rhi = 240, 245
dlo, dhi = 5, 12
elif opt.region == 'edrplus':
rlo, rhi = 235, 248
dlo, dhi = 5, 15
elif opt.region == 'edr-south':
rlo, rhi = 240, 245
dlo, dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo, rhi = 149.75, 150.75
dlo, dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo, rhi = 240, 250
dlo, dhi = 10, 15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo, rhi = 320., 360.
dlo, dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo, rhi = 240, 360
dlo, dhi = 20, 40
elif opt.region == 'north':
rlo, rhi = 120, 240
dlo, dhi = 20, 40
elif opt.region == 'northeast':
rlo, rhi = 0, 120
dlo, dhi = 20, 40
elif opt.region == 'southwest':
rlo, rhi = 240, 360
dlo, dhi = -20, 0
elif opt.region == 'south':
rlo, rhi = 120, 240
dlo, dhi = -20, 0
elif opt.region == 'southeast':
rlo, rhi = 0, 120
dlo, dhi = -20, 0
elif opt.region == 'southsoutheast':
rlo, rhi = 0, 120
dlo, dhi = -20, -10
elif opt.region == 'midwest':
rlo, rhi = 240, 360
dlo, dhi = 0, 20
elif opt.region == 'middle':
rlo, rhi = 120, 240
dlo, dhi = 0, 20
elif opt.region == 'mideast':
rlo, rhi = 0, 120
dlo, dhi = 0, 20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo, rhi = 250, 260
dlo, dhi = 30, 35
elif opt.region == 'deep2f3':
rlo, rhi = 351.25, 353.75
dlo, dhi = 0, 0.5
elif opt.region == 'virgo':
rlo, rhi = 185, 190
dlo, dhi = 10, 15
elif opt.region == 'virgo2':
rlo, rhi = 182, 192
dlo, dhi = 8, 18
elif opt.region == 'lsb':
rlo, rhi = 147.2, 147.8
dlo, dhi = -0.4, 0.4
elif opt.region == 'eboss-sgc':
# generous boundaries to make sure get all relevant images
# RA -45 to +45
# Dec -5 to +7
rlo, rhi = 310., 50.
dlo, dhi = -6., 6.
elif opt.region == 'eboss-ngc':
# generous boundaries to make sure get all relevant images
# NGC ELGs
# RA 115 to 175
# Dec 15 to 30
rlo, rhi = 122., 177.
dlo, dhi = 12., 32.
elif opt.region == 'mzls':
dlo, dhi = 30., 90.
elif opt.region == 'dr4-bootes':
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
#dlo,dhi = 34., 35.
#rlo,rhi = 209.5, 210.5
dlo, dhi = 33., 36.
rlo, rhi = 216.5, 219.5
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(
np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
(B.dec <= dhi))
log(len(B), 'bricks in range')
#for name in B.get('brickname'):
#print(name)
B.writeto('bricks-cut.fits')
I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs')
plt.clf()
plt.plot(B.ra, B.dec, 'b.')
plt.title('DR3 bricks')
plt.axis([360, 0, np.min(B.dec) - 1, np.max(B.dec) + 1])
plt.savefig('bricks.png')
if opt.brickq is not None:
B.cut(B.brickq == opt.brickq)
log('Cut to', len(B), 'with brickq =', opt.brickq)
if opt.touching:
keep = np.zeros(len(T), bool)
for j in J:
keep[j] = True
T.cut(keep)
log('Cut to', len(T), 'CCDs near bricks')
# Aside -- how many near DR1=1 CCDs?
if False:
T2 = D.get_ccds()
log(len(T2), 'CCDs')
T2.cut(T2.dr1 == 1)
log(len(T2), 'CCDs marked DR1=1')
log(len(B), 'bricks in range')
I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B2 = B[keep]
log('Total of', len(B2), 'bricks near CCDs with DR1=1')
for band in 'grz':
Tb = T2[T2.filter == band]
log(len(Tb), 'in filter', band)
I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec,
survey.bricksize)
good = np.zeros(len(B2), np.uint8)
for i in I:
good[i] = 1
B2.set('has_' + band, good)
B2.writeto('survey-bricks-in-dr1.fits')
sys.exit(0)
# sort by dec decreasing
#B.cut(np.argsort(-B.dec))
# RA increasing
B.cut(np.argsort(B.ra))
for b in B:
if opt.check:
fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3],
b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
if opt.check_coadd:
fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (
b.brickname[:3], b.brickname, b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
#print(b.brickname)
if opt.save_to_fits:
assert (opt.touching)
# Write cut tables to file
for tab, typ in zip([B, T], ['bricks', 'ccds']):
fn = '%s-%s-cut.fits' % (typ, opt.region)
if os.path.exists(fn):
os.remove(fn)
tab.writeto(fn)
print('Wrote %s' % fn)
# Write text files listing ccd and filename names
nm1, nm2 = 'ccds-%s.txt' % opt.region, 'filenames-%s.txt' % opt.region
if os.path.exists(nm1):
os.remove(nm1)
if os.path.exists(nm2):
os.remove(nm2)
f1, f2 = open(nm1, 'w'), open(nm2, 'w')
fns = list(set(T.get('image_filename')))
for fn in fns:
f2.write('%s\n' % fn.strip())
for ti in T:
f1.write('%s\n' % ti.get('image_filename').strip())
f1.close()
f2.close()
print('Wrote *-names.txt')
if opt.brickq_deps:
import qdo
from legacypipe.survey import on_bricks_dependencies
#... find Queue...
q = qdo.connect(opt.queue, create_ok=True)
print('Connected to QDO queue', opt.queue, q)
brick_to_task = dict()
I = survey.photometric_ccds(T)
print(len(I), 'CCDs are photometric')
T.cut(I)
I = survey.apply_blacklist(T)
print(len(I), 'CCDs are not blacklisted')
T.cut(I)
print(len(T), 'CCDs remaining')
T.wra = T.ra + (T.ra > 180) * -360
wra = rlo - 360
plt.clf()
plt.plot(T.wra, T.dec, 'b.')
ax = [wra, rhi, dlo, dhi]
plt.axis(ax)
plt.title('CCDs')
plt.savefig('q-ccds.png')
B.wra = B.ra + (B.ra > 180) * -360
# this slight overestimate (for DECam images) is fine
radius = 0.3
Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius, indexlist=True)
ikeep = []
for ib, (b, Iccd) in enumerate(zip(B, Iccds)):
if Iccd is None or len(Iccd) == 0:
print('No matched CCDs to brick', b.brickname)
continue
wcs = wcs_for_brick(b)
cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
print(len(cI), 'CCDs touching brick', b.brickname)
if len(cI) == 0:
continue
ikeep.append(ib)
B.cut(np.array(ikeep))
print('Cut to', len(B), 'bricks touched by CCDs')
for brickq in range(4):
I = np.flatnonzero(B.brickq == brickq)
print(len(I), 'bricks with brickq =', brickq)
J = np.flatnonzero(B.brickq < brickq)
preB = B[J]
reqs = []
if brickq > 0:
for b in B[I]:
# find brick dependencies
brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
# convert to task ids
taskdeps = [
brick_to_task.get(b.brickname, None) for b in brickdeps
]
# If we dropped a dependency brick from a previous brickq because
# of no overlapping CCDs, it won't appear in the brick_to_task map.
taskdeps = [t for t in taskdeps if t is not None]
reqs.append(taskdeps)
plt.clf()
plt.plot(B.wra, B.dec, '.', color='0.5')
plt.plot(B.wra[I], B.dec[I], 'b.')
plt.axis(ax)
plt.title('Bricks: brickq=%i' % brickq)
plt.savefig('q-bricks-%i.png' % brickq)
# submit to qdo queue
print('Queuing', len(B[I]), 'bricks')
if brickq == 0:
reqs = None
else:
assert (len(I) == len(reqs))
taskids = q.add_multiple(B.brickname[I], requires=reqs)
assert (len(taskids) == len(I))
print('Queued', len(taskids), 'bricks')
brick_to_task.update(dict(zip(B.brickname[I], taskids)))
if not (opt.calibs or opt.forced or opt.lsb):
sys.exit(0)
bands = 'grz'
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
if opt.touching:
allI = set()
for b in B:
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickid,
'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
if len(I) == 0:
continue
allI.update(I)
allI = list(allI)
allI.sort()
elif opt.near:
# Roughly brick radius + DECam image radius
radius = 0.35
allI, nil, nil = match_radec(T.ra,
T.dec,
B.ra,
B.dec,
radius,
nearest=True)
else:
allI = np.arange(len(T))
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
expstr = '%08i' % T.expnum[i]
outfn = os.path.join(
'forced', expstr[:5], expstr,
'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i]))
imgfn = os.path.join(survey.survey_dir, 'images',
T.image_filename[i].strip())
if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
and os.path.exists(imgfn[:-3])):
imgfn = imgfn[:-3]
#f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
# (imgfn, T.image_hdu[i], outfn))
f.write(
'python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n'
% (T.expnum[i], T.ccdname[i], outfn))
f.close()
log('Wrote', opt.out)
sys.exit(0)
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write(
'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
% (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
sys.exit(0)
log('Writing calibs to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if cmd is None:
cmd = ''
f.write(cmd + ' '.join(batch) + '\n')
cmd = None
if opt.command:
cmd = 'python legacypipe/run-calib.py '
if opt.opt is not None:
cmd += opt.opt + ' '
for j, i in enumerate(allI):
if opt.delete_sky or opt.delete_pvastrom:
log(j + 1, 'of', len(allI))
im = survey.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
log(' deleting:', im.pvwcsfn)
os.unlink(im.pvwcsfn)
if opt.check:
log(j + 1, 'of', len(allI))
im = survey.get_image_object(T[i])
if not im.run_calibs(im, just_check=True):
log('Calibs for', im.expnum, im.ccdname, im.calname,
'already done')
continue
if opt.command:
s = '%i-%s' % (T.expnum[i], T.ccdname[i])
prefix = 'python legacypipe/run-calib.py ' + opt.opt
#('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
# (T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
prefix = ''
if j < 10:
print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname',
T.ccdname[i], 'filename', T.image_filename[i])
if not opt.nper:
f.write(prefix + s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, cmd)
batch = []
if opt.check:
f.flush()
if len(batch):
write_batch(f, batch, cmd)
f.close()
log('Wrote', opt.out)
return 0
