def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(
description=
"This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
)
parser.add_argument('--calibs',
action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper',
type=int,
default=None,
help='Batch N calibs per line')
parser.add_argument(
'--byexp',
action='store_true',
default=False,
help='Run one whole exposure per job (not one CCD per job)')
parser.add_argument('--forced',
action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb',
action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--stage', help='Stage image files to given directory')
parser.add_argument('--touching',
action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--near',
action='store_true',
help='Quick cut to only CCDs near selected bricks')
parser.add_argument('--check-coadd',
action='store_true',
help='Check which coadds actually need to run.')
parser.add_argument('--out',
help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command',
action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--opt', help='With --command, extra options to add')
parser.add_argument('--maxra', type=float, help='Maximum RA to run')
parser.add_argument('--minra', type=float, help='Minimum RA to run')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--ignore_cuts',
action='store_true',
default=False,
help='no photometric cuts')
parser.add_argument('--save_to_fits',
action='store_true',
default=False,
help='save cut brick,ccd to fits table')
parser.add_argument(
'--name',
action='store',
default='dr3',
help='save with this suffix, e.g. refers to ccds table')
parser.add_argument('--delete-sky',
action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
parser.add_argument('--nccds',
action='store_true',
default=False,
help='Prints number of CCDs per brick')
parser.add_argument('--bands',
default='g,r,z',
help='Set bands to keep: comma-separated list.')
opt = parser.parse_args()
want_ccds = (opt.calibs or opt.forced or opt.lsb)
want_bricks = not want_ccds
survey = LegacySurveyData()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = survey.get_bricks()
log('Bricks Dec range:', B.dec.min(), B.dec.max())
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
if opt.ignore_cuts == False:
log('Applying CCD cuts...')
if 'ccd_cuts' in T.columns():
T.cut(T.ccd_cuts == 0)
log(len(T), 'CCDs survive cuts')
bands = opt.bands.split(',')
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
log('CCDs Dec range:', T.dec.min(), T.dec.max())
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo, dhi = -25, 40
rlo, rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(
test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo, dhi = -90, 90
rlo, rhi = 0, 360
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo, rhi = 240, 245
dlo, dhi = 5, 12
elif opt.region == 'dr8-decam':
rlo, rhi = 0, 360
dlo, dhi = -70, 40
log('DR8-DECam region')
elif opt.region == 'edrplus':
rlo, rhi = 235, 248
dlo, dhi = 5, 15
elif opt.region == 'edr-south':
rlo, rhi = 240, 245
dlo, dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo, rhi = 149.75, 150.75
dlo, dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo, rhi = 240, 250
dlo, dhi = 10, 15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo, rhi = 320., 360.
dlo, dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo, rhi = 240, 360
dlo, dhi = 20, 40
elif opt.region == 'north':
rlo, rhi = 120, 240
dlo, dhi = 20, 40
elif opt.region == 'northeast':
rlo, rhi = 0, 120
dlo, dhi = 20, 40
elif opt.region == 'southwest':
rlo, rhi = 240, 360
dlo, dhi = -20, 0
elif opt.region == 'south':
rlo, rhi = 120, 240
dlo, dhi = -20, 0
elif opt.region == 'southeast':
rlo, rhi = 0, 120
dlo, dhi = -20, 0
elif opt.region == 'southsoutheast':
rlo, rhi = 0, 120
dlo, dhi = -20, -10
elif opt.region == 'midwest':
rlo, rhi = 240, 360
dlo, dhi = 0, 20
elif opt.region == 'middle':
rlo, rhi = 120, 240
dlo, dhi = 0, 20
elif opt.region == 'mideast':
rlo, rhi = 0, 120
dlo, dhi = 0, 20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo, rhi = 250, 260
dlo, dhi = 30, 35
elif opt.region == 'deep2f2':
rlo, rhi = 251.4, 254.4
dlo, dhi = 34.6, 35.3
elif opt.region == 'deep2f3':
rlo, rhi = 351.25, 353.75
dlo, dhi = 0, 0.5
elif opt.region == 'deep3':
rlo, rhi = 214, 216
dlo, dhi = 52.25, 53.25
elif opt.region == 'virgo':
rlo, rhi = 185, 190
dlo, dhi = 10, 15
elif opt.region == 'virgo2':
rlo, rhi = 182, 192
dlo, dhi = 8, 18
elif opt.region == 'coma':
# van Dokkum et al Coma cluster ultra-diffuse galaxies: 3x3 field centered on Coma cluster
rc, dc = 195., 28.
dd = 1.5
cosdec = np.cos(np.deg2rad(dc))
rlo, rhi = rc - dd / cosdec, rc + dd / cosdec
dlo, dhi = dc - dd, dc + dd
elif opt.region == 'lsb':
rlo, rhi = 147.2, 147.8
dlo, dhi = -0.4, 0.4
elif opt.region == 'eboss-sgc':
# generous boundaries to make sure get all relevant images
# RA -45 to +45
# Dec -5 to +7
rlo, rhi = 310., 50.
dlo, dhi = -6., 6.
elif opt.region == 'eboss-ngc':
# generous boundaries to make sure get all relevant images
# NGC ELGs
# RA 115 to 175
# Dec 15 to 30
# rlo,rhi = 122., 177.
# dlo,dhi = 12., 32.
rlo, rhi = 126., 168.
dlo, dhi = 18., 33.
elif opt.region == 'mzls':
dlo, dhi = -10., 90. # -10: pull in Stripe 82 data too
elif opt.region == 'dr4-bootes':
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
#dlo,dhi = 34., 35.
#rlo,rhi = 209.5, 210.5
dlo, dhi = 33., 36.
rlo, rhi = 216.5, 219.5
elif opt.region == 'des-sn-x3':
#rlo,rhi = 36., 37.
#dlo,dhi = -5., -4.
rlo, rhi = 36., 36.5
dlo, dhi = -4.5, -4.
elif opt.region == 'ngc2632':
# open cluster
rlo, rhi = 129.0, 131.0
dlo, dhi = 19.0, 20.5
elif opt.region == 'dr8sky':
rlo, rhi = 35.0, 37.0
dlo, dhi = -3.0, -1.0
# ADM DR8 test regions, see, e.g.:
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR8#Testregions
elif opt.region == 'dr8-test-s82':
rlo, rhi = 0, 45
dlo, dhi = -1.25, 1.25
elif opt.region == 'dr8-test-hsc-sgc':
rlo, rhi = 30, 40
dlo, dhi = -6.5, -1.25
elif opt.region == 'dr8-test-hsc-ngc':
rlo, rhi = 177.5, 182.5
dlo, dhi = -1, 1
elif opt.region == 'dr8-test-edr':
rlo, rhi = 240, 245
dlo, dhi = 5, 12
elif opt.region == 'dr8-test-hsc-north':
rlo, rhi = 240, 250
dlo, dhi = 42, 45
elif opt.region == 'dr8-test-deep2-egs':
rlo, rhi = 213, 216.5
dlo, dhi = 52, 54
elif opt.region == 'dr8-test-overlap':
rlo, rhi = 132, 140.5
dlo, dhi = 31.5, 35
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if opt.minra is not None:
rlo = opt.minra
if opt.maxra is not None:
rhi = opt.maxra
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(
np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
(B.dec <= dhi))
log(len(B), 'bricks in range; cut Dec range', B.dec.min(), B.dec.max())
#for name in B.get('brickname'):
# print(name)
#B.writeto('bricks-cut.fits')
bricksize = 0.25
# A bit more than 0.25-degree brick radius + Bok image radius ~ 0.57
search_radius = 1.05 * np.sqrt(2.) * (bricksize +
(0.455 * 4096 / 3600.)) / 2.
log(len(T), 'CCDs')
log(len(B), 'Bricks')
I, J, d = match_radec(B.ra,
B.dec,
T.ra,
T.dec,
search_radius,
nearest=True)
B.cut(I)
log('Cut to', len(B), 'bricks near CCDs')
log('Bricks Dec range:', B.dec.min(), B.dec.max())
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# plt.title('DR3 bricks')
# plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
# plt.savefig('bricks.png')
if opt.touching:
I, J, d = match_radec(T.ra,
T.dec,
B.ra,
B.dec,
search_radius,
nearest=True)
# list the ones that will be cut
# drop = np.ones(len(T))
# drop[I] = False
# for i in np.flatnonzero(drop):
# from astrometry.util.starutil_numpy import degrees_between
# dists = degrees_between(B.ra, B.dec, T.ra[i], T.dec[i])
# mindist = min(dists)
# print('Dropping:', T.ra[i], T.dec[i], 'min dist', mindist, 'search_radius', search_radius)
T.cut(I)
log('Cut to', len(T), 'CCDs near bricks')
# sort by RA increasing
B.cut(np.argsort(B.ra))
if opt.save_to_fits:
assert (opt.touching)
# Write cut tables to file
for tab, typ in zip([B, T], ['bricks', 'ccds']):
fn = '%s-%s-cut.fits' % (typ, opt.region)
if os.path.exists(fn):
os.remove(fn)
tab.writeto(fn)
log('Wrote %s' % fn)
# Write text files listing ccd and filename names
# nm1,nm2= 'ccds-%s.txt'% opt.region,'filenames-%s.txt' % opt.region
# if os.path.exists(nm1):
# os.remove(nm1)
# if os.path.exists(nm2):
# os.remove(nm2)
# f1,f2=open(nm1,'w'),open(nm2,'w')
# fns= list(set(T.get('image_filename')))
# for fn in fns:
# f2.write('%s\n' % fn.strip())
# for ti in T:
# f1.write('%s\n' % ti.get('image_filename').strip())
# f1.close()
# f2.close()
# log('Wrote *-names.txt')
if opt.touching:
if want_bricks:
# Shortcut the list of bricks that are definitely touching CCDs --
# a brick-ccd pair within this radius must be touching.
closest_radius = 0.95 * (bricksize + 0.262 * 2048 / 3600.) / 2.
J1, nil, nil = match_radec(B.ra,
B.dec,
T.ra,
T.dec,
closest_radius,
nearest=True)
log(len(J1), 'of', len(B), 'bricks definitely touch CCDs')
tocheck = np.ones(len(B), bool)
tocheck[J1] = False
J2 = []
for j in np.flatnonzero(tocheck):
b = B[j]
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickname)
if len(I) == 0:
continue
J2.append(j)
J = np.hstack((J1, J2))
J = np.sort(J).astype(int)
B.cut(J)
log('Cut to', len(B), 'bricks touching CCDs')
else:
J = []
allI = set()
for j, b in enumerate(B):
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickname)
if len(I) == 0:
continue
allI.update(I)
J.append(j)
allI = list(allI)
allI.sort()
B.cut(np.array(J))
log('Cut to', len(B), 'bricks touching CCDs')
elif opt.near:
# Find CCDs near bricks
allI, nil, nil = match_radec(T.ra,
T.dec,
B.ra,
B.dec,
search_radius,
nearest=True)
# Find bricks near CCDs
J, nil, nil = match_radec(B.ra,
B.dec,
T.ra,
T.dec,
search_radius,
nearest=True)
B.cut(J)
log('Cut to', len(B), 'bricks near CCDs')
else:
allI = np.arange(len(T))
if opt.byexp:
nil, eI = np.unique(T.expnum[allI], return_index=True)
allI = allI[eI]
print('Cut to', len(allI), 'expnums')
if opt.nccds:
from queue import Queue
from threading import Thread
log('Checking number of CCDs per brick')
def worker():
while True:
i = q.get()
if i is None:
break
b = B[i]
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(b.brickname, len(I))
q.task_done()
q = Queue()
num_threads = 24
threads = []
for i in range(num_threads):
t = Thread(target=worker)
t.start()
threads.append(t)
for i in range(len(B)):
q.put(i)
q.join()
for i in range(num_threads):
q.put(None)
for t in threads:
t.join()
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
if want_bricks:
# Print the list of bricks and exit.
for b in B:
print(b.brickname)
if opt.save_to_fits:
B.writeto('bricks-%s-touching.fits' % opt.region)
if not want_ccds:
sys.exit(0)
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.stage is not None:
cmd_pat = 'rsync -LRarv %s %s'
fns = set()
for iccd in allI:
im = survey.get_image_object(T[iccd])
fns.update([
im.imgfn, im.wtfn, im.dqfn, im.psffn, im.merged_psffn,
im.merged_splineskyfn, im.splineskyfn
])
for i, fn in enumerate(fns):
print('File', i + 1, 'of', len(fns), ':', fn)
if not os.path.exists(fn):
print('No such file:', fn)
continue
base = survey.get_survey_dir()
if base.endswith('/'):
base = base[:-1]
rel = os.path.relpath(fn, base)
dest = os.path.join(opt.stage, rel)
print('Dest:', dest)
if os.path.exists(dest):
print('Exists:', dest)
continue
cmd = cmd_pat % ('%s/./%s' % (base, rel), opt.stage)
print(cmd)
rtn = os.system(cmd)
assert (rtn == 0)
sys.exit(0)
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
expstr = '%08i' % T.expnum[i]
imgfn = os.path.join(survey.survey_dir, 'images',
T.image_filename[i].strip())
if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
and os.path.exists(imgfn[:-3])):
imgfn = imgfn[:-3]
outfn = os.path.join(
expstr[:5], expstr, 'forced-%s-%s-%s.fits' %
(T.camera[i].strip(), expstr, T.ccdname[i]))
f.write(
'python legacypipe/forced_photom.py --apphot --derivs --catalog-dir /project/projectdirs/cosmo/data/legacysurvey/dr7/ %i %s forced/%s\n'
% (T.expnum[i], T.ccdname[i], outfn))
f.close()
log('Wrote', opt.out)
fn = 'forced-ccds.fits'
T[allI].writeto(fn)
print('Wrote', fn)
sys.exit(0)
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write(
'python legacyanalysis/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
% (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
sys.exit(0)
log('Writing calibs to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if cmd is None:
cmd = ''
f.write(cmd + ' '.join(batch) + '\n')
cmd = None
if opt.command:
cmd = 'python legacypipe/run-calib.py '
if opt.opt is not None:
cmd += opt.opt + ' '
for j, i in enumerate(allI):
if opt.delete_sky:
log(j + 1, 'of', len(allI))
im = survey.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.command:
if opt.byexp:
s = '--expnum %i' % (T.expnum[i])
else:
s = '%i-%s' % (T.expnum[i], T.ccdname[i])
prefix = 'python legacypipe/run-calib.py '
if opt.opt is not None:
prefix = prefix + opt.opt
#('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
# (T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
prefix = ''
if j < 10:
print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname',
T.ccdname[i], 'filename', T.image_filename[i])
if not opt.nper:
f.write(prefix + s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, cmd)
batch = []
if len(batch):
write_batch(f, batch, cmd)
f.close()
log('Wrote', opt.out)
return 0
def main():
import argparse
parser = argparse.ArgumentParser(
description='This script creates small self-contained data sets that '
'are useful for test cases of the pipeline codes.')
parser.add_argument('ccds', help='CCDs table describing region to grab')
parser.add_argument('outdir', help='Output directory name')
parser.add_argument('brick', help='Brick containing these images')
parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image')
parser.add_argument('--fpack', action='store_true', default=False)
parser.add_argument('--pad', action='store_true', default=False,
help='Keep original image size, but zero out pixels outside ROI')
args = parser.parse_args()
C = fits_table(args.ccds)
print(len(C), 'CCDs in', args.ccds)
C.camera = np.array([c.strip() for c in C.camera])
survey = LegacySurveyData()
bricks = survey.get_bricks_readonly()
outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])]
assert(len(outbricks) == 1)
outsurvey = LegacySurveyData(survey_dir = args.outdir)
trymakedirs(args.outdir)
outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz'))
targetwcs = wcs_for_brick(outbricks[0])
H,W = targetwcs.shape
tycho = fits_table(os.path.join(survey.get_survey_dir(), 'tycho2.fits.gz'))
print('Read', len(tycho), 'Tycho-2 stars')
ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec)
margin = 100
tycho.cut(ok * (tx > -margin) * (tx < W+margin) *
(ty > -margin) * (ty < H+margin))
print('Cut to', len(tycho), 'Tycho-2 stars within brick')
del ok,tx,ty
tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz'))
outccds = C.copy()
for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1',
'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1',
'plver', 'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver',
'psfplver' ]:
outccds.delete_column(c)
outccds.image_hdu[:] = 1
# Convert to list to avoid truncating filenames
outccds.image_filename = [fn for fn in outccds.image_filename]
for iccd,ccd in enumerate(C):
decam = (ccd.camera.strip() == 'decam')
bok = (ccd.camera.strip() == '90prime')
im = survey.get_image_object(ccd)
print('Got', im)
slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1))
tim = im.get_tractor_image(slc, pixPsf=True, splinesky=True,
subsky=False, nanomaggies=False)
print('Tim:', tim.shape)
psf = tim.getPsf()
print('PSF:', psf)
psfex = psf.psfex
print('PsfEx:', psfex)
outim = outsurvey.get_image_object(ccd)
print('Output image:', outim)
print('Image filename:', outim.imgfn)
trymakedirs(outim.imgfn, dir=True)
imgdata = tim.getImage()
dqdata = tim.dq
if decam:
# DECam-specific code remaps the DQ codes... re-read raw
print('Reading data quality from', im.dqfn, 'hdu', im.hdu)
dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice)
ivdata = tim.getInvvar()
if args.pad:
# Create zero image of full size, copy in data.
fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype)
fullsize[slc] = imgdata
imgdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype)
fullsize[slc] = dqdata
dqdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype)
fullsize[slc] = ivdata
ivdata = fullsize
else:
# Adjust the header WCS by x0,y0
crpix1 = tim.hdr['CRPIX1']
crpix2 = tim.hdr['CRPIX2']
tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0
tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0
# Add image extension to filename
# fitsio doesn't compress .fz by default, so drop .fz suffix
outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits')
# if bok:
# outim.whtfn = outim.whtfn .replace('.wht.fits', '-%s.wht.fits' % im.ccdname)
# if not args.fpack:
# outim.whtfn = outim.whtfn .replace('.fits.fz', '.fits')
# else:
if True:
outim.wtfn = outim.wtfn .replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.wtfn = outim.wtfn .replace('.fits.fz', '.fits')
if outim.dqfn is not None:
outim.dqfn = outim.dqfn .replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.dqfn = outim.dqfn .replace('.fits.fz', '.fits')
if bok:
outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname)
ccdfn = outim.imgfn
ccdfn = ccdfn.replace(outsurvey.get_image_dir(),'')
if ccdfn.startswith('/'):
ccdfn = ccdfn[1:]
outccds.image_filename[iccd] = ccdfn
print('Changed output filenames to:')
print(outim.imgfn)
print(outim.dqfn)
ofn = outim.imgfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fitsio.write(ofn, None, header=tim.primhdr, clobber=True)
fitsio.write(ofn, imgdata, header=tim.hdr, extname=ccd.ccdname)
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
h,w = tim.shape
if not args.pad:
outccds.width[iccd] = w
outccds.height[iccd] = h
outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0
outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0
wcs = Tan(*[float(x) for x in
[ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2,
ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]])
if args.pad:
subwcs = wcs
else:
subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h)
outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center()
#if not bok:
if True:
print('Weight filename:', outim.wtfn)
wfn = outim.wtfn
# else:
# print('Weight filename:', outim.whtfn)
# wfn = outim.whtfn
trymakedirs(wfn, dir=True)
ofn = wfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fitsio.write(ofn, None, header=tim.primhdr, clobber=True)
fitsio.write(ofn, ivdata, header=tim.hdr, extname=ccd.ccdname)
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
if outim.dqfn is not None:
print('DQ filename', outim.dqfn)
trymakedirs(outim.dqfn, dir=True)
ofn = outim.dqfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fitsio.write(ofn, None, header=tim.primhdr, clobber=True)
fitsio.write(ofn, dqdata, header=tim.hdr, extname=ccd.ccdname)
if args.fpack:
cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
print('PSF filename:', outim.psffn)
trymakedirs(outim.psffn, dir=True)
psfex.writeto(outim.psffn)
if not bok:
print('Sky filename:', outim.splineskyfn)
sky = tim.getSky()
print('Sky:', sky)
trymakedirs(outim.splineskyfn, dir=True)
sky.write_fits(outim.splineskyfn)
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
# WISE
if args.wise is not None:
from wise.forcedphot import unwise_tiles_touching_wcs
from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs,
get_unwise_tractor_image, get_unwise_tile_dir)
# Read WCS...
print('Reading TAN wcs header from', args.wise)
targetwcs = Tan(args.wise)
tiles = unwise_tiles_touching_wcs(targetwcs)
print('Cut to', len(tiles), 'unWISE tiles')
H,W = targetwcs.shape
r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]),
np.array([H/2, H/2, 1, H ]))
roiradec = [r[0], r[1], d[2], d[3]]
unwise_dir = os.environ['UNWISE_COADDS_DIR']
wise_out = os.path.join(args.outdir, 'images', 'unwise')
print('Will write WISE outputs to', wise_out)
unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR']
wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr')
print('Will write WISE time-resolved outputs to', wise_tr_out)
W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_neo1-atlas.fits'))
print('Read', len(W), 'time-resolved WISE coadd tiles')
W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id]))
print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth')
# Write the time-resolved index subset.
W.writeto(os.path.join(wise_tr_out, 'time_resolved_neo1-atlas.fits'))
# this ought to be enough for anyone =)
Nepochs = 5
wisedata = []
# full depth
for band in [1,2,3,4]:
wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band))
# time-resolved
for band in [1,2]:
# W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2)
bitmask = (1 << (band-1))
for e in range(Nepochs):
# Which tiles have images for this epoch?
I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask)
if len(I) == 0:
continue
print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I])
edir = os.path.join(unwise_tr_dir, 'e%03i' % e)
eoutdir = os.path.join(wise_tr_out, 'e%03i' % e)
wisedata.append((edir, eoutdir, tiles.coadd_id[I], band))
wrote_masks = set()
for indir, outdir, tiles, band in wisedata:
for tile in tiles:
wanyband = 'w'
tim = get_unwise_tractor_image(indir, tile, band,
bandname=wanyband, roiradecbox=roiradec)
print('Got unWISE tim', tim)
print(tim.shape)
thisdir = get_unwise_tile_dir(outdir, tile)
print('Directory for this WISE tile:', thisdir)
base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band))
print('Base filename:', base)
masked = True
mu = 'm' if masked else 'u'
imfn = base + 'img-%s.fits' % mu
ivfn = base + 'invvar-%s.fits.gz' % mu
nifn = base + 'n-%s.fits.gz' % mu
nufn = base + 'n-u.fits.gz'
#print('WISE image header:', tim.hdr)
# Adjust the header WCS by x0,y0
wcs = tim.wcs.wcs
tim.hdr['CRPIX1'] = wcs.crpix[0]
tim.hdr['CRPIX2'] = wcs.crpix[1]
H,W = tim.shape
tim.hdr['IMAGEW'] = W
tim.hdr['IMAGEH'] = H
print('WCS:', wcs)
print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2'])
trymakedirs(imfn, dir=True)
fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True)
print('Wrote', imfn)
fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True)
print('Wrote', ivfn)
fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True)
print('Wrote', nifn)
fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True)
print('Wrote', nufn)
if not (indir,tile) in wrote_masks:
print('Looking for mask file for', indir, tile)
# record that we tried this dir/tile combo
wrote_masks.add((indir,tile))
for idir in indir.split(':'):
tdir = get_unwise_tile_dir(idir, tile)
maskfn = 'unwise-%s-msk.fits.gz' % tile
fn = os.path.join(tdir, maskfn)
print('Mask file:', fn)
if os.path.exists(fn):
print('Reading', fn)
(x0,x1,y0,y1) = tim.roi
roislice = (slice(y0,y1), slice(x0,x1))
F = fitsio.FITS(fn)[0]
hdr = F.read_header()
M = F[roislice]
outfn = os.path.join(thisdir, maskfn)
fitsio.write(outfn, M, header=tim.hdr, clobber=True)
print('Wrote', outfn)
break
outC = outsurvey.get_ccds_readonly()
for iccd,ccd in enumerate(outC):
outim = outsurvey.get_image_object(ccd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True, splinesky=True)
print('Got output tim:', otim)
def main(survey=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if survey is None:
survey = LegacySurveyData()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(np.array([f.strip() == opt.filename for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert(len(T) == 1)
ccd = T[0]
im = survey.get_image_object(ccd)
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True,
constant_invvar=opt.constant_invvar)
print('Got tim:', tim)
print('Read image:', Time()-t0)
if opt.catfn in ['DR1', 'DR2', 'DR3']:
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=survey)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = survey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
if len(T):
TT.append(T)
if len(TT) == 0:
print('No sources to photometer.')
return 0
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
surveydir = survey.get_survey_dir()
del survey
cat = read_fits_catalog(T)
# print('Got cat:', cat)
print('Read catalog:', Time()-t0)
print('Forced photom...')
opti = None
forced_kwargs = {}
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
#forced_kwargs.update(verbose=True)
for src in cat:
# Limit sizes of huge models
from tractor.galaxy import ProfileGalaxy
if isinstance(src, ProfileGalaxy):
px,py = tim.wcs.positionToPixel(src.getPosition())
h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval)
MAXHALF = 128
if h > MAXHALF:
print('halfsize', h,'for',src,'-> setting to',MAXHALF)
src.halfsize = MAXHALF
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
disable_galaxy_cache()
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32)
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
if opt.forced:
if opt.plots is None:
forced_kwargs.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True, fitstats=True,
shared_params=False, priors=False, **forced_kwargs)
if opt.plots:
(data,mod,ie,chi,roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([src.getBrightness().getFlux(tim.band)
for src in cat]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2 .astype(np.float32)
print('Forced photom:', Time()-t0)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap.astype(np.float32)
ap = 1./(np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap.astype(np.float32)
print('Aperture photom:', Time()-t0)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, surveydir)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE',
value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
comment='Name of this image'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'}
columns = F.get_columns()
for i,col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
print('Getting model image...')
mod = tr.getModelImage(tim)
fitsio.write(opt.save_model, mod, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
print('Finished forced phot:', Time()-t0)
return 0
def main(survey=None, opt=None):
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
import pylab as plt
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.expnum)
filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
filename = opt.expnum
# Try parsing HDU number
try:
hdu = int(opt.ccdname)
ccdname = None
except:
hdu = -1
ccdname = opt.ccdname
if survey is None:
survey = LegacySurveyData()
catsurvey = survey
if opt.catalog_dir is not None:
catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir)
if filename is not None and hdu >= 0:
# FIXME -- try looking up in CCDs file?
# Read metadata from file
print('Warning: faking metadata from file contents')
T = exposure_metadata([filename], hdus=[hdu])
print('Metadata:')
T.about()
if not 'ccdzpt' in T.columns():
phdr = fitsio.read_header(filename)
T.ccdzpt = np.array([phdr['MAGZERO']])
print('WARNING: using header MAGZERO')
T.ccdraoff = np.array([0.])
T.ccddecoff = np.array([0.])
print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if hdu >= 0:
T.cut(T.image_hdu == hdu)
print(len(T), 'with HDU', hdu)
if filename is not None:
T.cut(np.array([f.strip() == filename for f in T.image_filename]))
print(len(T), 'with filename', filename)
if opt.camera is not None:
T.cut(T.camera == opt.camera)
print(len(T), 'with camera', opt.camera)
assert (len(T) == 1)
ccd = T[0]
im = survey.get_image_object(ccd)
if opt.do_calib:
#from legacypipe.survey import run_calibs
#kwa = dict(splinesky=True)
#run_calibs((im, kwa))
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
splinesky=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf)
print('Got tim:', tim)
print('Read image:', Time() - t0)
if opt.catfn in ['DR1', 'DR2', 'DR3', 'DR5', 'DR']:
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = catsurvey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) *
(yy >= -margin) * (yy <= (H + margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
for col in ['out_of_bounds', 'left_blob']:
if col in T.get_columns():
T.cut(T.get(col) == False)
print('Cut to', len(T), 'on', col)
if len(T):
TT.append(T)
if len(TT) == 0:
print('No sources to photometer.')
return 0
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
print('Total of', len(T), 'catalog sources')
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
surveydir = survey.get_survey_dir()
del survey
kwargs = {}
cols = T.get_columns()
if 'flux_r' in cols and not 'decam_flux_r' in cols:
kwargs.update(fluxPrefix='')
cat = read_fits_catalog(T, **kwargs)
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
print('Read catalog:', Time() - t0)
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
ps=ps)
t0 = Time()
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F)).astype(np.int32)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32)
F.ra = T.ra
F.dec = T.dec
ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
F.mask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1),
np.clip(np.round(F.x).astype(int), 0, w - 1)]
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, surveydir)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='EXPOSURE',
value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
comment='Name of this image'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2'
}
columns = F.get_columns()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
print('Getting model image...')
tr = Tractor([tim], cat)
mod = tr.getModelImage(tim)
fitsio.write(opt.save_model, mod, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
print('Finished forced phot:', Time() - t0)
return 0
