def main(survey=None, opt=None, args=None):
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if args is None:
args = sys.argv[1:]
print('forced_photom.py', ' '.join(args))
if opt is None:
parser = get_parser()
opt = parser.parse_args(args)
import logging
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
t0 = Time()
if survey is None:
survey = LegacySurveyData(survey_dir=opt.survey_dir,
cache_dir=opt.cache_dir,
output_dir=opt.out_dir)
if opt.skip:
if opt.out is not None:
outfn = opt.out
else:
outfn = survey.find_file('forced',
output=True,
camera=opt.camera,
expnum=opt.expnum)
if os.path.exists(outfn):
print('Ouput file exists:', outfn)
return 0
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
return -1
if opt.out is None and opt.out_dir is None:
print('Must supply either --out or --out-dir')
return -1
if opt.expnum is None and opt.out is None:
print('If no --expnum is given, must supply --out filename')
return -1
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Cache CCDs files before the find_ccds call...
# Copy required files into the cache?
if opt.pre_cache:
def copy_files_to_cache(fns):
for fn in fns:
cachefn = fn.replace(survey.survey_dir, survey.cache_dir)
if not cachefn.startswith(survey.cache_dir):
print('Skipping', fn)
continue
outdir = os.path.dirname(cachefn)
trymakedirs(outdir)
print('Copy', fn)
print(' to', cachefn)
shutil.copyfile(fn, cachefn)
assert (survey.cache_dir is not None)
fnset = set()
fn = survey.find_file('bricks')
fnset.add(fn)
fns = survey.find_file('ccd-kds')
fnset.update(fns)
copy_files_to_cache(fnset)
# Read metadata from survey-ccds.fits table
ccds = survey.find_ccds(camera=opt.camera,
expnum=opt.expnum,
ccdname=opt.ccdname)
print(len(ccds), 'with camera', opt.camera, 'and expnum', opt.expnum,
'and ccdname', opt.ccdname)
# sort CCDs
ccds.cut(np.lexsort((ccds.ccdname, ccds.expnum, ccds.camera)))
# If there is only one catalog survey_dir, we pass it to get_catalog_in_wcs
# as the northern survey.
catsurvey_north = survey
catsurvey_south = None
if opt.catalog_dir_north is not None:
assert (opt.catalog_dir_south is not None)
assert (opt.catalog_resolve_dec_ngc is not None)
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
elif opt.catalog_dir is not None:
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)
# Copy required CCD & calib files into the cache?
if opt.pre_cache:
assert (survey.cache_dir is not None)
fnset = set()
for ccd in ccds:
im = survey.get_image_object(ccd)
for key in im.get_cacheable_filename_variables():
fn = getattr(im, key)
if fn is None or not (os.path.exists(fn)):
continue
fnset.add(fn)
copy_files_to_cache(fnset)
args = []
for ccd in ccds:
args.append((survey, catsurvey_north, catsurvey_south,
opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))
if opt.threads:
from astrometry.util.multiproc import multiproc
from astrometry.util.timingpool import TimingPool, TimingPoolMeas
pool = TimingPool(opt.threads)
poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(poolmeas)
mp = multiproc(None, pool=pool)
tm = Time()
FF = mp.map(bounce_one_ccd, args)
print('Multi-processing forced-phot:', Time() - tm)
del mp
Time.measurements.remove(poolmeas)
del poolmeas
pool.close()
pool.join()
del pool
else:
FF = map(bounce_one_ccd, args)
FF = [F for F in FF if F is not None]
if len(FF) == 0:
print('No photometry results to write.')
return 0
# Keep only the first header
_, version_hdr, _, _ = FF[0]
# unpack results
outlier_masks = [m for _, _, m, _ in FF]
outlier_hdrs = [h for _, _, _, h in FF]
FF = [F for F, _, _, _ in FF]
F = merge_tables(FF)
if len(ccds):
version_hdr.delete('CPHDU')
version_hdr.delete('CCDNAME')
from legacypipe.utils import add_bits
from legacypipe.bits import DQ_BITS
add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
for i, ap in enumerate(apertures_arcsec):
version_hdr.add_record(
dict(name='APRAD%i' % i,
value=ap,
comment='(optical) Aperture radius, in arcsec'))
unitmap = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2',
'apflux': 'nanomaggy',
'apflux_ivar': '1/nanomaggy^2',
'psfdepth': '1/nanomaggy^2',
'galdepth': '1/nanomaggy^2',
'sky': 'nanomaggy/arcsec^2',
'psfsize': 'arcsec',
'fwhm': 'pixels',
'ccdrarms': 'arcsec',
'ccddecrms': 'arcsec',
'ra': 'deg',
'dec': 'deg',
'skyrms': 'counts/sec',
'dra': 'arcsec',
'ddec': 'arcsec',
'dra_ivar': '1/arcsec^2',
'ddec_ivar': '1/arcsec^2'
}
columns = F.get_columns()
order = [
'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'fwhm', 'ccd_cuts',
'airmass', 'sky', 'skyrms', 'psfdepth', 'galdepth', 'ccdzpt',
'ccdrarms', 'ccddecrms', 'ccdphrms', 'ra', 'dec', 'flux', 'flux_ivar',
'fracflux', 'rchisq', 'fracmasked', 'fracin', 'apflux', 'apflux_ivar',
'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar'
]
columns = [c for c in order if c in columns]
units = [unitmap.get(c, '') for c in columns]
if opt.out is not None:
outdir = os.path.dirname(opt.out)
if len(outdir):
trymakedirs(outdir)
tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.out))
fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
F.writeto(tmpfn, units=units, append=True, columns=columns)
os.rename(tmpfn, opt.out)
print('Wrote', opt.out)
else:
with survey.write_output('forced',
camera=opt.camera,
expnum=opt.expnum) as out:
F.writeto(None,
fits_object=out.fits,
primheader=version_hdr,
units=units,
columns=columns)
print('Wrote', out.real_fn)
if opt.outlier_mask is not None:
# Add outlier bit meanings to the primary header
version_hdr.add_record(
dict(name='COMMENT', value='Outlier mask bit meanings'))
version_hdr.add_record(
dict(name='OUTL_POS',
value=1,
comment='Outlier mask bit for Positive outlier'))
version_hdr.add_record(
dict(name='OUTL_NEG',
value=2,
comment='Outlier mask bit for Negative outlier'))
if opt.outlier_mask == 'default':
outdir = os.path.join(opt.out_dir, 'outlier-masks')
camexp = set(zip(ccds.camera, ccds.expnum))
for c, e in camexp:
I = np.flatnonzero((ccds.camera == c) * (ccds.expnum == e))
ccd = ccds[I[0]]
imfn = ccd.image_filename.strip()
outfn = os.path.join(outdir, imfn.replace('.fits',
'-outlier.fits'))
trymakedirs(outfn, dir=True)
tempfn = outfn.replace('.fits', '-tmp.fits')
with fitsio.FITS(tempfn, 'rw', clobber=True) as fits:
fits.write(None, header=version_hdr)
for i in I:
mask = outlier_masks[i]
_, _, _, meth, tile = survey.get_compression_args(
'outliers_mask', shape=mask.shape)
fits.write(mask,
header=outlier_hdrs[i],
extname=ccds.ccdname[i],
compress=meth,
tile_dims=tile)
os.rename(tempfn, outfn)
print('Wrote', outfn)
elif opt.outlier_mask is not None:
with fitsio.FITS(opt.outlier_mask, 'rw', clobber=True) as F:
F.write(None, header=version_hdr)
for i, (hdr, mask) in enumerate(zip(outlier_hdrs, outlier_masks)):
_, _, _, meth, tile = survey.get_compression_args(
'outliers_mask', shape=mask.shape)
F.write(mask,
header=hdr,
extname=ccds.ccdname[i],
compress=meth,
tile_dims=tile)
print('Wrote', opt.outlier_mask)
tnow = Time()
print('Total:', tnow - t0)
return 0
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--plots', action='store_true')
parser.add_argument('--brick', help='Brick name to run')
parser.add_argument(
'--input-dir',
default='/global/projecta/projectdirs/cosmo/work/legacysurvey/dr7')
#/global/cscratch1/sd/desiproc/dr7out')
parser.add_argument('--survey-dir',
default='/global/cscratch1/sd/dstn/dr7-depthcut')
parser.add_argument('--output-dir',
default='/global/cscratch1/sd/dstn/bright')
opt = parser.parse_args()
plots = opt.plots
ps = PlotSequence('bright')
brickname = opt.brick
insurvey = LegacySurveyData(opt.input_dir, cache_dir=opt.survey_dir)
outsurvey = LegacySurveyData(opt.output_dir, output_dir=opt.output_dir)
bfn = insurvey.find_file('blobmap', brick=brickname)
print('Found blob map', bfn)
blobs = fitsio.read(bfn)
h, w = blobs.shape
brick = insurvey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brick)
radius = np.sqrt(2.) * 0.25 * 1.01
neighbors = insurvey.get_bricks_near(brick.ra, brick.dec, radius)
print(len(neighbors), 'bricks nearby')
def showbool(X):
d = downsample_max(X, 8)
h, w = X.shape
plt.imshow(d,
interpolation='nearest',
origin='lower',
vmin=0,
vmax=1,
extent=[0, w, 0, h],
cmap='gray')
brightblobs = set()
for nb in neighbors:
if nb.brickname == brickname:
# ignore myself!
continue
print('Neighbor:', nb.brickname)
mfn = insurvey.find_file('maskbits', brick=nb.brickname)
if not os.path.exists(mfn):
print('No maskbits file:', mfn)
continue
maskbits = fitsio.read(mfn)
bright = ((maskbits & MASKBITS['BRIGHT']) > 0)
print(np.sum(bright > 0), 'BRIGHT pixels set')
primary = (maskbits & MASKBITS['NPRIMARY'] == 0)
print(np.sum(primary), 'PRIMARY pixels set')
edge = binary_dilation(primary, structure=np.ones((3, 3), bool))
edge = edge * np.logical_not(primary)
brightedge = edge & bright
if plots:
plt.clf()
showbool(bright)
plt.title('bright: brick %s' % nb.brickname)
ps.savefig()
# plt.clf()
# showbool(primary)
# plt.title('PRIMARY, brick %s' % nb.brickname)
# ps.savefig()
#
# plt.clf()
# showbool(edge)
# plt.title('boundary, brick %s' % nb.brickname)
# ps.savefig()
plt.clf()
showbool(brightedge)
plt.title('bright at edge, brick %s' % nb.brickname)
ps.savefig()
nwcs = wcs_for_brick(nb)
yy, xx = np.nonzero(brightedge)
print(len(yy), 'bright edge pixels')
if len(yy) == 0:
continue
rr, dd = nwcs.pixelxy2radec(xx + 1, yy + 1)
print('RA range', rr.min(), rr.max(), 'vs brick', brick.ra1, brick.ra2)
print('Dec range', dd.min(), dd.max(), 'vs brick', brick.dec1,
brick.dec2)
# Find pixels that are within this brick's unique area
I, = np.nonzero((rr >= brick.ra1) * (rr <= brick.ra2) *
(dd >= brick.dec1) * (dd <= brick.dec2))
if plots:
plt.clf()
plt.plot(
[brick.ra1, brick.ra1, brick.ra2, brick.ra2, brick.ra1],
[brick.dec1, brick.dec2, brick.dec2, brick.dec1, brick.dec1],
'b-')
plt.plot(rr, dd, 'k.')
plt.plot(rr[I], dd[I], 'r.')
plt.title('Bright pixels from %s' % nb.brickname)
ps.savefig()
if len(I) == 0:
print('No edge pixels touch')
#plt.plot(br,bd, 'b-')
continue
#print('Edge pixels touch!')
#plt.plot(br,bd, 'r-', zorder=20)
ok, x, y = brickwcs.radec2pixelxy(rr[I], dd[I])
x = np.round(x).astype(int) - 1
y = np.round(y).astype(int) - 1
print('Pixel ranges X', x.min(), x.max(), 'Y', y.min(), y.max())
assert (np.all((x >= 0) * (x < w) * (y >= 0) * (y < h)))
print('Adding blobs:', np.unique(blobs[y, x]))
brightblobs.update(blobs[y, x])
print('Blobs touching bright pixels:', brightblobs)
print()
brightblobs.discard(-1)
if len(brightblobs) == 0:
print('No neighboring bright blobs to update!')
return
print('Updating', len(brightblobs), 'blobs:', brightblobs)
tmap = np.zeros(blobs.max() + 2, bool)
for b in brightblobs:
tmap[b + 1] = True
touching = tmap[blobs + 1]
if plots:
plt.clf()
showbool(touching)
plt.title('Blobs touching bright, brick %s' % brickname)
ps.savefig()
mfn = insurvey.find_file('maskbits', brick=brickname)
maskbits, hdr = fitsio.read(mfn, header=True)
updated = maskbits | (MASKBITS['BRIGHT'] * touching)
if np.all(maskbits == updated):
print('No bits updated! (Bright stars were already masked)')
return
maskbits = updated
if plots:
plt.clf()
showbool((maskbits & MASKBITS['BRIGHT']) > 0)
plt.title('New maskbits map for BRIGHT, brick %s' % brickname)
ps.savefig()
with outsurvey.write_output('maskbits', brick=brickname) as out:
out.fits.write(maskbits, hdr=hdr)
tfn = insurvey.find_file('tractor', brick=brickname)
phdr = fitsio.read_header(tfn, ext=0)
hdr = fitsio.read_header(tfn, ext=1)
T = fits_table(tfn)
print('Read', len(T), 'sources')
print('Bright:', Counter(T.brightstarinblob))
iby = np.clip(np.round(T.by), 0, h - 1).astype(int)
ibx = np.clip(np.round(T.bx), 0, w - 1).astype(int)
if plots:
before = np.flatnonzero(T.brightstarinblob)
T.brightstarinblob |= touching[iby, ibx]
print('Bright:', Counter(T.brightstarinblob))
# yuck -- copy the TUNIT headers from input to output.
units = [
hdr.get('TUNIT%i' % (i + 1), '') for i in range(len(T.get_columns()))
]
if plots:
plt.clf()
showbool((maskbits & MASKBITS['BRIGHT']) > 0)
ax = plt.axis()
after = np.flatnonzero(T.brightstarinblob)
plt.plot(T.bx[before], T.by[before], 'gx')
plt.plot(T.bx[after], T.by[after], 'r.')
plt.axis(ax)
plt.title('sources with brightstarinblob, brick %s' % brickname)
ps.savefig()
with outsurvey.write_output('tractor', brick=brickname) as out:
T.writeto(None,
fits_object=out.fits,
primheader=phdr,
header=hdr,
units=units)