def __init__(
self,
expnum=None,
ccdname=None,
ccdwcs=None,
pattern='/project/projectdirs/cosmo/work/ps1/cats/chunks-qz-star-v3/ps1-%(hp)05d.fits'
):
"""Read PS1 or gaia sources for an exposure number + CCD name or CCD WCS
Args:
expnum, ccdname: select catalogue with these
ccdwcs: or select catalogue with this
pattern: absolute path and wildcard for PS1 or Gaia catalogues
dr: /project/projectdirs/cosmo/work/
PS1: ${dr}/ps1/cats/chunks-qz-star-v3/ps1-%(hp)05d.fits
PS1-Gaia: ${dr}/gaia/chunks-ps1-gaia/chunk-%(hp)05d.fits
"""
assert ('ps1' in pattern or 'gaia' in pattern)
#assert(ps1_or_gaia in ['ps1','ps1_gaia'])
#if ps1_or_gaia == 'ps1':
# # PS1 "qz" directory
# # e.g. /project/projectdirs/cosmo/work/ps1/cats/chunks-qz-star-v2
# self.catdir= os.getenv('PS1CAT_DIR')
#elif ps1_or_gaia == 'ps1_gaia':
# # PS1-Gaia "qz" matches-only directory
# # e.g. /project/projectdirs/cosmo/work/gaia/chunks-ps1-gaia
# self.catdir= os.getenv('PS1_GAIA_MATCHES')
#fnpattern = os.path.join(self.catdir, prefix + '-%(hp)05d.fits')
super(ps1cat, self).__init__(pattern)
if ccdwcs is None:
from legacypipe.survey import LegacySurveyData
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
self.ccdwcs = im.get_wcs()
else:
self.ccdwcs = ccdwcs
def __init__(self, expnum=None, ccdname=None, ccdwcs=None):
"""Read PS1 or gaia sources for an exposure number + CCD name or CCD WCS
Args:
expnum, ccdname: select catalogue with these
ccdwcs: or select catalogue with this
"""
self.ps1catdir = os.getenv('PS1CAT_DIR')
if self.ps1catdir is None:
raise ValueError(
'You must have the PS1CAT_DIR environment variable set to point to healpixed PS1 catalogs'
)
fnpattern = os.path.join(self.ps1catdir, 'ps1-%(hp)05d.fits')
super(ps1cat, self).__init__(fnpattern)
if ccdwcs is None:
from legacypipe.survey import LegacySurveyData
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
self.ccdwcs = im.get_wcs()
else:
self.ccdwcs = ccdwcs
def __init__(self, expnum=None, ccdname=None, ccdwcs=None):
"""Initialize the class with either the exposure number *and* CCD name, or
directly with the WCS of the CCD of interest.
"""
# GAIA and PS1 info, gaia for astrometry, ps1 for photometry
self.gaiadir = os.getenv('GAIACAT_DIR')
# PS1 only
self.ps1dir = os.getenv('PS1CAT_DIR') # PS1 only
if self.ps1dir is None:
raise ValueError('Need PS1CAT_DIR environment variable to be set.')
if self.gaiadir is None:
print(
'WARNING: GAIACAT_DIR environment variable not set: using Pan-STARRS1 for astrometry'
)
self.nside = 32
if ccdwcs is None:
from legacypipe.survey import LegacySurveyData
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
self.ccdwcs = im.get_wcs()
else:
self.ccdwcs = ccdwcs
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'--expnum',
type=str,
help='Run specified exposure numbers (can be comma-separated list')
parser.add_argument(
'--all-found',
action='store_true',
default=False,
help='Only write output if all required input files are found')
parser.add_argument('--ccds',
help='Set ccds.fits file to load, default is all')
parser.add_argument('--continue',
dest='con',
help='Continue even if one exposure is bad',
action='store_true',
default=False)
parser.add_argument('--outdir',
help='Output directory, default %(default)s',
default='calib')
opt = parser.parse_args()
survey = LegacySurveyData()
if opt.ccds:
ccds = fits_table(opt.ccds)
ccds = survey.cleanup_ccds_table(ccds)
survey.ccds = ccds
if opt.expnum is not None:
expnums = [(None, int(x, 10)) for x in opt.expnum.split(',')]
else:
ccds = survey.get_ccds()
expnums = set(zip(ccds.camera, ccds.expnum))
print(len(expnums), 'unique camera+expnums')
for i, (camera, expnum) in enumerate(expnums):
print()
print('Exposure', i + 1, 'of', len(expnums), ':', camera, 'expnum',
expnum)
if camera is None:
C = survey.find_ccds(expnum=expnum)
print(len(C), 'CCDs with expnum', expnum)
camera = C.camera[0]
print('Set camera to', camera)
C = survey.find_ccds(expnum=expnum, camera=camera)
print(len(C), 'CCDs with expnum', expnum, 'and camera', camera)
im0 = survey.get_image_object(C[0])
skyoutfn = im0.merged_skyfn
psfoutfn = im0.merged_psffn
print('Checking for', skyoutfn)
print('Checking for', psfoutfn)
if os.path.exists(skyoutfn) and os.path.exists(psfoutfn):
print('Exposure', expnum, 'is done already')
continue
if not os.path.exists(skyoutfn):
try:
merge_splinesky(survey, expnum, C, skyoutfn, opt)
except:
if not opt.con:
raise
import traceback
traceback.print_exc()
print('Exposure failed:', expnum, '. Continuing...')
if not os.path.exists(psfoutfn):
try:
merge_psfex(survey, expnum, C, psfoutfn, opt)
except:
if not opt.con:
raise
import traceback
traceback.print_exc()
print('Exposure failed:', expnum, '. Continuing...')
def main(survey=None, opt=None):
print(' '.join(sys.argv))
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = tlast = Time()
if opt.skip and os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing first arg as exposure number (otherwise, it's a filename)
try:
expnum = int(opt.expnum)
filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
filename = opt.expnum
# Try parsing HDU: "all" or HDU name or HDU number.
all_hdus = (opt.ccdname == 'all')
hdu = -1
ccdname = None
if not all_hdus:
try:
hdu = int(opt.ccdname)
except:
ccdname = opt.ccdname
if survey is None:
survey = LegacySurveyData(survey_dir=opt.survey_dir)
catsurvey_north = survey
catsurvey_south = None
if opt.catalog_dir_north is not None:
assert (opt.catalog_dir_south is not None)
assert (opt.catalog_resolve_dec_ngc is not None)
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
if opt.catalog_dir is not None:
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)
if filename is not None and hdu >= 0:
# FIXME -- try looking up in CCDs file?
# Read metadata from file
print('Warning: faking metadata from file contents')
T = exposure_metadata([filename], hdus=[hdu])
print('Metadata:')
T.about()
if not 'ccdzpt' in T.columns():
phdr = fitsio.read_header(filename)
T.ccdzpt = np.array([phdr['MAGZERO']])
print('WARNING: using header MAGZERO')
T.ccdraoff = np.array([0.])
T.ccddecoff = np.array([0.])
print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and ccdname', ccdname)
if hdu >= 0:
T.cut(T.image_hdu == hdu)
print(len(T), 'with HDU', hdu)
if filename is not None:
T.cut(np.array([f.strip() == filename for f in T.image_filename]))
print(len(T), 'with filename', filename)
if opt.camera is not None:
T.cut(T.camera == opt.camera)
print(len(T), 'with camera', opt.camera)
if not all_hdus:
assert (len(T) == 1)
args = []
for ccd in T:
args.append((survey, catsurvey_north, catsurvey_south,
opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))
if opt.threads:
from astrometry.util.multiproc import multiproc
from astrometry.util.timingpool import TimingPool, TimingPoolMeas
pool = TimingPool(opt.threads)
poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(poolmeas)
mp = multiproc(None, pool=pool)
tm = Time()
FF = mp.map(bounce_one_ccd, args)
print('Multi-processing forced-phot:', Time() - tm)
else:
FF = map(bounce_one_ccd, args)
FF = [F for F in FF if F is not None]
if len(FF) == 0:
print('No photometry results to write.')
return 0
# Keep only the first header
_, version_hdr = FF[0]
FF = [F for F, hdr in FF]
F = merge_tables(FF)
if all_hdus:
version_hdr.delete('CPHDU')
version_hdr.delete('CCDNAME')
units = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2',
'apflux': 'nanomaggy',
'apflux_ivar': '1/nanomaggy^2',
'psfdepth': '1/nanomaggy^2',
'galdepth': '1/nanomaggy^2',
'sky': 'nanomaggy/arcsec^2',
'psfsize': 'arcsec'
}
if opt.derivs:
units.update({
'dra': 'arcsec',
'ddec': 'arcsec',
'dra_ivar': '1/arcsec^2',
'ddec_ivar': '1/arcsec^2'
})
columns = F.get_columns()
order = [
'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'ccd_cuts',
'airmass', 'sky', 'psfdepth', 'galdepth', 'ra', 'dec', 'flux',
'flux_ivar', 'fracflux', 'rchisq', 'fracmasked', 'apflux',
'apflux_ivar', 'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar',
'ddec_ivar'
]
columns = [c for c in order if c in columns]
# Set units headers (must happen after column ordering is set!)
hdr = fitsio.FITSHDR()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.outfn))
fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
F.writeto(tmpfn, header=hdr, append=True, columns=columns)
os.rename(tmpfn, opt.outfn)
print('Wrote', opt.outfn)
tnow = Time()
print('Total:', tnow - t0)
return 0
def main(survey=None, opt=None, args=None):
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if args is None:
args = sys.argv[1:]
print('forced_photom.py', ' '.join(args))
if opt is None:
parser = get_parser()
opt = parser.parse_args(args)
import logging
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
t0 = Time()
if survey is None:
survey = LegacySurveyData(survey_dir=opt.survey_dir,
cache_dir=opt.cache_dir,
output_dir=opt.out_dir)
if opt.skip:
if opt.out is not None:
outfn = opt.out
else:
outfn = survey.find_file('forced',
output=True,
camera=opt.camera,
expnum=opt.expnum)
if os.path.exists(outfn):
print('Ouput file exists:', outfn)
return 0
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
return -1
if opt.out is None and opt.out_dir is None:
print('Must supply either --out or --out-dir')
return -1
if opt.expnum is None and opt.out is None:
print('If no --expnum is given, must supply --out filename')
return -1
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Cache CCDs files before the find_ccds call...
# Copy required files into the cache?
if opt.pre_cache:
def copy_files_to_cache(fns):
for fn in fns:
cachefn = fn.replace(survey.survey_dir, survey.cache_dir)
if not cachefn.startswith(survey.cache_dir):
print('Skipping', fn)
continue
outdir = os.path.dirname(cachefn)
trymakedirs(outdir)
print('Copy', fn)
print(' to', cachefn)
shutil.copyfile(fn, cachefn)
assert (survey.cache_dir is not None)
fnset = set()
fn = survey.find_file('bricks')
fnset.add(fn)
fns = survey.find_file('ccd-kds')
fnset.update(fns)
copy_files_to_cache(fnset)
# Read metadata from survey-ccds.fits table
ccds = survey.find_ccds(camera=opt.camera,
expnum=opt.expnum,
ccdname=opt.ccdname)
print(len(ccds), 'with camera', opt.camera, 'and expnum', opt.expnum,
'and ccdname', opt.ccdname)
# sort CCDs
ccds.cut(np.lexsort((ccds.ccdname, ccds.expnum, ccds.camera)))
# If there is only one catalog survey_dir, we pass it to get_catalog_in_wcs
# as the northern survey.
catsurvey_north = survey
catsurvey_south = None
if opt.catalog_dir_north is not None:
assert (opt.catalog_dir_south is not None)
assert (opt.catalog_resolve_dec_ngc is not None)
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
elif opt.catalog_dir is not None:
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)
# Copy required CCD & calib files into the cache?
if opt.pre_cache:
assert (survey.cache_dir is not None)
fnset = set()
for ccd in ccds:
im = survey.get_image_object(ccd)
for key in im.get_cacheable_filename_variables():
fn = getattr(im, key)
if fn is None or not (os.path.exists(fn)):
continue
fnset.add(fn)
copy_files_to_cache(fnset)
args = []
for ccd in ccds:
args.append((survey, catsurvey_north, catsurvey_south,
opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))
if opt.threads:
from astrometry.util.multiproc import multiproc
from astrometry.util.timingpool import TimingPool, TimingPoolMeas
pool = TimingPool(opt.threads)
poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(poolmeas)
mp = multiproc(None, pool=pool)
tm = Time()
FF = mp.map(bounce_one_ccd, args)
print('Multi-processing forced-phot:', Time() - tm)
del mp
Time.measurements.remove(poolmeas)
del poolmeas
pool.close()
pool.join()
del pool
else:
FF = map(bounce_one_ccd, args)
FF = [F for F in FF if F is not None]
if len(FF) == 0:
print('No photometry results to write.')
return 0
# Keep only the first header
_, version_hdr, _, _ = FF[0]
# unpack results
outlier_masks = [m for _, _, m, _ in FF]
outlier_hdrs = [h for _, _, _, h in FF]
FF = [F for F, _, _, _ in FF]
F = merge_tables(FF)
if len(ccds):
version_hdr.delete('CPHDU')
version_hdr.delete('CCDNAME')
from legacypipe.utils import add_bits
from legacypipe.bits import DQ_BITS
add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
for i, ap in enumerate(apertures_arcsec):
version_hdr.add_record(
dict(name='APRAD%i' % i,
value=ap,
comment='(optical) Aperture radius, in arcsec'))
unitmap = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2',
'apflux': 'nanomaggy',
'apflux_ivar': '1/nanomaggy^2',
'psfdepth': '1/nanomaggy^2',
'galdepth': '1/nanomaggy^2',
'sky': 'nanomaggy/arcsec^2',
'psfsize': 'arcsec',
'fwhm': 'pixels',
'ccdrarms': 'arcsec',
'ccddecrms': 'arcsec',
'ra': 'deg',
'dec': 'deg',
'skyrms': 'counts/sec',
'dra': 'arcsec',
'ddec': 'arcsec',
'dra_ivar': '1/arcsec^2',
'ddec_ivar': '1/arcsec^2'
}
columns = F.get_columns()
order = [
'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'fwhm', 'ccd_cuts',
'airmass', 'sky', 'skyrms', 'psfdepth', 'galdepth', 'ccdzpt',
'ccdrarms', 'ccddecrms', 'ccdphrms', 'ra', 'dec', 'flux', 'flux_ivar',
'fracflux', 'rchisq', 'fracmasked', 'fracin', 'apflux', 'apflux_ivar',
'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar'
]
columns = [c for c in order if c in columns]
units = [unitmap.get(c, '') for c in columns]
if opt.out is not None:
outdir = os.path.dirname(opt.out)
if len(outdir):
trymakedirs(outdir)
tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.out))
fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
F.writeto(tmpfn, units=units, append=True, columns=columns)
os.rename(tmpfn, opt.out)
print('Wrote', opt.out)
else:
with survey.write_output('forced',
camera=opt.camera,
expnum=opt.expnum) as out:
F.writeto(None,
fits_object=out.fits,
primheader=version_hdr,
units=units,
columns=columns)
print('Wrote', out.real_fn)
if opt.outlier_mask is not None:
# Add outlier bit meanings to the primary header
version_hdr.add_record(
dict(name='COMMENT', value='Outlier mask bit meanings'))
version_hdr.add_record(
dict(name='OUTL_POS',
value=1,
comment='Outlier mask bit for Positive outlier'))
version_hdr.add_record(
dict(name='OUTL_NEG',
value=2,
comment='Outlier mask bit for Negative outlier'))
if opt.outlier_mask == 'default':
outdir = os.path.join(opt.out_dir, 'outlier-masks')
camexp = set(zip(ccds.camera, ccds.expnum))
for c, e in camexp:
I = np.flatnonzero((ccds.camera == c) * (ccds.expnum == e))
ccd = ccds[I[0]]
imfn = ccd.image_filename.strip()
outfn = os.path.join(outdir, imfn.replace('.fits',
'-outlier.fits'))
trymakedirs(outfn, dir=True)
tempfn = outfn.replace('.fits', '-tmp.fits')
with fitsio.FITS(tempfn, 'rw', clobber=True) as fits:
fits.write(None, header=version_hdr)
for i in I:
mask = outlier_masks[i]
_, _, _, meth, tile = survey.get_compression_args(
'outliers_mask', shape=mask.shape)
fits.write(mask,
header=outlier_hdrs[i],
extname=ccds.ccdname[i],
compress=meth,
tile_dims=tile)
os.rename(tempfn, outfn)
print('Wrote', outfn)
elif opt.outlier_mask is not None:
with fitsio.FITS(opt.outlier_mask, 'rw', clobber=True) as F:
F.write(None, header=version_hdr)
for i, (hdr, mask) in enumerate(zip(outlier_hdrs, outlier_masks)):
_, _, _, meth, tile = survey.get_compression_args(
'outliers_mask', shape=mask.shape)
F.write(mask,
header=hdr,
extname=ccds.ccdname[i],
compress=meth,
tile_dims=tile)
print('Wrote', opt.outlier_mask)
tnow = Time()
print('Total:', tnow - t0)
return 0
def psf_residuals(expnum,
ccdname,
stampsize=35,
nstar=30,
magrange=(13, 17),
verbose=0,
splinesky=False):
# Set the debugging level.
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
pngprefix = 'qapsf-{}-{}'.format(expnum, ccdname)
# Gather all the info we need about this CCD.
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
band = ccd.filter
ps1band = dict(g=0, r=1, i=2, z=3, Y=4)
print('Band {}'.format(band))
#scales = dict(g=0.0066, r=0.01, z=0.025)
#vmin, vmax = np.arcsinh(-1), np.arcsinh(100)
#print(scales[band])
im = survey.get_image_object(ccd)
iminfo = im.get_image_info()
H, W = iminfo['dims']
wcs = im.get_wcs()
# Choose a uniformly selected subset of PS1 stars on this CCD.
ps1 = ps1cat(ccdwcs=wcs)
cat = ps1.get_stars(band=band, magrange=magrange)
rand = np.random.RandomState(seed=expnum * ccd.ccdnum)
these = rand.choice(len(cat) - 1, nstar, replace=False)
#these = rand.random_integers(0,len(cat)-1,nstar)
cat = cat[these]
cat = cat[np.argsort(cat.median[:, ps1band[band]])] # sort by magnitude
#print(cat.nmag_ok)
get_tim_kwargs = dict(pixPsf=True, splinesky=splinesky)
# Make a QAplot of the positions of all the stars.
tim = im.get_tractor_image(**get_tim_kwargs)
img = tim.getImage()
#img = tim.getImage()/scales[band]
fig = plt.figure(figsize=(5, 10))
ax = fig.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
#ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest',
# origin='lower',vmin=vmax,vmax=vmax)
ax.imshow(img, **tim.ima)
ax.axis('off')
ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.format(
band, expnum, ccdname, ccd.airmass, ccd.seeing))
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ax.text(xpos,
ypos,
'{:2d}'.format(istar + 1),
color='red',
horizontalalignment='left')
circ = plt.Circle((xpos, ypos), radius=30, color='g', fill=False, lw=1)
ax.add_patch(circ)
#radec = wcs.radec_bounds()
#ax.scatter(cat.ra,cat.dec)
#ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998])
#ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015])
#ax.set_xlabel('$RA\ (deg)$',fontsize=18)
#ax.set_ylabel('$Dec\ (deg)$',fontsize=18)
fig.savefig(pngprefix + '-ccd.png', bbox_inches='tight')
# Initialize the many-stamp QAplot
ncols = 3
nrows = np.ceil(nstar / ncols).astype('int')
inchperstamp = 2.0
fig = plt.figure(figsize=(inchperstamp * 3 * ncols, inchperstamp * nrows))
irow = 0
icol = 0
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
mag = ps1star.median[ps1band[band]] # r-band
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ix, iy = int(xpos), int(ypos)
# create a little tractor Image object around the star
slc = (slice(max(iy - stampsize, 0), min(iy + stampsize + 1, H)),
slice(max(ix - stampsize, 0), min(ix + stampsize + 1, W)))
# The PSF model 'const2Psf' is the one used in DR1: a 2-component
# Gaussian fit to PsfEx instantiated in the image center.
tim = im.get_tractor_image(slc=slc, **get_tim_kwargs)
stamp = tim.getImage()
ivarstamp = tim.getInvvar()
# Initialize a tractor PointSource from PS1 measurements
flux = NanoMaggies.magToNanomaggies(mag)
star = PointSource(RaDecPos(ra, dec), NanoMaggies(**{band: flux}))
# Fit just the source RA,Dec,flux.
tractor = Tractor([tim], [star])
tractor.freezeParam('images')
print('2-component MOG:', tim.psf)
tractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_mog = tractor.getModelImage(0)
chi2_mog = -2.0 * tractor.getLogLikelihood()
mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
# Now change the PSF model to a pixelized PSF model from PsfEx instantiated
# at this place in the image.
psf = PixelizedPsfEx(im.psffn)
tim.psf = psf.constantPsfAt(xpos, ypos)
#print('PSF model:', tim.psf)
#tractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_psfex = tractor.getModelImage(0)
chi2_psfex = -2.0 * tractor.getLogLikelihood()
mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
#mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95])
sig = np.std((stamp - model_psfex)[ivarstamp > 0])
mn, mx = [-2.0 * sig, 5 * sig]
# Generate a QAplot.
if (istar > 0) and (istar % (ncols) == 0):
irow = irow + 1
icol = 3 * istar - 3 * ncols * irow
#print(istar, irow, icol, icol+1, icol+2)
ax1 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol),
aspect='equal')
ax1.axis('off')
#ax1.imshow(stamp, **tim.ima)
ax1.imshow(stamp,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax1.text(0.1,
0.9,
'{:2d}'.format(istar + 1),
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax1.transAxes)
ax2 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 1),
aspect='equal')
ax2.axis('off')
#ax2.imshow(stamp-model_mog, **tim.ima)
ax2.imshow(stamp - model_mog,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax2.text(0.1,
0.9,
'MoG',
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax2.transAxes)
ax2.text(0.08,
0.08,
'{:.3f}'.format(mag_mog),
color='white',
horizontalalignment='left',
verticalalignment='bottom',
transform=ax2.transAxes)
#ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14)
#ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex))
ax3 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 2),
aspect='equal')
ax3.axis('off')
#ax3.imshow(stamp-model_psfex, **tim.ima)
ax3.imshow(stamp - model_psfex,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax3.text(0.1,
0.9,
'PSFEx',
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax3.transAxes)
ax3.text(0.08,
0.08,
'{:.3f}'.format(mag_psfex),
color='white',
horizontalalignment='left',
verticalalignment='bottom',
transform=ax3.transAxes)
if istar == (nstar - 1):
break
fig.savefig(pngprefix + '-stargrid.png', bbox_inches='tight')
from astrometry.util.plotutils import *
from legacyanalysis.ps1cat import ps1cat
from legacypipe.survey import LegacySurveyData
from tractor import Image, PointSource, PixPos, NanoMaggies, Tractor
ps = PlotSequence('rewcs')
expnum, ccdname = 431109, 'N14'
cat = ps1cat(expnum=expnum, ccdname=ccdname)
stars = cat.get_stars()
print len(stars), 'stars'
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum,ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
wcs = im.get_wcs()
tim = im.get_tractor_image(pixPsf=True, splinesky=True)
margin = 15
ok,stars.xx,stars.yy = wcs.radec2pixelxy(stars.ra, stars.dec)
stars.xx -= 1.
stars.yy -= 1.
W,H = wcs.get_width(), wcs.get_height()
stars.ix = np.round(stars.xx).astype(int)
stars.iy = np.round(stars.yy).astype(int)
stars.cut((stars.ix >= margin) * (stars.ix < (W-margin)) *
(stars.iy >= margin) * (stars.iy < (H-margin)))
plt.clf()
def psf_residuals(expnum,ccdname,stampsize=35,nstar=30,
magrange=(13,17),verbose=0, splinesky=False):
# Set the debugging level.
if verbose==0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)
pngprefix = 'qapsf-{}-{}'.format(expnum,ccdname)
# Gather all the info we need about this CCD.
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum,ccdname=ccdname)[0]
band = ccd.filter
ps1band = dict(g=0,r=1,i=2,z=3,Y=4)
print('Band {}'.format(band))
#scales = dict(g=0.0066, r=0.01, z=0.025)
#vmin, vmax = np.arcsinh(-1), np.arcsinh(100)
#print(scales[band])
im = survey.get_image_object(ccd)
iminfo = im.get_image_info()
H,W = iminfo['dims']
wcs = im.get_wcs()
# Choose a uniformly selected subset of PS1 stars on this CCD.
ps1 = ps1cat(ccdwcs=wcs)
cat = ps1.get_stars(band=band,magrange=magrange)
rand = np.random.RandomState(seed=expnum*ccd.ccdnum)
these = rand.choice(len(cat)-1,nstar,replace=False)
#these = rand.random_integers(0,len(cat)-1,nstar)
cat = cat[these]
cat = cat[np.argsort(cat.median[:,ps1band[band]])] # sort by magnitude
#print(cat.nmag_ok)
get_tim_kwargs = dict(pixPsf=True, splinesky=splinesky)
# Make a QAplot of the positions of all the stars.
tim = im.get_tractor_image(**get_tim_kwargs)
img = tim.getImage()
#img = tim.getImage()/scales[band]
fig = plt.figure(figsize=(5,10))
ax = fig.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
#ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest',
# origin='lower',vmin=vmax,vmax=vmax)
ax.imshow(img, **tim.ima)
ax.axis('off')
ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.
format(band,expnum,ccdname,ccd.airmass,ccd.seeing))
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ax.text(xpos,ypos,'{:2d}'.format(istar+1),color='red',
horizontalalignment='left')
circ = plt.Circle((xpos,ypos),radius=30,color='g',fill=False,lw=1)
ax.add_patch(circ)
#radec = wcs.radec_bounds()
#ax.scatter(cat.ra,cat.dec)
#ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998])
#ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015])
#ax.set_xlabel('$RA\ (deg)$',fontsize=18)
#ax.set_ylabel('$Dec\ (deg)$',fontsize=18)
fig.savefig(pngprefix+'-ccd.png',bbox_inches='tight')
# Initialize the many-stamp QAplot
ncols = 3
nrows = np.ceil(nstar/ncols).astype('int')
inchperstamp = 2.0
fig = plt.figure(figsize=(inchperstamp*3*ncols,inchperstamp*nrows))
irow = 0
icol = 0
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
mag = ps1star.median[ps1band[band]] # r-band
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ix,iy = int(xpos), int(ypos)
# create a little tractor Image object around the star
slc = (slice(max(iy-stampsize, 0), min(iy+stampsize+1, H)),
slice(max(ix-stampsize, 0), min(ix+stampsize+1, W)))
# The PSF model 'const2Psf' is the one used in DR1: a 2-component
# Gaussian fit to PsfEx instantiated in the image center.
tim = im.get_tractor_image(slc=slc, **get_tim_kwargs)
stamp = tim.getImage()
ivarstamp = tim.getInvvar()
# Initialize a tractor PointSource from PS1 measurements
flux = NanoMaggies.magToNanomaggies(mag)
star = PointSource(RaDecPos(ra,dec), NanoMaggies(**{band: flux}))
# Fit just the source RA,Dec,flux.
tractor = Tractor([tim], [star])
tractor.freezeParam('images')
print('2-component MOG:', tim.psf)
tractor.printThawedParams()
for step in range(50):
dlnp,X,alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_mog = tractor.getModelImage(0)
chi2_mog = -2.0*tractor.getLogLikelihood()
mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
# Now change the PSF model to a pixelized PSF model from PsfEx instantiated
# at this place in the image.
psf = PixelizedPsfEx(im.psffn)
tim.psf = psf.constantPsfAt(xpos, ypos)
#print('PSF model:', tim.psf)
#tractor.printThawedParams()
for step in range(50):
dlnp,X,alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_psfex = tractor.getModelImage(0)
chi2_psfex = -2.0*tractor.getLogLikelihood()
mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
#mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95])
sig = np.std((stamp-model_psfex)[ivarstamp>0])
mn, mx = [-2.0*sig,5*sig]
# Generate a QAplot.
if (istar>0) and (istar%(ncols)==0):
irow = irow+1
icol = 3*istar - 3*ncols*irow
#print(istar, irow, icol, icol+1, icol+2)
ax1 = plt.subplot2grid((nrows,3*ncols), (irow,icol), aspect='equal')
ax1.axis('off')
#ax1.imshow(stamp, **tim.ima)
ax1.imshow(stamp,cmap='gray',interpolation='nearest',
origin='lower',vmin=mn,vmax=mx)
ax1.text(0.1,0.9,'{:2d}'.format(istar+1),color='white',
horizontalalignment='left',verticalalignment='top',
transform=ax1.transAxes)
ax2 = plt.subplot2grid((nrows,3*ncols), (irow,icol+1), aspect='equal')
ax2.axis('off')
#ax2.imshow(stamp-model_mog, **tim.ima)
ax2.imshow(stamp-model_mog,cmap='gray',interpolation='nearest',
origin='lower',vmin=mn,vmax=mx)
ax2.text(0.1,0.9,'MoG',color='white',
horizontalalignment='left',verticalalignment='top',
transform=ax2.transAxes)
ax2.text(0.08,0.08,'{:.3f}'.format(mag_mog),color='white',
horizontalalignment='left',verticalalignment='bottom',
transform=ax2.transAxes)
#ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14)
#ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex))
ax3 = plt.subplot2grid((nrows,3*ncols), (irow,icol+2), aspect='equal')
ax3.axis('off')
#ax3.imshow(stamp-model_psfex, **tim.ima)
ax3.imshow(stamp-model_psfex,cmap='gray',interpolation='nearest',
origin='lower',vmin=mn,vmax=mx)
ax3.text(0.1,0.9,'PSFEx',color='white',
horizontalalignment='left',verticalalignment='top',
transform=ax3.transAxes)
ax3.text(0.08,0.08,'{:.3f}'.format(mag_psfex),color='white',
horizontalalignment='left',verticalalignment='bottom',
transform=ax3.transAxes)
if istar==(nstar-1):
break
fig.savefig(pngprefix+'-stargrid.png',bbox_inches='tight')
def main():
fn = '/global/cscratch1/sd/dstn/c4d_190730_024955_ori/c4d_190730_024955_ori.52.fits'
survey_dir = '/global/cscratch1/sd/dstn/subtractor-survey-dir'
imagedir = os.path.join(survey_dir, 'images')
trymakedirs(imagedir)
calibdir = os.path.join(survey_dir, 'calib')
psfexdir = os.path.join(calibdir, 'decam', 'psfex-merged')
trymakedirs(psfexdir)
skydir = os.path.join(calibdir, 'decam', 'splinesky-merged')
trymakedirs(skydir)
basename = os.path.basename(fn)
basename = basename.replace('.fits', '')
# Output filenames for legacyzpts calibration/zeropoint files
f, photfn = tempfile.mkstemp()
os.close(f)
surveyfn = os.path.join(survey_dir, 'survey-ccds-%s.fits.gz' % basename)
annfn = os.path.join(survey_dir, 'annotated-%s.fits' % basename)
mp = multiproc()
survey = LegacySurveyData(survey_dir)
# Use the subclass above to handle DECam images!
survey.image_typemap.update(decam=GoldsteinDecamImage)
# Grab the exposure number and CCD name
hdr = fitsio.read_header(fn)
expnum = hdr['EXPNUM']
ccdname = hdr['EXTNAME'].strip()
print('Exposure', expnum, 'CCD', ccdname)
import logging
lvl = logging.INFO
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
if not os.path.exists(surveyfn):
# Run calibrations and zeropoints
runit(fn,
photfn,
surveyfn,
annfn,
mp,
survey=survey,
camera='decam',
debug=False,
choose_ccd=ccdname,
splinesky=True,
calibdir=calibdir,
measureclass=GoldsteinDecamMeasurer)
# Find catalog sources touching this CCD
ccds = survey.find_ccds(expnum=expnum, ccdname=ccdname)
assert (len(ccds) == 1)
ccd = ccds[0]
print('Got CCD', ccd)
# Create Tractor image
im = survey.get_image_object(ccd)
print('Got image:', im)
# Look at this sub-image, or the whole chip?
#zoomslice=None
zoomslice = (slice(0, 1000), slice(0, 1000))
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
splinesky=True,
hybridPsf=True,
normalizePsf=True,
old_calibs_ok=True)
print('Got tim:', tim)
# Read catalog files touching this CCD
catsurvey = LegacySurveyData(
'/global/project/projectdirs/cosmo/work/legacysurvey/dr8/south')
T = get_catalog_in_wcs(tim.subwcs, catsurvey)
print('Got', len(T), 'DR8 catalog sources within CCD')
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
from legacypipe.survey import radec_at_mjd
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra, dec = radec_at_mjd(T.ra[I], T.dec[I], T.ref_epoch[I].astype(float),
T.pmra[I], T.pmdec[I], T.parallax[I],
tim.time.toMjd())
T.ra[I] = ra
T.dec[I] = dec
# Create Tractor Source objects from the catalog
cat = read_fits_catalog(T, bands=tim.band)
print('Created', len(cat), 'source objects')
# Render model image!
tr = Tractor([tim], cat)
mod = tr.getModelImage(0)
# plots
ima = dict(interpolation='nearest',
origin='lower',
vmin=-2 * tim.sig1,
vmax=10 * tim.sig1,
cmap='gray')
plt.clf()
plt.imshow(tim.getImage(), **ima)
plt.title('Image')
plt.savefig('img.jpg')
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model')
plt.savefig('mod.jpg')
plt.clf()
plt.imshow(tim.getImage() - mod, **ima)
plt.title('Residual')
plt.savefig('res.jpg')
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--force',
action='store_true',
help='Run calib processes even if files already exist?')
parser.add_argument('--survey-dir', help='Override LEGACY_SURVEY_DIR')
parser.add_argument(
'--expnum',
type=str,
help='Cut to a single or set of exposures; comma-separated list')
parser.add_argument('--extname',
'--ccdname',
help='Cut to a single extension/CCD name')
parser.add_argument('--no-psf',
dest='psfex',
action='store_false',
help='Do not compute PsfEx calibs')
parser.add_argument('--no-sky',
dest='sky',
action='store_false',
help='Do not compute sky models')
parser.add_argument('--run-se',
action='store_true',
help='Run SourceExtractor')
parser.add_argument('--no-splinesky',
dest='splinesky',
default=True,
action='store_false',
help='Use constant, not splinesky')
parser.add_argument('--threads',
type=int,
help='Run multi-threaded',
default=None)
parser.add_argument('--continue',
dest='cont',
default=False,
action='store_true',
help='Continue even if one file fails?')
parser.add_argument('--plot-base',
help='Make plots with this base filename')
parser.add_argument(
'--blob-mask-dir',
type=str,
default=None,
help=
'The base directory to search for blob masks during sky model construction'
)
parser.add_argument('-v',
'--verbose',
dest='verbose',
action='count',
default=0,
help='Make more verbose')
parser.add_argument('args', nargs=argparse.REMAINDER)
opt = parser.parse_args()
import logging
if opt.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
survey = LegacySurveyData(survey_dir=opt.survey_dir)
T = None
if len(opt.args) == 0:
if opt.expnum is not None:
expnums = set([int(e) for e in opt.expnum.split(',')])
T = merge_tables([
survey.find_ccds(expnum=e, ccdname=opt.extname)
for e in expnums
])
print('Cut to', len(T), 'with expnum in', expnums, 'and extname',
opt.extname)
opt.args = range(len(T))
else:
parser.print_help()
return 0
ps = None
if opt.plot_base is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plot_base)
survey_blob_mask = None
if opt.blob_mask_dir is not None:
survey_blob_mask = LegacySurveyData(opt.blob_mask_dir)
args = []
for a in opt.args:
# Check for "expnum-ccdname" format.
if '-' in str(a):
words = a.split('-')
assert (len(words) == 2)
expnum = int(words[0])
ccdname = words[1]
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
if len(T) != 1:
print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname',
ccdname, ':', I)
print('WARNING: skipping this expnum,ccdname')
continue
t = T[0]
else:
i = int(a)
print('Index', i)
t = T[i]
im = survey.get_image_object(t)
print('Running', im.name)
kwargs = dict(psfex=opt.psfex,
sky=opt.sky,
ps=ps,
survey=survey,
survey_blob_mask=survey_blob_mask)
if opt.force:
kwargs.update(force=True)
if opt.run_se:
kwargs.update(se=True)
if opt.splinesky:
kwargs.update(splinesky=True)
if opt.cont:
kwargs.update(noraise=True)
if opt.threads:
args.append((im, kwargs))
else:
run_calibs((im, kwargs))
if opt.threads:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
mp.map(time_run_calibs, args)
return 0
from astrometry.util.plotutils import *
from legacyanalysis.ps1cat import ps1cat
from legacypipe.survey import LegacySurveyData
from tractor import Image, PointSource, PixPos, NanoMaggies, Tractor
ps = PlotSequence('rewcs')
expnum, ccdname = 431109, 'N14'
cat = ps1cat(expnum=expnum, ccdname=ccdname)
stars = cat.get_stars()
print len(stars), 'stars'
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
wcs = im.get_wcs()
tim = im.get_tractor_image(pixPsf=True, splinesky=True)
margin = 15
ok, stars.xx, stars.yy = wcs.radec2pixelxy(stars.ra, stars.dec)
stars.xx -= 1.
stars.yy -= 1.
W, H = wcs.get_width(), wcs.get_height()
stars.ix = np.round(stars.xx).astype(int)
stars.iy = np.round(stars.yy).astype(int)
stars.cut((stars.ix >= margin) * (stars.ix < (W - margin)) *
(stars.iy >= margin) * (stars.iy < (H - margin)))
plt.clf()
def main(survey=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if survey is None:
survey = LegacySurveyData()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(np.array([f.strip() == opt.filename for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert(len(T) == 1)
ccd = T[0]
im = survey.get_image_object(ccd)
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True,
constant_invvar=opt.constant_invvar)
print('Got tim:', tim)
print('Read image:', Time()-t0)
if opt.catfn in ['DR1', 'DR2', 'DR3']:
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=survey)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = survey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
if len(T):
TT.append(T)
if len(TT) == 0:
print('No sources to photometer.')
return 0
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
surveydir = survey.get_survey_dir()
del survey
cat = read_fits_catalog(T)
# print('Got cat:', cat)
print('Read catalog:', Time()-t0)
print('Forced photom...')
opti = None
forced_kwargs = {}
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
#forced_kwargs.update(verbose=True)
for src in cat:
# Limit sizes of huge models
from tractor.galaxy import ProfileGalaxy
if isinstance(src, ProfileGalaxy):
px,py = tim.wcs.positionToPixel(src.getPosition())
h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval)
MAXHALF = 128
if h > MAXHALF:
print('halfsize', h,'for',src,'-> setting to',MAXHALF)
src.halfsize = MAXHALF
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
disable_galaxy_cache()
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32)
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
if opt.forced:
if opt.plots is None:
forced_kwargs.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True, fitstats=True,
shared_params=False, priors=False, **forced_kwargs)
if opt.plots:
(data,mod,ie,chi,roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([src.getBrightness().getFlux(tim.band)
for src in cat]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2 .astype(np.float32)
print('Forced photom:', Time()-t0)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap.astype(np.float32)
ap = 1./(np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap.astype(np.float32)
print('Aperture photom:', Time()-t0)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, surveydir)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE',
value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
comment='Name of this image'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'}
columns = F.get_columns()
for i,col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
print('Getting model image...')
mod = tr.getModelImage(tim)
fitsio.write(opt.save_model, mod, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
print('Finished forced phot:', Time()-t0)
return 0
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--force',
action='store_true',
help='Run calib processes even if files already exist?')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument(
'--expnum',
type=str,
help='Cut to a single or set of exposures; comma-separated list')
parser.add_argument('--extname',
'--ccdname',
help='Cut to a single extension/CCD name')
parser.add_argument('--no-psf',
dest='psfex',
action='store_false',
help='Do not compute PsfEx calibs')
parser.add_argument('--no-sky',
dest='sky',
action='store_false',
help='Do not compute sky models')
parser.add_argument('--run-se',
action='store_true',
help='Run SourceExtractor')
parser.add_argument('--splinesky',
action='store_true',
help='Spline sky, not constant')
parser.add_argument('--threads',
type=int,
help='Run multi-threaded',
default=None)
parser.add_argument('--continue',
dest='cont',
default=False,
action='store_true',
help='Continue even if one file fails?')
parser.add_argument('--plot-base',
help='Make plots with this base filename')
# actually this doesn't work for calibs...
#parser.add_argument('--outdir', dest='output_dir', default=None,
# help='Set output base directory')
parser.add_argument('args', nargs=argparse.REMAINDER)
opt = parser.parse_args()
survey = LegacySurveyData() #output_dir=opt.output_dir)
T = None
if opt.ccds is not None:
T = fits_table(opt.ccds)
T = survey.cleanup_ccds_table(T)
print('Read', len(T), 'from', opt.ccds)
#else:
# T = survey.get_ccds()
# #print len(T), 'CCDs'
if len(opt.args) == 0:
if opt.expnum is not None:
expnums = set([int(e) for e in opt.expnum.split(',')])
#T.cut(np.array([e in expnums for e in T.expnum]))
T = merge_tables([
survey.find_ccds(expnum=e, ccdname=opt.extname)
for e in expnums
])
print('Cut to', len(T), 'with expnum in', expnums, 'and extname',
opt.extname)
#if opt.extname is not None:
# T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname]))
# print('Cut to', len(T), 'with extname =', opt.extname)
opt.args = range(len(T))
ps = None
if opt.plot_base is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plot_base)
args = []
for a in opt.args:
# Check for "expnum-ccdname" format.
if '-' in str(a):
words = a.split('-')
assert (len(words) == 2)
expnum = int(words[0])
ccdname = words[1]
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
if len(T) != 1:
print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname',
ccdname, ':', I)
print('WARNING: skipping this expnum,ccdname')
continue
t = T[0]
else:
i = int(a)
print('Index', i)
t = T[i]
#print('CCDnmatch', t.ccdnmatch)
#if t.ccdnmatch < 20 and not opt.force:
# print('Skipping ccdnmatch = %i' % t.ccdnmatch)
# continue
im = survey.get_image_object(t)
print('Running', im.name)
kwargs = dict(psfex=opt.psfex, sky=opt.sky, ps=ps, survey=survey)
if opt.force:
kwargs.update(force=True)
if opt.run_se:
kwargs.update(se=True)
if opt.splinesky:
kwargs.update(splinesky=True)
if opt.cont:
kwargs.update(noraise=True)
if opt.threads:
args.append((im, kwargs))
else:
run_calibs((im, kwargs))
if opt.threads:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads)
mp.map(time_run_calibs, args)
return 0
def main(survey=None, opt=None):
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
import pylab as plt
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.expnum)
filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
filename = opt.expnum
# Try parsing HDU number
try:
hdu = int(opt.ccdname)
ccdname = None
except:
hdu = -1
ccdname = opt.ccdname
if survey is None:
survey = LegacySurveyData()
catsurvey = survey
if opt.catalog_dir is not None:
catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir)
if filename is not None and hdu >= 0:
# FIXME -- try looking up in CCDs file?
# Read metadata from file
print('Warning: faking metadata from file contents')
T = exposure_metadata([filename], hdus=[hdu])
print('Metadata:')
T.about()
if not 'ccdzpt' in T.columns():
phdr = fitsio.read_header(filename)
T.ccdzpt = np.array([phdr['MAGZERO']])
print('WARNING: using header MAGZERO')
T.ccdraoff = np.array([0.])
T.ccddecoff = np.array([0.])
print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if hdu >= 0:
T.cut(T.image_hdu == hdu)
print(len(T), 'with HDU', hdu)
if filename is not None:
T.cut(np.array([f.strip() == filename for f in T.image_filename]))
print(len(T), 'with filename', filename)
if opt.camera is not None:
T.cut(T.camera == opt.camera)
print(len(T), 'with camera', opt.camera)
assert (len(T) == 1)
ccd = T[0]
im = survey.get_image_object(ccd)
if opt.do_calib:
#from legacypipe.survey import run_calibs
#kwa = dict(splinesky=True)
#run_calibs((im, kwa))
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
splinesky=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf)
print('Got tim:', tim)
print('Read image:', Time() - t0)
if opt.catfn in ['DR1', 'DR2', 'DR3', 'DR5', 'DR']:
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = catsurvey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) *
(yy >= -margin) * (yy <= (H + margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
for col in ['out_of_bounds', 'left_blob']:
if col in T.get_columns():
T.cut(T.get(col) == False)
print('Cut to', len(T), 'on', col)
if len(T):
TT.append(T)
if len(TT) == 0:
print('No sources to photometer.')
return 0
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
print('Total of', len(T), 'catalog sources')
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
surveydir = survey.get_survey_dir()
del survey
kwargs = {}
cols = T.get_columns()
if 'flux_r' in cols and not 'decam_flux_r' in cols:
kwargs.update(fluxPrefix='')
cat = read_fits_catalog(T, **kwargs)
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
print('Read catalog:', Time() - t0)
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
ps=ps)
t0 = Time()
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F)).astype(np.int32)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32)
F.ra = T.ra
F.dec = T.dec
ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
F.mask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1),
np.clip(np.round(F.x).astype(int), 0, w - 1)]
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, surveydir)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='EXPOSURE',
value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
comment='Name of this image'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2'
}
columns = F.get_columns()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
print('Getting model image...')
tr = Tractor([tim], cat)
mod = tr.getModelImage(tim)
fitsio.write(opt.save_model, mod, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
print('Finished forced phot:', Time() - t0)
return 0