def main():
B = fits_table(
'/global/cfs/cdirs/cosmo/data/legacysurvey/dr8/survey-bricks.fits.gz')
B.ll, B.bb = radectolb(B.ra, B.dec)
I = np.flatnonzero((B.dec > -70) * (np.abs(B.bb) > 10))
B[I].writeto('bricks-for-gaia.fits')
BG = B[I]
BG = BG[np.argsort(-BG.dec)]
# healpixes = set()
# nside = 32
# for r,d in zip(BG.ra,BG.dec):
# hpxy = radecdegtohealpix(r, d, nside)
# hpring = healpix_xy_to_ring(hpxy, nside)
# healpixes.add(hpring)
# hr,hd = [],[]
# for hp in healpixes:
# hp = healpix_ring_to_xy(hp, nside)
# r,d = healpix_to_radecdeg(hp, nside, 0.5, 0.5)
# hr.append(r)
# hd.append(d)
# plt.plot(hr, hd, 'b.', alpha=0.1);
survey = LegacySurveyData('/global/cfs/cdirs/cosmo/work/legacysurvey/dr9')
#BG = BG[:100]
# GG = []
# for i,brick in enumerate(BG):
# G = one_brick(brick, survey)
# GG.append(G)
mp = multiproc(32)
GG = []
iset = 0
while len(BG):
N = 10000
outfn = '/global/cscratch1/sd/dstn/gaia-mask-dr9-set%i.fits' % iset
if os.path.exists(outfn):
Gset = fits_table(outfn)
print('Read', outfn)
nb = len(set(Gset.brickname))
if nb != N:
print('Warning: file contains', nb, 'bricks, vs', N)
else:
Gset = mp.map(bounce_one_brick,
[(brick, survey) for brick in BG[:N]])
Gset = [G for G in Gset if G is not None]
Gset = merge_tables(Gset, columns='fillzero')
Gset.writeto(outfn)
GG.append(Gset)
iset += 1
BG = BG[N:]
G = merge_tables(GG, columns='fillzero')
G.writeto('/global/cscratch1/sd/dstn/gaia-mask-dr9.fits')
def main():
survey = LegacySurveyData()
ccds = survey.get_ccds_readonly()
print(len(ccds), 'CCDs')
ccds = ccds[ccds.ccd_cuts == 0]
print(len(ccds), 'good CCDs')
# Find bricks touched by >=1 CCD
bricks = survey.get_bricks_readonly()
bricks = bricks[(bricks.dec > -20) * (bricks.dec < 35.)]
print(len(bricks), 'bricks in Dec range')
I, J, d = match_radec(bricks.ra,
bricks.dec,
ccds.ra,
ccds.dec,
0.5,
nearest=True)
bricks = bricks[I]
print(len(bricks), 'bricks')
bands = ['g', 'r', 'z']
nexps = {}
for b in bands:
ne = np.zeros(len(bricks), np.int16)
nexps[b] = ne
bricks.set('nexp_' + b, ne)
npix = {}
for b in bands:
n = np.zeros(len(bricks), np.int64)
npix[b] = n
bricks.set('npix_' + b, n)
for b in bands:
n = np.zeros(len(bricks), np.float32)
bricks.set('psfdepth_' + b, n)
args = enumerate(bricks)
mp = multiproc(8)
R = mp.map(one_brick, args)
for ibrick, res in enumerate(R):
if res is None:
continue
(npix, nexps, depths) = res
for band in bands:
bricks.get('npix_' + band)[ibrick] = npix[band]
bricks.get('nexp_' + band)[ibrick] = nexps[band]
bricks.get('psfdepth_' + band)[ibrick] = depths[band]
bricks.cut((bricks.nexp_g + bricks.nexp_r + bricks.nexp_z) > 0)
bricks.writeto('/global/cscratch1/sd/dstn/bricks-nexp.fits')
def organize_by_brick(sample_fns,
sbricks,
outdir=None,
seed=None,
prefix=None):
'''
For each sample_fn, split into bricks
get brick sample fn for that brick and sample
write it if does not exist
sample_fn -- sample_seed.fits file assigned to that mpi task
sbricks -- survey bricks table cut to radec region
'''
dr = get_bybrick_dir(outdir=outdir)
for sample_fn in sample_fns:
# Skip if already looped over bricks for this sample
check_done = os.path.join(dr, get_sample_fn(seed=seed, prefix=prefix))
check_done = check_done.replace('.fits', '_done.txt')
if os.path.exists(check_done):
print('check_done exists: %s' % check_done)
continue
#
sample = fits_table(sample_fn)
print('sample min,max ra,dec= %f %f %f %f' % (sample.ra.min(),sample.ra.max(),\
sample.dec.min(),sample.dec.max()))
# Loop over survey bricks
survey = LegacySurveyData()
for sbrick in sbricks:
# Get output fn for this brick and sample
fn = os.path.join(
dr,
get_brick_sample_fn(brickname=sbrick.brickname,
seed=seed,
prefix=prefix))
if os.path.exists(fn):
continue
# Cut sample by brick's bounds
brickinfo = survey.get_brick_by_name(sbrick.brickname)
brickwcs = wcs_for_brick(brickinfo)
ra1, ra2, dec1, dec2 = brickwcs.radec_bounds()
keep= (sample.ra >= ra1)*(sample.ra <= ra2)*\
(sample.dec >= dec1)*(sample.dec <= dec2)
sample2 = sample.copy()
if np.where(keep)[0].size > 0:
sample2.cut(keep)
sample2.writeto(fn)
print('Wrote %s' % fn)
else:
print('WARNING: sample=%s has no ra,dec in brick=%s' %
(sample_fn, sbrick.brickname))
# This sample is done
with open(check_done, 'w') as foo:
foo.write('done')
def get_grid_randoms(truth_fn, bricknames=[], south=True, seed=None):
rng = np.random.RandomState(seed=seed)
randoms = 0
survey = LegacySurveyData(survey_dir='/global/cfs/cdirs/cosmo/work/legacysurvey/dr9')
for iseed,brickname in enumerate(bricknames):
randoms += get_grid_in_brick(survey,brickname,rng=rng)
randoms.photsys = randoms.full('S' if south else 'N')
truth = get_truth(truth_fn,south=south)
randoms.fill(sample_from_truth(randoms,truth,rng=rng),index_self=None,index_other=None)
return randoms
def __init__(self, ls_dir=None, outdir=None, savedir=None, jpeg=False):
"""outdir: required
ls_dir: not needed if env var LEGACY_SURVEY_DIR already set
save_dir: where write hdf5 files, outdir if None
"""
self.outdir = outdir
self.jpeg = jpeg
if ls_dir:
os.environ["LEGACY_SURVEY_DIR"] = ls_dir
self.savedir = savedir
if self.savedir is None:
self.savedir = self.outdir
self.survey = LegacySurveyData()
def main(outfn='ccds-annotated.fits', ccds=None, **kwargs):
survey = LegacySurveyData(ccds=ccds)
if ccds is None:
ccds = survey.get_ccds()
# Set to True if we successfully read the calibration products and computed
# annotated values
init_annotations(ccds)
annotate(ccds, **kwargs)
print('Writing to', outfn)
ccds.writeto(outfn)
print('Wrote', outfn)
def __init__(self, **kw):
super(analysis_setup, self).__init__(**kw)
# raise ValueError('hey')
self.tol = Tolerances().get(survey=self.survey,
bands=self.bands,
obj=self.obj)
self.config_dir= os.path.join(os.path.dirname(self.outdir),
'testcase_%s_%s' % \
(kw['survey'],kw['bands']))
self.rsdir = 'rs0'
survey = LegacySurveyData()
brickinfo = get_brickinfo_hack(survey, self.brick)
self.brickwcs = wcs_for_brick(brickinfo)
def queue():
if False:
survey = LegacySurveyData()
ccds = survey.get_ccds()
bricks = survey.get_bricks()
print(len(bricks), 'bricks')
print(len(ccds), 'CCDs')
bricks.cut((bricks.dec >= -30) * (bricks.dec <= 30))
print(len(bricks), 'in Dec [-30, +30]')
I = survey.photometric_ccds(ccds)
ccds.cut(I)
print(len(ccds), 'pass photometric cut')
I, J, d = match_radec(bricks.ra,
bricks.dec,
ccds.ra,
ccds.dec,
0.5,
nearest=True)
print(len(I), 'bricks with CCDs nearby')
bricks.cut(I)
bricknames = bricks.brickname
else:
# DR7: use Martin's list of bricks w/ CCD coverage
f = open('nccds.dat')
bricknames = []
for line in f.readlines():
words = line.strip().split(' ')
brick = words[0]
nccd = int(words[1])
if nccd > 0:
bricknames.append(brick)
# qdo
bb = bricknames
while len(bb):
print(' '.join(bb[:100]))
bb = bb[100:]
return
mp = multiproc(16)
N = len(bricks)
args = [(brick, i, N, plots, {}) for i, brick in enumerate(bricks)]
mp.map(run_one_brick, args)
def getbrickfiles(brickname=None):
survey = LegacySurveyData()
brickinfo = survey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
ccdinfo = survey.ccds_touching_wcs(brickwcs)
nccd = len(ccdinfo)
calibdir = survey.get_calib_dir()
imagedir = survey.survey_dir
# Construct image file names and the calibration file names.
expnum = ccdinfo.expnum
ccdname = ccdinfo.ccdname
psffiles = list()
skyfiles = list()
imagefiles = list()
for ccd in ccdinfo:
info = survey.get_image_object(ccd)
for attr in ['imgfn', 'dqfn', 'wtfn']:
fn = getattr(info, attr).replace(imagedir + '/', '')
#if '160108_073601' in fn:
# pdb.set_trace()
imagefiles.append(fn)
psffiles.append(info.psffn.replace(calibdir, 'calib'))
skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))
#for ii in range(nccd):
#exp = '{0:08d}'.format(expnum[ii])
#rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
#psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
#skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
#imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))
#print(np.array(imagefiles))
#print(np.array(psffiles))
#print(np.array(skyfiles))
return imagefiles, psffiles, skyfiles
def __init__(
self,
expnum=None,
ccdname=None,
ccdwcs=None,
pattern='/project/projectdirs/cosmo/work/ps1/cats/chunks-qz-star-v3/ps1-%(hp)05d.fits'
):
"""Read PS1 or gaia sources for an exposure number + CCD name or CCD WCS
Args:
expnum, ccdname: select catalogue with these
ccdwcs: or select catalogue with this
pattern: absolute path and wildcard for PS1 or Gaia catalogues
dr: /project/projectdirs/cosmo/work/
PS1: ${dr}/ps1/cats/chunks-qz-star-v3/ps1-%(hp)05d.fits
PS1-Gaia: ${dr}/gaia/chunks-ps1-gaia/chunk-%(hp)05d.fits
"""
assert ('ps1' in pattern or 'gaia' in pattern)
#assert(ps1_or_gaia in ['ps1','ps1_gaia'])
#if ps1_or_gaia == 'ps1':
# # PS1 "qz" directory
# # e.g. /project/projectdirs/cosmo/work/ps1/cats/chunks-qz-star-v2
# self.catdir= os.getenv('PS1CAT_DIR')
#elif ps1_or_gaia == 'ps1_gaia':
# # PS1-Gaia "qz" matches-only directory
# # e.g. /project/projectdirs/cosmo/work/gaia/chunks-ps1-gaia
# self.catdir= os.getenv('PS1_GAIA_MATCHES')
#fnpattern = os.path.join(self.catdir, prefix + '-%(hp)05d.fits')
super(ps1cat, self).__init__(pattern)
if ccdwcs is None:
from legacypipe.survey import LegacySurveyData
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
self.ccdwcs = im.get_wcs()
else:
self.ccdwcs = ccdwcs
def __init__(self, expnum=None, ccdname=None, ccdwcs=None):
"""Read PS1 or gaia sources for an exposure number + CCD name or CCD WCS
Args:
expnum, ccdname: select catalogue with these
ccdwcs: or select catalogue with this
"""
self.ps1catdir = os.getenv('PS1CAT_DIR')
if self.ps1catdir is None:
raise ValueError(
'You must have the PS1CAT_DIR environment variable set to point to healpixed PS1 catalogs'
)
fnpattern = os.path.join(self.ps1catdir, 'ps1-%(hp)05d.fits')
super(ps1cat, self).__init__(fnpattern)
if ccdwcs is None:
from legacypipe.survey import LegacySurveyData
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
self.ccdwcs = im.get_wcs()
else:
self.ccdwcs = ccdwcs
def __init__(self, expnum=None, ccdname=None, ccdwcs=None):
"""Initialize the class with either the exposure number *and* CCD name, or
directly with the WCS of the CCD of interest.
"""
# GAIA and PS1 info, gaia for astrometry, ps1 for photometry
self.gaiadir = os.getenv('GAIACAT_DIR')
# PS1 only
self.ps1dir = os.getenv('PS1CAT_DIR') # PS1 only
if self.ps1dir is None:
raise ValueError('Need PS1CAT_DIR environment variable to be set.')
if self.gaiadir is None:
print(
'WARNING: GAIACAT_DIR environment variable not set: using Pan-STARRS1 for astrometry'
)
self.nside = 32
if ccdwcs is None:
from legacypipe.survey import LegacySurveyData
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
self.ccdwcs = im.get_wcs()
else:
self.ccdwcs = ccdwcs
def rbmain():
travis = 'travis' in sys.argv
extra_args = [
'--old-calibs-ok',
#'--verbose',
]
if travis:
extra_args.extend(
['--no-wise-ceres', '--no-gaia', '--no-large-galaxies'])
if 'ceres' in sys.argv:
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--ceres',
'--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres',
'--no-depth-cut'
])
sys.exit(0)
# demo RexGalaxy, with plots
if False:
from legacypipe.survey import RexGalaxy
from tractor import NanoMaggies, PixPos
from tractor import Image, GaussianMixturePSF, LinearPhotoCal
from legacypipe.survey import LogRadius
rex = RexGalaxy(PixPos(1., 2.), NanoMaggies(r=3.), LogRadius(0.))
print('Rex:', rex)
print('Rex params:', rex.getParams())
print('Rex nparams:', rex.numberOfParams())
H, W = 100, 100
tim = Image(data=np.zeros((H, W), np.float32),
inverr=np.ones((H, W), np.float32),
psf=GaussianMixturePSF(1., 0., 0., 4., 4., 0.),
photocal=LinearPhotoCal(1., band='r'))
derivs = rex.getParamDerivatives(tim)
print('Derivs:', len(derivs))
print('Rex params:', rex.getParamNames())
import pylab as plt
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('rex')
for d, nm in zip(derivs, rex.getParamNames()):
plt.clf()
plt.imshow(d.patch, interpolation='nearest', origin='lower')
plt.title('Derivative %s' % nm)
ps.savefig()
sys.exit(0)
# Test RexGalaxy
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6-rex'
main(args=[
'--brick',
'1102p240',
'--zoom',
'500',
'600',
'650',
'750',
'--force-all',
'--no-write',
'--no-wise',
#'--rex', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + extra_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0] == 'PSF ')
cmd = (
'(cd %s && sha256sum -c %s)' %
(outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
print(cmd)
rtn = os.system(cmd)
assert (rtn == 0)
# Test with a Tycho-2 star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir,
'--outdir', outdir
] + extra_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0] == 'PSF ')
# Test that we can run splinesky calib if required...
from legacypipe.decam import DecamImage
DecamImage.splinesky_boxsize = 128
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4'
fn = os.path.join(surveydir, 'calib', 'decam', 'splinesky', '00431',
'00431608', 'decam-00431608-N3.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image',
'--no-hybrid-psf', '--survey-dir', surveydir, '--outdir', outdir
] + extra_args + ['-v'])
print('Checking for calib file', fn)
assert (os.path.exists(fn))
# Wrap-around, hybrid PSF
surveydir = os.path.join(os.path.dirname(__file__), 'testcase8')
outdir = 'out-testcase8'
main(args=[
'--brick',
'1209p050',
'--zoom',
'720',
'1095',
'3220',
'3500',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + extra_args)
# Test with a Tycho-2 star + another saturated star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase7')
outdir = 'out-testcase7'
# remove --no-gaia
my_extra_args = [a for a in extra_args if a != '--no-gaia']
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia-dr2')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick',
'1102p240',
'--zoom',
'250',
'350',
'1550',
'1650',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + my_extra_args)
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 4)
# Check skipping blobs outside the brick's unique area.
# (this now doesn't detect any sources at all, reasonably)
# surveydir = os.path.join(os.path.dirname(__file__), 'testcase5')
# outdir = 'out-testcase5'
#
# fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits')
# if os.path.exists(fn):
# os.unlink(fn)
#
# main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150',
# '--force-all', '--no-write', '--coadd-bw',
# '--survey-dir', surveydir,
# '--early-coadds',
# '--outdir', outdir] + extra_args)
#
# assert(os.path.exists(fn))
# T = fits_table(fn)
# assert(len(T) == 1)
# Custom RA,Dec; blob ra,dec.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4b'
# Catalog written with one entry (--blobradec)
fn = os.path.join(outdir, 'tractor', 'cus',
'tractor-custom-186743p25461.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--radec', '186.743965', '25.461788', '--width', '250', '--height',
'250', '--force-all', '--no-write', '--no-wise', '--blobradec',
'186.740369', '25.453855', '--survey-dir', surveydir, '--outdir',
outdir
] + extra_args)
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 1)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
outdir = 'out-testcase3'
checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle')
if os.path.exists(checkpoint_fn):
os.unlink(checkpoint_fn)
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
] + extra_args)
# Read catalog into Tractor sources to test read_fits_catalog
from legacypipe.catalog import read_fits_catalog
from legacypipe.survey import LegacySurveyData, GaiaSource
from tractor.galaxy import DevGalaxy
from tractor import PointSource
survey = LegacySurveyData(survey_dir=outdir)
fn = survey.find_file('tractor', brick='2447p120')
print('Checking', fn)
T = fits_table(fn)
cat = read_fits_catalog(T, fluxPrefix='')
print('Read catalog:', cat)
assert (len(cat) == 2)
src = cat[0]
assert (type(src) == DevGalaxy)
assert (np.abs(src.pos.ra - 244.77973) < 0.00001)
assert (np.abs(src.pos.dec - 12.07233) < 0.00001)
src = cat[1]
print('Source', src)
assert (type(src) in [PointSource, GaiaSource])
assert (np.abs(src.pos.ra - 244.77830) < 0.00001)
assert (np.abs(src.pos.dec - 12.07250) < 0.00001)
# DevGalaxy(pos=RaDecPos[244.77975494973529, 12.072348111713127], brightness=NanoMaggies: g=19.2, r=17.9, z=17.1, shape=re=2.09234, e1=-0.198453, e2=0.023652,
# PointSource(RaDecPos[244.77833280764278, 12.072521274981987], NanoMaggies: g=25, r=23, z=21.7)
# Check that we can run again, using that checkpoint file.
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
] + extra_args)
# Assert...... something?
# Test --checkpoint without --threads
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1'
] + extra_args)
# MzLS + BASS data
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass')
# main(args=['--brick', '3521p002', '--zoom', '2400', '2450', '1200', '1250',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass'])
# From Kaylan's Bootes pre-DR4 run
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3')
# main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass3'] + extra_args)
# With plots!
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', 'out-testcase3', '--plots', '--nblobs', '1'
] + extra_args)
# Decals Image Simulations
# Uncomment WHEN galsim build for Travis
#os.environ["DECALS_SIM_DIR"]= os.path.join(os.path.dirname(__file__),'image_sims')
#brick= '2447p120'
#sim_main(args=['--brick', brick, '-n', '2', '-o', 'STAR', \
# '-ic', '1', '--rmag-range', '18', '26', '--threads', '1',\
# '--zoom', '1020', '1070', '2775', '2815'])
# Check if correct files written out
#rt_dir= os.path.join(os.getenv('DECALS_SIM_DIR'),brick,'star','001')
#assert( os.path.exists(os.path.join(rt_dir,'../','metacat-'+brick+'-star.fits')) )
#for fn in ['tractor-%s-star-01.fits' % brick,'simcat-%s-star-01.fits' % brick]:
# assert( os.path.exists(os.path.join(rt_dir,fn)) )
#for fn in ['image','model','resid','simscoadd']:
# assert( os.path.exists(os.path.join(rt_dir,'qa-'+brick+'-star-'+fn+'-01.jpg')) )
if not travis:
# With ceres
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--ceres',
'--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres'
] + extra_args)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--mzls',
action='store_true',
help='Set MzLS (default: DECaLS)')
parser.add_argument('--ann', help='Set annotated-CCDs file')
opt = parser.parse_args()
if opt.mzls:
from mosaic import MosaicNominalCalibration
from camera_mosaic import database_filename, camera_name
nom = MosaicNominalCalibration()
obstatus_fn = 'obstatus/mosaic-tiles_obstatus.fits'
out_fn = 'mosaic-obstatus-depth.fits'
bands = 'z'
declo, dechi = -5, 90
bad_expid_fn = 'obstatus/bad_expid.txt'
else:
from decam import DecamNominalCalibration
from camera_decam import database_filename, camera_name
nom = DecamNominalCalibration()
# ln -s ~/observing/obstatus/bad_expid.txt obstatus/decam-bad_expid.txt
obstatus_fn = 'obstatus/decam-tiles_obstatus.fits'
out_fn = 'decam-obstatus-depth.fits'
bad_expid_fn = 'obstatus/decam-bad_expid.txt'
bands = 'grz'
declo, dechi = -20, 35
f = open(bad_expid_fn)
bad_expids = set()
for line in f:
line = line.strip()
if len(line) == 0:
continue
if line[0] == '#':
continue
words = line.split()
try:
expnum = int(words[0])
except:
print('Skipping line:', line)
continue
bad_expids.add(expnum)
print('Read', len(bad_expids), 'bad exposure numbers')
# Convert copilot db to fits.
import obsdb
from copilot import db_to_fits
obsdb.django_setup(database_filename=database_filename)
ccds = obsdb.MeasuredCCD.objects.all()
copilot = db_to_fits(ccds)
all_copilot = copilot.copy()
fn = 'copilot.fits'
copilot.writeto(fn)
print('Wrote', fn)
print(len(copilot), 'measured CCDs in copilot database')
copilot.cut(np.array([c.strip() == camera_name for c in copilot.camera]))
print(len(copilot), 'copilot CCDs with camera = "%s"' % camera_name)
copilot.cut(copilot.expnum > 0)
print(len(copilot), 'measured CCDs in copilot database with EXPNUM')
print('Copilot expfactor extremes:',
np.percentile(copilot.expfactor[copilot.expfactor != 0], [1, 99]))
survey = LegacySurveyData()
if opt.ann:
ccds = fits_table(opt.ann)
else:
print('Reading annotated CCDs files...')
ccds = survey.get_annotated_ccds()
print(len(ccds), 'CCDs')
# Fix parsing of OBJECT field to tileid...
from obsbot import get_tile_id_from_name
tileids = []
for o in ccds.object:
tid = get_tile_id_from_name(o.strip())
if tid is None:
tid = 0
tileids.append(tid)
tileids = np.array(tileids)
print(len(np.unique(tileids)),
'unique tile ids in annotated file, from OBJECT')
print(len(np.unique(ccds.tileid)),
'unique tile ids in ann file from TILEID')
D = np.flatnonzero(tileids != ccds.tileid)
print(len(D), 'different tileids')
print('From OBJECT:', tileids[D])
print('From TILEID:', ccds.tileid[D])
ccds.tileid = tileids
O = fits_table(obstatus_fn)
print(len(O), 'tiles')
if opt.mzls:
from camera_mosaic import fix_expnums
# Fix MzLS exposure numbers with wrong leading "3".
fix_expnums(ccds.expnum)
# Also fix leading "3" in expnums in OBSTATUS file
fix_expnums(O.z_expnum)
# And copilot database
fix_expnums(copilot.expnum)
print('Z_EXPNUM range:', O.z_expnum.min(), 'min >0:',
O.z_expnum[O.z_expnum > 0].min(), O.z_expnum.max())
print('Pass numbers:', np.unique(O.get('pass')))
if opt.mzls:
goodtiles = (O.in_desi * (O.dec > 30) * (O.get('pass') <= 3))
print(sum(goodtiles), 'tiles of interest')
else:
goodtiles = (O.in_desi * (O.get('pass') <= 3))
print(sum(goodtiles), 'tiles in the footprint')
#O.cut(goodtiles)
#print('Cut to', len(O), 'tiles of interest')
# *after* fixing tileids
allccds = ccds.copy()
# Map tile IDs back to index in the obstatus file.
tileid_to_index = np.empty(max(O.tileid) + 1, int)
tileid_to_index[:] = -1
tileid_to_index[O.tileid] = np.arange(len(O))
assert (len(np.unique(O.tileid)) == len(O))
I = tileid_to_index[O.tileid]
assert (np.all(I == np.arange(len(O))))
# Look at whether exposures from other programs are near our tile centers.
# Basically nope.
# plt.clf()
# e,K = np.unique(ccds.expnum, return_index=True)
# I,J,d = match_radec(O.ra, O.dec, ccds.ra_bore[K], ccds.dec_bore[K],
# 1./60., nearest=True)
# KK = K[np.flatnonzero(ccds.tileid[K] > 0)]
# I,J,d2 = match_radec(O.ra, O.dec, ccds.ra_bore[KK], ccds.dec_bore[KK],
# 1./60., nearest=True)
# ha = dict(range=(0., 60.), bins=60, histtype='step')
# plt.hist(d * 3600., color='b', **ha)
# plt.hist(d2 * 3600., color='r', **ha)
# plt.xlabel('Distance from tile to nearest DECam boresight (arcsec)')
# plt.savefig('dists.png')
notileids = ccds[ccds.tileid <= 0]
print(len(notileids), 'CCDs have no tileid')
I, J, d = match_radec(notileids.ra_bore,
notileids.dec_bore,
O.ra,
O.dec,
0.5,
nearest=True)
plt.clf()
plt.hist(d, bins=50)
plt.xlabel('Distance to nearest tile center (deg)')
plt.savefig('tiledist.png')
plt.clf()
plt.hist(d * 3600, bins=50, range=(0, 30))
plt.xlabel('Distance to nearest tile center (arcsec)')
plt.savefig('tiledist2.png')
ccds.cut(ccds.tileid > 0)
print(len(ccds), 'CCDs with tileid')
expnums, I = np.unique(ccds.expnum, return_index=True)
print(len(expnums), 'unique exposures (with tileids)')
ccds.photometric = (ccds.ccd_cuts == 0)
# Compute the mean depth per exposure
E = ccds[I]
for expnum in expnums:
I = np.flatnonzero(ccds.expnum == expnum)
j = np.flatnonzero(E.expnum == expnum)
assert (len(j) == 1)
j = j[0]
E.photometric[j] = np.all(ccds.photometric[I])
#E.photometric[j] = np.all(ccds.ccd_cuts[I] == 0)
if len(np.unique(ccds.photometric[I])) == 2:
print('Exposure', expnum,
'has photometric and non-photometric CCDs')
non = I[ccds.photometric[I] == False]
phot = I[ccds.photometric[I]]
if opt.mzls and len(phot) == 3:
print('Accepting an exposure with 3 good CCDs')
E.photometric[j] = True
# And remove this exposure from the bad_expid list.
if expnum in bad_expids:
bad_expids.remove(expnum)
print('Removing exposure', expnum, 'from bad_expid file')
continue
for ii in non:
print(
' http://legacysurvey.org/viewer-dev/?ra=%.3f&dec=%.3f&zoom=11&ccds3&bad=%i-%s'
% (ccds.ra_center[ii], ccds.dec_center[ii], expnum,
ccds.ccdname[ii]))
print(
' http://legacysurvey.org/viewer-dev/ccd/decals-dr5/decam-%s-%s-%s/'
% (ccds.expnum[ii], ccds.ccdname[ii], ccds.filter[ii]))
print(' image:', ccds.image_filename[I][0])
print(' boresight:', ccds.ra_bore[I][0], ccds.dec_bore[I][0])
#print(' ccdnames:', ccds.ccdname[I])
print(' photometric:', len(phot), ', non-photometric:', len(non))
print(' median phot depth:', np.median(ccds.galdepth[phot]))
#print(' depth:', ccds.galdepth[I])
print(' non-photometric CCDs:', ccds.ccdname[non])
print(' depths:', ccds.galdepth[non])
print(' ccdnmatch', ccds.ccdnmatch[non], 'vs',
ccds.ccdnmatch[phot])
print(' ccdtransp:', ccds.ccdtransp[non], 'vs',
ccds.ccdtransp[phot])
print(' ccd zpt vs frame zpt:',
ccds.ccdzpt[non] - ccds.zpt[non])
dp = ccds.ccdzpt[phot] - ccds.zpt[phot]
print(' phot ccds zpt vs frame: range', dp.min(), dp.max(),
'mean', dp.mean())
whitelist = [
346662,
346664,
346665, # S3/S29 striping
346754, # one bad chip, wispy
346967,
347304, # M5 globular
347664, # zpt scatter
347744, # weird eye-shaped ghost; but lots of cov.
347755,
347768,
347769,
347782, # shallow, zpt scatter -- wispy pattern on focal plane
347918,
347920, # straddling transparency cut
347934,
347936,
347941,
347945,
347947, # zpt scatter
392377,
392380,
393173, # bright star
393671, # bright star
393672,
393673, # scatter
425339,
425340, # strangely low ccdnmatch
426225,
430808, # globular cluster
431640, # bright star
431644, # globular
432154, # one amp high bias
432159, # transp. on boundary
432179, # one amp high bias, +
432747,
432748,
432751, # scatter
433305,
433306, # bright star
497062,
497064,
497065, # low ccdnmatch
509516,
509517, # bright star
511247,
511263, # low ccdnmatch
511513,
511514, # bright star
512303, # bright star
520560, # bright star
521782, # scatter
522212, # bright star
535138, # bright stars, satellite hits?
535141,
535142,
535143,
535149, # low ccdnmatch
535210, # bright star
535695, # globular
536065, # globular
536385, # strangely zero ccdnmatch
547761, # nice galaxy
548257, # bright star
553779, # scatter
553795, # shallow
554284, # marginal zpt
563659, # zpt scatter
563850, # mild striping
563852, # ??
583118, # bright stars?
592621, # marginal zpt, scatter
592859, # some pattern noise
605068, # ??
625710, # strangely low ccdnmatch
631005, # bright star
634493, # globular
634786, # strangely low ccdnmatch
634877, # globular
635535, # bright star, glob
635962, # low ccdnmatch
635973, # bias level?
636018, # bias level?
637688, # bright star
]
blacklist = [
425328, # 2.7" seeing
488244,
488256,
488260,
488261,
488263,
488268, # weird striping
488270, # weird striping
496913,
496917,
496918,
496919,
496920,
496921,
496922, # 3" seeing
496923,
496925,
496926,
496927,
496928,
496930, # 3" seeing
509162,
509163,
509166,
509172,
509176,
509182,
509202, # 3" seeing
535471, # 4" seeing!
535498, # 3" seeing
548218, # double PSF -- telescope moved?
563835, # striping
563842, # striping
]
if expnum in whitelist:
print('** Exposure', expnum,
'in whitelist -- marking as photometric')
E.photometric[j] = True
# Don't include zeros in computing average depths!
Igood = I[(ccds.galdepth[I] > 0) * (ccds.ccdzpt[I] < 30)]
if len(Igood) > 0:
E.galdepth[j] = np.mean(ccds.galdepth[Igood])
else:
E.galdepth[j] = 0.
del expnums
keep = np.array([not (expnum in bad_expids) for expnum in ccds.expnum])
ccds.cut(keep)
print(len(ccds), 'CCDs NOT in the bad_expids file')
keep = np.array([not (expnum in bad_expids) for expnum in copilot.expnum])
copilot.cut(keep)
print(len(copilot), 'copilot exposures NOT in the bad_expids file')
keep = np.array([not (expnum in bad_expids) for expnum in E.expnum])
E.cut(keep)
print(len(E), 'CCD Exposures NOT in the bad_expids file')
# plt.clf()
# plt.plot(O.ra, O.dec, 'k.')
# plt.axis([360,0,-25,35])
# plt.title('All tiles')
# plt.savefig('tiles-all.png')
#
# print('in_desi:', np.unique(O.in_desi))
# plt.clf()
# J = np.flatnonzero(O.in_desi == 1)
# plt.plot(O.ra[J], O.dec[J], 'k.')
# plt.axis([360,0,-25,35])
# plt.title('In DESI')
# plt.savefig('tiles-desi.png')
#
# print('in_des:', np.unique(O.in_des))
# plt.clf()
# J = np.flatnonzero(O.in_des == 1)
# plt.plot(O.ra[J], O.dec[J], 'k.')
# plt.axis([360,0,-25,35])
# plt.title('IN DES')
# plt.savefig('tiles-des.png')
#print('Number of exposures of each tile:')
#print(Counter(E.tileid).most_common())
print()
print()
print('Number of exposures of tiles:')
for band in bands:
I = np.flatnonzero(E.filter == band)
c = Counter(E.tileid[I])
c2 = Counter([v for k, v in c.most_common()])
print(' ', band, 'band:', c2.most_common())
# Detection inverse-variance is the quantity that adds when there are
# multiple exposures.
# detsig1 = ccds.sig1 / ccds.galnorm_mean
# depth = 5. * detsig1
# # that's flux in nanomaggies -- convert to mag
# ccds.galdepth = -2.5 * (np.log10(depth) - 9)
with np.errstate(divide='ignore', over='ignore'):
# actually 5*detsig1...
detsig = 10.**((E.galdepth - 22.5) / -2.5)
E.detiv = 1. / detsig**2
E.detiv[E.galdepth == 0] = 0.
print('Smallest detivs:', E.detiv[np.argsort(E.detiv)[:10]])
print('w/ galdepths:', E.galdepth[np.argsort(E.detiv)[:10]])
print('Smallest positive detivs:',
E.detiv[np.argsort(E.detiv + 1e12 * (E.detiv == 0))[:10]])
print('w/ galdepths:',
E.galdepth[np.argsort(E.detiv + 1e12 * (E.detiv == 0))[:10]])
for band in bands:
print()
print('------------------')
print(band, 'band.')
# "I" indexes into exposures E.
I = np.flatnonzero(
(E.filter == band) * E.photometric * np.isfinite(E.detiv))
print(len(I), 'photometric exposures in', band)
# "iv" is parallel to O; will be converted to "galdepth".
iv = np.zeros(len(O), np.float32)
# "J" indexes into obstatus tiles O.
J = tileid_to_index[E.tileid[I]]
assert (np.all((J >= 0) * (J < len(O))))
assert (np.all(O.tileid[J] == E.tileid[I]))
#print('tileid range', E.tileid[I].min(), E.tileid[I].max())
# d = np.array([degrees_between(*a) for a in
# zip(E.ra_bore[I], E.dec_bore[I], O.ra[J], O.dec[J])])
# print('Degrees between tiles & exposures:', d)
np.add.at(iv, J, E.detiv[I])
print('galdepth range:', E.galdepth[I].min(), E.galdepth[I].max())
print('detiv range:', E.detiv[I].min(), E.detiv[I].max())
#print('index range:', J.min(), J.max())
nexp = np.zeros(len(O), int)
np.add.at(nexp, J, 1)
print('tile exposure counts:', Counter(nexp))
# convert iv back to galdepth in mags
with np.errstate(divide='ignore'):
galdepth = -2.5 * (np.log10(np.sqrt(1. / iv)) - 9)
galdepth[iv == 0] = 0.
# Shallowest before extinction correction
#I = np.argsort(iv + 1e6*(iv == 0))
I = np.argsort(galdepth + 50. * (galdepth == 0))
print(
'Shallowest depth estimates from annotated CCDs file, before extinction:'
)
for i in I[:10]:
print(' ', galdepth[i], 'iv', iv[i], 'tile', O.tileid[i],
'expnum',
O.get('%s_expnum' % band)[i])
e = O.get('%s_expnum' % band)[i]
j = np.flatnonzero(E.expnum == e)
print(' galdepth', E.galdepth[j])
fid = nom.fiducial_exptime(band)
extinction = O.ebv_med * fid.A_co
#print('Extinction range:', extinction.min(), extinction.max())
galdepth -= extinction
galdepth[iv == 0] = 0.
# Shallowest galdepth > 0
I = np.argsort(galdepth + 50. * (galdepth == 0))
print('Shallowest depth estimates from annotated CCDs file:')
for i in I[:10]:
print(' ', galdepth[i], 'tile', O.tileid[i], 'expnum',
O.get('%s_expnum' % band)[i])
#print('galdepth deciles:', np.percentile(galdepth, [0,10,20,30,40,50,60,70,80,90,100]))
# Z_DONE, Z_EXPNUM but no Z_DEPTH
missing_depth = np.flatnonzero(
(O.get('%s_expnum' % band) > 0) * (O.get('%s_done' % band) == 1) *
(galdepth == 0))
print('Found', len(missing_depth), 'tiles with', band,
'DONE and EXPNUM but no DEPTH; setting to DEPTH=30')
print(' eg, EXPNUMs',
O.get('%s_expnum' % band)[missing_depth[:10]], 'DATE',
O.get('%s_date' % band)[missing_depth[:10]])
# Don't actually update 'galdepth[missing_depth]' until after this next check...
# Flag tiles that have *only* non-photometric exposures with depth = 1.
I = np.flatnonzero((E.filter == band) * np.logical_not(E.photometric))
print(len(I), 'exposures are non-photometric in', band, 'band')
J = tileid_to_index[E.tileid[I]]
only_nonphot = J[galdepth[J] == 0.]
print(len(only_nonphot), 'tiles have only non-photometric exposures')
print('Marking', len(only_nonphot), 'non-photometric tiles in', band,
'with depth=1')
orig_galdepth = galdepth.copy()
galdepth[missing_depth] = 30.
galdepth[only_nonphot] = 1.
J = tileid_to_index[E.tileid[I]]
nonphot = (galdepth[J] == 1.)
print('Non-photometric galdepths:', E.galdepth[I])
print('Non-photometric galdepths:', E.galdepth[I[nonphot]])
plt.clf()
phot = np.flatnonzero((E.filter == band) * E.photometric)
plt.hist(E.galdepth[phot],
range=(18, 26),
bins=50,
histtype='step',
color='b',
label='Photometric')
plt.hist(E.galdepth[I[nonphot]],
range=(18, 26),
bins=50,
histtype='step',
color='r',
label='Non-phot')
plt.legend()
plt.savefig('nonphot-%s.png' % band)
# expnum_to_copilot = np.empty(expnums.max()+1, int)
# expnum_to_copilot[:] = -1
# expnum_to_copilot[copilot.expnum] = np.arange(len(copilot))
expnum_to_copilot = dict([(e, i)
for i, e in enumerate(copilot.expnum)])
if False:
# Let's check the accuracy of the copilot's depth estimates...
target_exptime = copilot.expfactor * fid.exptime
# What fraction of the target exposure time did we take?
depth_factor = copilot.exptime / target_exptime
nomdepth = fid.single_exposure_depth
depth = nomdepth + 2.5 * np.log10(np.sqrt(depth_factor))
#print('Copilot predicted depths:', depth)
IC = np.array(
[expnum_to_copilot.get(e, -1) for e in allccds.expnum])
K = np.flatnonzero(IC >= 0)
ext = np.array([
e['ugrizY'.index(f)]
for e, f in zip(allccds.decam_extinction, allccds.filter)
])
dd = allccds.galdepth - ext
print('Making scatterplot...', len(K), 'points')
plt.clf()
#plt.plot(dd[K], depth[IC[K]], 'b.', alpha=0.2, mec='none')
plt.scatter(dd[K],
depth[IC[K]],
c=np.clip(copilot.expfactor[IC[K]], 0, 2),
s=10,
alpha=0.2,
edgecolors='none')
plt.colorbar()
plt.xlabel('Pipeline depth')
plt.ylabel('Copilot depth proxy')
plt.plot([20, 25], [20, 25], 'k-', alpha=0.25)
plt.plot([20, 25], [20 + 0.1, 25 + 0.1], 'k--', alpha=0.25)
plt.plot([20, 25], [20 - 0.1, 25 - 0.1], 'k--', alpha=0.25)
plt.axis([20.5, 23, 21, 23.5])
plt.title(
'Copilot vs Pipeline depth estimates. (color = exp.factor)')
plt.savefig('depth-copilot-%s.png' % band)
print('Made scatterplot')
plt.clf()
ha = dict(bins=60, range=(0, 30), log=True, histtype='step')
plt.hist(O.get('%s_depth' % band), color='k', label='Before', **ha)
plt.hist(orig_galdepth, color='b', label='Annotated CCDs', **ha)
# Do we have measurements for any of these missing tiles in the copilot db?
for code in [30, 0]:
Igal = np.flatnonzero(
(O.get('%s_expnum' % band) > 0) *
(O.get('%s_done' % band) == 1) * (galdepth == code))
expnums = O.get('%s_expnum' % band)[Igal]
print(
len(expnums),
'still marked DONE, with EXPNUM, but missing DEPTH, with code =',
code)
Ihuh = np.flatnonzero(
(O.get('%s_done' % band) == 1) * (galdepth == code))
print(len(Ihuh), 'tiles marked DONE, without EXPNUM, and DEPTH =',
code)
if len(Ihuh):
print('Tile ids:', O.tileid[Ihuh])
for t in O.tileid[Ihuh]:
I = np.flatnonzero(E.tileid == t)
print(' tile', t, ': exposure numbers:', E.expnum[I])
print(' with depths', E.galdepth[I])
i = tileid_to_index[t]
if i >= 0:
print(' depth', galdepth[i])
else:
print(' no depth')
# Within an arcmin?
I, J, d = match_radec(O.ra[Ihuh],
O.dec[Ihuh],
all_copilot.rabore,
all_copilot.decbore,
1. / 60.,
nearest=True)
print('For', len(Ihuh), 'weird tiles,')
print(len(I), 'matches within an arcmin in the copilot db')
print('Smallest distances:', d[np.argsort(d)[:10]])
I, J, d = match_radec(O.ra[Ihuh],
O.dec[Ihuh],
allccds.ra_bore,
allccds.dec_bore,
1. / 60.,
nearest=True)
print(len(I), 'matches within an arcmin in the CCDs table')
print('Smallest distances:', d[np.argsort(d)[:10]])
O[Ihuh].writeto('weird-%s-%i.fits' % (band, code))
if len(expnums) == 0:
continue
IC = np.array([expnum_to_copilot.get(e, -1) for e in expnums])
K = np.flatnonzero(IC >= 0)
expnums = expnums[K]
# these are the indices into O / galdepth
Igal = Igal[K]
co = copilot[IC[K]]
print(len(expnums), 'matched to copilot database')
target_exptime = co.expfactor * fid.exptime
# What fraction of the target exposure time did we take?
depth_factor = co.exptime / target_exptime
nomdepth = fid.single_exposure_depth
print('Nominal single-exposure depth:', nomdepth)
co.depth = nomdepth + 2.5 * np.log10(np.sqrt(depth_factor))
print('Copilot predicted depths:', co.depth)
J = np.flatnonzero(np.isfinite(co.depth))
co = co[J]
# indices into O
Igal = Igal[J]
print(len(Igal), 'good copilot depth estimates')
pcts = [0, 1, 5, 25, 50, 75, 95, 99, 100]
print('Copilot depth percentiles:', np.percentile(co.depth, pcts))
print('Shallowest exposures:')
I = np.argsort(co.depth)
for i in I[:10]:
print(' Expnum', co.expnum[i], 'depth', co.depth[i],
'exptime', co.exptime[i])
co[I].writeto('depths.fits')
from astrometry.util.starutil_numpy import mjdtodate
print('Copilot-matched entries:')
I = np.argsort(co.expnum)
for i in I:
print(' EXPNUM', co.expnum[i], 'date',
mjdtodate(co.mjd_obs[i]), ' copilot name',
co.filename[i])
# e = co.expnum[i]
# I = np.flatnonzero(allccds.expnum == e-1)
# fn1 = None
# fn2 = None
# if len(I):
# print(' CCDs file contains', len(I), 'entries for expnum', e-1)
# print(' filename', allccds.image_filename[I[0]])
# fn1 = allccds.image_filename[I[0]]
# else:
# print(' No CCDs file entries for expnum', e-1)
# I = np.flatnonzero(allccds.expnum == e+1)
# if len(I):
# print(' CCDs file contains', len(I), 'entries for expnum', e+1)
# print(' filename', allccds.image_filename[I[0]])
# fn2 = allccds.image_filename[I[0]]
# else:
# print(' No CCDs file entries for expnum', e+1)
#
# if fn1 is not None and fn2 is not None:
# full1 = os.path.join(survey.get_image_dir(), fn1)
# #print('Full path 1:', full1)
# if os.path.exists(full1):
# print('exists')
# full2 = os.path.join(survey.get_image_dir(), fn2)
# #print('Full path 2:', full2)
# if os.path.exists(full2):
# print('exists')
# if os.path.exists(full1) and os.path.exists(full2):
# dir1 = os.path.dirname(full1)
# dir2 = os.path.dirname(full2)
# if dir1 == dir2:
# #print('dir:', dir1)
# fns = os.listdir(dir1)
# fns.sort()
# fns = [fn for fn in fns if ('oki' in fn or 'ooi' in fn)]
# base1 = os.path.basename(full1)
# base2 = os.path.basename(full2)
# i1 = fns.index(base1)
# i2 = fns.index(base2)
# print('Files found at list elements', i1, i2)
# #print(fns[i1:i2+1])
# for fn in fns[i1:i2+1]:
# print('EXPNUM', e, 'range', os.path.join(os.path.dirname(fn1), fn))
# if i1 + 4 == i2:
# print('EXPNUM', e, 'expected', os.path.join(os.path.dirname(fn1), fns[i1+2]))
# if i1 + 2 == i2:
# print('EXPNUM', e, 'expected', os.path.join(os.path.dirname(fn1), fns[i1+1]))
#print('Before:', galdepth[Igal])
galdepth[Igal] = co.depth
#print('After:', galdepth[Igal])
Igal = np.flatnonzero(
(O.get('%s_expnum' % band) > 0) *
(O.get('%s_done' % band) == 1) * (galdepth == code))
expnums = O.get('%s_expnum' % band)[Igal]
print(
len(expnums),
'still marked DONE, with EXPNUM, but missing DEPTH with code =',
code, 'after copilot patching')
print('Exposure numbers:', expnums)
print('Exposure dates:', O.get('%s_date' % band)[Igal])
print('Date counter:',
Counter(O.get('%s_date' % band)[Igal]).most_common())
O.set('%s_depth' % band, galdepth)
plt.hist(O.get('%s_depth' % band), color='r', label='After', **ha)
plt.savefig('depth-hist-%s.png' % band)
#print('Depth deciles: [', ', '.join(['%.3f' % f for f in np.percentile(O.get('%s_depth' % band), [0,10,20,30,40,50,60,70,80,90,100])]) + ']')
rlo, rhi = 0, 360
dlo, dhi = declo, dechi
J = np.flatnonzero(
(O.in_desi == 1) * (O.in_des == 0) * (O.dec > dlo) * (O.dec < dhi))
print('Median E(B-V) in DECaLS area:', np.median(O.ebv_med[J]))
print('Median extinction in DECaLS area, %s band:' % band,
np.median(extinction[J]))
I2 = np.flatnonzero((O.get('%s_expnum' % band) > 0) *
(O.get('%s_depth' % band) == 30) *
(O.get('%s_done' % band) == 1) * (O.in_desi == 1))
print(len(I2), 'with EXPNUM and DONE and IN_DESI, but no DEPTH')
# Sort by expnum
I2 = I2[np.argsort(O.get('%s_expnum' % band)[I2])]
print('Exposure numbers:', sorted(O.get('%s_expnum' % band)[I2]))
print('Dates:', sorted(O.get('%s_date' % band)[I2]))
print('Dates:', np.unique(O.get('%s_date' % band)[I2]))
for i in I2:
print(' date',
O.get('%s_date' % band)[i], 'expnum',
O.get('%s_expnum' % band)[i])
# for i2,o in zip(I2, O[I2]):
# print()
# e = o.get('%s_expnum' % band)
# print(' Expnum', e, 'orig galdepth', orig_galdepth[i2])
# date = o.get('%s_date' % band)
# print(' Date', date)
# print(' Pass', o.get('pass'), 'Tile', o.tileid)
# jj = np.flatnonzero(allccds.expnum == e)
# print(' In DESI:', o.in_desi, 'In DES:', o.in_des)
# print(' ', len(jj), 'matching CCDs')
# if len(jj) == 0:
# continue
# print(' CCDs OBJECT', [ob.strip() for ob in allccds.object[jj]])
# print(' CCDs Tileid', allccds.tileid[jj])
# print(' CCDs galdepth', allccds.galdepth[jj])
# print(' CCDs photometric', allccds.photometric[jj])
#
# ii = np.flatnonzero(E.expnum == e)
# print(' ', len(ii), 'Exposures matching')
# if len(ii):
# ee = E[ii[0]]
# print(' exposure tileid', ee.tileid)
# print(' index', tileid_to_index[ee.tileid])
# print(' vs i2=', i2)
# print(' only_nonphot', only_nonphot[i2], 'missing_depth', missing_depth[i2])
#
# kk = np.flatnonzero(allccds.tileid == o.tileid)
# kk = np.array(sorted(set(kk) - set(jj)))
# print(' ', len(kk), 'other CCDs of this tile')
# #print('Dates:', O.get('%s_date' % band)[I])
from astrometry.util.plotutils import antigray
rr, dd = np.meshgrid(np.linspace(rlo, rhi, 720),
np.linspace(dlo, dhi, 360))
JJ, II, d = match_radec(rr.ravel(),
dd.ravel(),
O.ra,
O.dec,
1.5,
nearest=True)
indesi = np.zeros(rr.shape, bool)
indesi.flat[JJ] = ((O.in_desi[II] == 1) * (O.in_des[II] == 0))
plt.figure(figsize=(14, 6))
plt.subplots_adjust(left=0.1, right=0.99)
for passnum in [1, 2, 3]:
print('Pass', passnum)
plt.clf()
J = np.flatnonzero(
(O.in_desi == 1) * (O.in_des == 0) * (O.dec > dlo) *
(O.dec < dhi) * (O.get('pass') == passnum))
#plt.plot(O.ra[J], O.dec[J], 'k.', alpha=0.5)
# Plot the gray background showing the in_desi footprint
plt.imshow(indesi,
extent=[rlo, rhi, dlo, dhi],
vmin=0,
vmax=4,
cmap=antigray,
aspect='auto',
interpolation='nearest',
origin='lower')
depth = O.get('%s_depth' % band)
#J = np.flatnonzero((O.get('pass') == passnum) * (depth > 0))
J = np.flatnonzero(
(O.get('pass') == passnum) * (depth > 1) * (depth < 30))
# print('Depths:', depth[J])
pct = np.percentile(depth[J],
[0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100])
#print('Depth deciles:', np.percentile(depth[J], [0,10,20,30,40,50,60,70,80,90,100]))
print('Depth deciles: [',
', '.join(['%.3f' % f for f in pct]) + ']')
if len(J) == 0:
sys.exit(0)
target = fid.single_exposure_depth
print('Target depth:', target)
cmap = cmap_discretize('RdBu', 11)
dm = 0.275
plt.scatter(O.ra[J],
O.dec[J],
c=depth[J] - target,
linewidths=0,
cmap=cmap,
vmin=-dm,
vmax=+dm,
zorder=-10,
s=1)
plt.colorbar(ticks=np.arange(-0.25, 0.251, 0.05))
hh, ww = rr.shape
rgba = np.zeros((hh, ww, 4), np.float32)
JJ, II, d = match_radec(rr.ravel(),
dd.ravel(),
O.ra[J],
O.dec[J],
1.,
nearest=True)
Jy, Jx = np.unravel_index(JJ, rr.shape)
rgba[Jy, Jx, :] = cmap((np.clip(depth[J[II]] - target, -dm, dm) -
(-dm)) / (dm - (-dm)))
plt.imshow(rgba,
extent=[rlo, rhi, dlo, dhi],
aspect='auto',
interpolation='nearest',
origin='lower')
I = np.flatnonzero((depth == 0) * (O.get('%s_done' % band) == 1) *
(O.get('pass') == passnum))
plt.plot(O.ra[I], O.dec[I], 'g.')
plt.title('Band %s, Pass %i' % (band, passnum))
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.axis([rhi, rlo, dlo, dhi])
plt.savefig('depth-%s-%i.png' % (band, passnum))
plt.clf()
print('Fiducial single-exposure-depth:', fid.single_exposure_depth)
for passnum in [1, 2, 3]:
depth = O.get('%s_depth' % band)
J = np.flatnonzero(
(O.get('pass') == passnum) * (depth > 1) * (depth < 30))
depth = depth[J]
print('Pass', passnum)
print(sum(depth < fid.single_exposure_depth - 0.25), 'of',
len(depth), 'tiles are more than 0.25 mag shallow')
odepth = O.get('%s_depth' % band)
K = np.flatnonzero(
(O.get('%s_done' % band) == 0) * (O.get('pass') == passnum) *
(odepth > 1) * (odepth < 30))
print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
len(odepth[K]),
'DONE=0 tiles are more than 0.25 mag shallow')
for k in K:
print(' EXPNUM',
O.get('%s_expnum' % band)[k], 'DATE',
O.get('%s_date' % band)[k], 'DEPTH',
O.get('%s_depth' % band)[k])
K = np.flatnonzero(
(O.get('%s_done' % band) == 1) * (O.get('pass') == passnum) *
(odepth > 1) * (odepth < 30))
print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
len(odepth[K]),
'DONE=1 tiles are more than 0.25 mag shallow')
K = np.flatnonzero((O.get('%s_done' % band) == 1) *
(O.get('pass') == passnum) * (odepth == 1))
print(len(K), 'DONE=1 tiles have DEPTH=1 (non-photometric)')
K = np.flatnonzero((O.get('%s_done' % band) == 1) *
(O.get('pass') == passnum) * (odepth == 30))
print(len(K), 'DONE=1 tiles have DEPTH=30 (unknown depth)')
K = np.flatnonzero((O.get('%s_done' % band) == 1) *
(O.get('pass') == passnum) * (odepth == 0))
print(len(K), 'DONE=1 tiles have DEPTH=0')
K = np.flatnonzero((O.get('%s_done' % band) == 0) *
(O.get('pass') == passnum) * (odepth != 0))
print(len(K), 'tiles have DONE=0 but DEPTH != 0')
mlo, mhi = 21, 24
plt.hist(np.clip(depth, mlo, mhi),
bins=100,
range=(mlo, mhi),
histtype='step',
color=' bgr'[passnum],
label='Pass %i' % passnum)
plt.axvline(fid.single_exposure_depth, color='k')
plt.axvline(fid.single_exposure_depth - 0.25,
color='k',
linestyle='--')
plt.xlabel('Depth (mag)')
plt.legend(loc='upper left')
plt.title('Depth: %s' % band)
plt.savefig('depth-%s.png' % band)
for passnum in [1, 2, 3]:
depth = O.get('%s_depth' % band)
roi = ((O.in_desi == 1) * (O.in_des == 0) * (O.dec > dlo) *
(O.dec < dhi) * (O.get('pass') == passnum))
J = np.flatnonzero(roi)
done = np.flatnonzero(roi * (O.get('%s_done' % band) == 1))
redo = np.flatnonzero(roi * np.logical_or(
(depth > 1) * (depth < 30) *
(depth < fid.single_exposure_depth - 0.25), depth == 1))
print(
'Band %s, pass %i: total tiles %i, done %i, redo %i, keep %i' %
(band, passnum, len(J), len(done), len(redo),
len(done) - len(redo)))
A = np.flatnonzero(roi * (depth > 1) * (depth < 30) *
(depth > fid.single_exposure_depth - 0.25))
B = np.flatnonzero(roi * (depth > 1) * (depth < 30) *
(depth <= fid.single_exposure_depth - 0.25))
C = np.flatnonzero(roi * (depth == 1))
D = np.flatnonzero(roi * (depth == 30))
print(
'Band %s, pass %i: total tiles: %i, A: %i, B: %i, C: %i, D: %i'
% (band, passnum, len(J), len(A), len(B), len(C), len(D)))
plt.clf()
plt.plot(O.ra[J], O.dec[J], 'ko', alpha=0.1)
plt.plot(O.ra[done], O.dec[done], 'k.')
plt.plot(O.ra[redo], O.dec[redo], 'r.')
plt.axis([360, 0, -20, 38])
plt.title('Tiles to redo: %s band, pass %i: %i of %i' %
(band, passnum, len(redo), len(done)))
plt.savefig('redo-%s-%i.png' % (band, passnum))
if band == 'z':
redo = np.flatnonzero(
(O.get('pass') == passnum) * np.logical_or(
(depth > 1) * (depth < 30) *
(depth < fid.single_exposure_depth - 0.5), depth == 1))
A = np.flatnonzero(roi * (depth > 1) * (depth < 30) *
(depth > fid.single_exposure_depth - 0.5))
B = np.flatnonzero(roi * (depth > 1) * (depth < 30) *
(depth <= fid.single_exposure_depth - 0.5))
print(
'Band %s, pass %i: total tiles: %i, A: %i, B: %i, C: %i, D: %i (shallow = 0.5 mag less than target)'
% (band, passnum, len(J), len(A), len(B), len(C), len(D)))
plt.clf()
plt.plot(O.ra[J], O.dec[J], 'ko', alpha=0.1)
plt.plot(O.ra[done], O.dec[done], 'k.')
plt.plot(O.ra[redo], O.dec[redo], 'r.')
plt.axis([360, 0, -20, 38])
plt.title(
'Tiles to redo (> 0.5 mag shallow): %s band, pass %i: %i of %i'
% (band, passnum, len(redo), len(done)))
plt.savefig('redo2-%s-%i.png' % (band, passnum))
print('Passes 1-3 combined:')
depth = O.get('%s_depth' % band)
J = np.flatnonzero((depth > 1) * (depth < 30))
depth = depth[J]
print(sum(depth < fid.single_exposure_depth - 0.25), 'of', len(depth),
'tiles are more than 0.25 mag shallow')
odepth = O.get('%s_depth' % band)
K = np.flatnonzero(
(O.get('%s_done' % band) == 0) * (odepth > 1) * (odepth < 30))
print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
len(odepth[K]), 'DONE=0 tiles are more than 0.25 mag shallow')
for k in K:
print(' EXPNUM',
O.get('%s_expnum' % band)[k], 'DATE',
O.get('%s_date' % band)[k], 'DEPTH',
O.get('%s_depth' % band)[k])
K = np.flatnonzero(
(O.get('%s_done' % band) == 1) * (odepth > 1) * (odepth < 30))
print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
len(odepth[K]), 'DONE=1 tiles are more than 0.25 mag shallow')
K = np.flatnonzero((O.get('%s_done' % band) == 1) * (odepth > 1) *
(odepth < 30) * goodtiles)
print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
len(odepth[K]),
'interesting DONE=1 tiles are more than 0.25 mag shallow')
K = np.flatnonzero((O.get('%s_done' % band) == 1) * (odepth == 1))
print(len(K), 'DONE=1 tiles have DEPTH=1 (non-photometric)')
K = np.flatnonzero(
(O.get('%s_done' % band) == 1) * (odepth == 1) * goodtiles)
print(len(K),
'interesting DONE=1 tiles have DEPTH=1 (non-photometric)')
K = np.flatnonzero((O.get('%s_done' % band) == 1) * (odepth == 30))
print(len(K), 'DONE=1 tiles have DEPTH=30 (unknown depth)')
K = np.flatnonzero(
(O.get('%s_done' % band) == 1) * (odepth == 30) * goodtiles)
print(len(K), 'interesting DONE=1 tiles have DEPTH=30 (unknown depth)')
K = np.flatnonzero((O.get('%s_done' % band) == 1) * (odepth == 0))
print(len(K), 'DONE=1 tiles have DEPTH=0')
K = np.flatnonzero(
(O.get('%s_done' % band) == 1) * (odepth == 0) * goodtiles)
print(len(K), 'interesting DONE=1 tiles have DEPTH=0')
K = np.flatnonzero((O.get('%s_done' % band) == 0) * (odepth != 0))
print(len(K), 'tiles have DONE=0 but DEPTH != 0')
O.writeto(out_fn)
def main(survey=None, opt=None, args=None):
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if args is None:
args = sys.argv[1:]
print('forced_photom.py', ' '.join(args))
if opt is None:
parser = get_parser()
opt = parser.parse_args(args)
import logging
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
t0 = Time()
if survey is None:
survey = LegacySurveyData(survey_dir=opt.survey_dir,
cache_dir=opt.cache_dir,
output_dir=opt.out_dir)
if opt.skip:
if opt.out is not None:
outfn = opt.out
else:
outfn = survey.find_file('forced',
output=True,
camera=opt.camera,
expnum=opt.expnum)
if os.path.exists(outfn):
print('Ouput file exists:', outfn)
return 0
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
return -1
if opt.out is None and opt.out_dir is None:
print('Must supply either --out or --out-dir')
return -1
if opt.expnum is None and opt.out is None:
print('If no --expnum is given, must supply --out filename')
return -1
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Cache CCDs files before the find_ccds call...
# Copy required files into the cache?
if opt.pre_cache:
def copy_files_to_cache(fns):
for fn in fns:
cachefn = fn.replace(survey.survey_dir, survey.cache_dir)
if not cachefn.startswith(survey.cache_dir):
print('Skipping', fn)
continue
outdir = os.path.dirname(cachefn)
trymakedirs(outdir)
print('Copy', fn)
print(' to', cachefn)
shutil.copyfile(fn, cachefn)
assert (survey.cache_dir is not None)
fnset = set()
fn = survey.find_file('bricks')
fnset.add(fn)
fns = survey.find_file('ccd-kds')
fnset.update(fns)
copy_files_to_cache(fnset)
# Read metadata from survey-ccds.fits table
ccds = survey.find_ccds(camera=opt.camera,
expnum=opt.expnum,
ccdname=opt.ccdname)
print(len(ccds), 'with camera', opt.camera, 'and expnum', opt.expnum,
'and ccdname', opt.ccdname)
# sort CCDs
ccds.cut(np.lexsort((ccds.ccdname, ccds.expnum, ccds.camera)))
# If there is only one catalog survey_dir, we pass it to get_catalog_in_wcs
# as the northern survey.
catsurvey_north = survey
catsurvey_south = None
if opt.catalog_dir_north is not None:
assert (opt.catalog_dir_south is not None)
assert (opt.catalog_resolve_dec_ngc is not None)
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
elif opt.catalog_dir is not None:
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)
# Copy required CCD & calib files into the cache?
if opt.pre_cache:
assert (survey.cache_dir is not None)
fnset = set()
for ccd in ccds:
im = survey.get_image_object(ccd)
for key in im.get_cacheable_filename_variables():
fn = getattr(im, key)
if fn is None or not (os.path.exists(fn)):
continue
fnset.add(fn)
copy_files_to_cache(fnset)
args = []
for ccd in ccds:
args.append((survey, catsurvey_north, catsurvey_south,
opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))
if opt.threads:
from astrometry.util.multiproc import multiproc
from astrometry.util.timingpool import TimingPool, TimingPoolMeas
pool = TimingPool(opt.threads)
poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(poolmeas)
mp = multiproc(None, pool=pool)
tm = Time()
FF = mp.map(bounce_one_ccd, args)
print('Multi-processing forced-phot:', Time() - tm)
del mp
Time.measurements.remove(poolmeas)
del poolmeas
pool.close()
pool.join()
del pool
else:
FF = map(bounce_one_ccd, args)
FF = [F for F in FF if F is not None]
if len(FF) == 0:
print('No photometry results to write.')
return 0
# Keep only the first header
_, version_hdr, _, _ = FF[0]
# unpack results
outlier_masks = [m for _, _, m, _ in FF]
outlier_hdrs = [h for _, _, _, h in FF]
FF = [F for F, _, _, _ in FF]
F = merge_tables(FF)
if len(ccds):
version_hdr.delete('CPHDU')
version_hdr.delete('CCDNAME')
from legacypipe.utils import add_bits
from legacypipe.bits import DQ_BITS
add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
for i, ap in enumerate(apertures_arcsec):
version_hdr.add_record(
dict(name='APRAD%i' % i,
value=ap,
comment='(optical) Aperture radius, in arcsec'))
unitmap = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2',
'apflux': 'nanomaggy',
'apflux_ivar': '1/nanomaggy^2',
'psfdepth': '1/nanomaggy^2',
'galdepth': '1/nanomaggy^2',
'sky': 'nanomaggy/arcsec^2',
'psfsize': 'arcsec',
'fwhm': 'pixels',
'ccdrarms': 'arcsec',
'ccddecrms': 'arcsec',
'ra': 'deg',
'dec': 'deg',
'skyrms': 'counts/sec',
'dra': 'arcsec',
'ddec': 'arcsec',
'dra_ivar': '1/arcsec^2',
'ddec_ivar': '1/arcsec^2'
}
columns = F.get_columns()
order = [
'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'fwhm', 'ccd_cuts',
'airmass', 'sky', 'skyrms', 'psfdepth', 'galdepth', 'ccdzpt',
'ccdrarms', 'ccddecrms', 'ccdphrms', 'ra', 'dec', 'flux', 'flux_ivar',
'fracflux', 'rchisq', 'fracmasked', 'fracin', 'apflux', 'apflux_ivar',
'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar'
]
columns = [c for c in order if c in columns]
units = [unitmap.get(c, '') for c in columns]
if opt.out is not None:
outdir = os.path.dirname(opt.out)
if len(outdir):
trymakedirs(outdir)
tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.out))
fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
F.writeto(tmpfn, units=units, append=True, columns=columns)
os.rename(tmpfn, opt.out)
print('Wrote', opt.out)
else:
with survey.write_output('forced',
camera=opt.camera,
expnum=opt.expnum) as out:
F.writeto(None,
fits_object=out.fits,
primheader=version_hdr,
units=units,
columns=columns)
print('Wrote', out.real_fn)
if opt.outlier_mask is not None:
# Add outlier bit meanings to the primary header
version_hdr.add_record(
dict(name='COMMENT', value='Outlier mask bit meanings'))
version_hdr.add_record(
dict(name='OUTL_POS',
value=1,
comment='Outlier mask bit for Positive outlier'))
version_hdr.add_record(
dict(name='OUTL_NEG',
value=2,
comment='Outlier mask bit for Negative outlier'))
if opt.outlier_mask == 'default':
outdir = os.path.join(opt.out_dir, 'outlier-masks')
camexp = set(zip(ccds.camera, ccds.expnum))
for c, e in camexp:
I = np.flatnonzero((ccds.camera == c) * (ccds.expnum == e))
ccd = ccds[I[0]]
imfn = ccd.image_filename.strip()
outfn = os.path.join(outdir, imfn.replace('.fits',
'-outlier.fits'))
trymakedirs(outfn, dir=True)
tempfn = outfn.replace('.fits', '-tmp.fits')
with fitsio.FITS(tempfn, 'rw', clobber=True) as fits:
fits.write(None, header=version_hdr)
for i in I:
mask = outlier_masks[i]
_, _, _, meth, tile = survey.get_compression_args(
'outliers_mask', shape=mask.shape)
fits.write(mask,
header=outlier_hdrs[i],
extname=ccds.ccdname[i],
compress=meth,
tile_dims=tile)
os.rename(tempfn, outfn)
print('Wrote', outfn)
elif opt.outlier_mask is not None:
with fitsio.FITS(opt.outlier_mask, 'rw', clobber=True) as F:
F.write(None, header=version_hdr)
for i, (hdr, mask) in enumerate(zip(outlier_hdrs, outlier_masks)):
_, _, _, meth, tile = survey.get_compression_args(
'outliers_mask', shape=mask.shape)
F.write(mask,
header=hdr,
extname=ccds.ccdname[i],
compress=meth,
tile_dims=tile)
print('Wrote', opt.outlier_mask)
tnow = Time()
print('Total:', tnow - t0)
return 0
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--plots', action='store_true')
parser.add_argument('--brick', help='Brick name to run')
parser.add_argument(
'--input-dir',
default='/global/projecta/projectdirs/cosmo/work/legacysurvey/dr7')
#/global/cscratch1/sd/desiproc/dr7out')
parser.add_argument('--survey-dir',
default='/global/cscratch1/sd/dstn/dr7-depthcut')
parser.add_argument('--output-dir',
default='/global/cscratch1/sd/dstn/bright')
opt = parser.parse_args()
plots = opt.plots
ps = PlotSequence('bright')
brickname = opt.brick
insurvey = LegacySurveyData(opt.input_dir, cache_dir=opt.survey_dir)
outsurvey = LegacySurveyData(opt.output_dir, output_dir=opt.output_dir)
bfn = insurvey.find_file('blobmap', brick=brickname)
print('Found blob map', bfn)
blobs = fitsio.read(bfn)
h, w = blobs.shape
brick = insurvey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brick)
radius = np.sqrt(2.) * 0.25 * 1.01
neighbors = insurvey.get_bricks_near(brick.ra, brick.dec, radius)
print(len(neighbors), 'bricks nearby')
def showbool(X):
d = downsample_max(X, 8)
h, w = X.shape
plt.imshow(d,
interpolation='nearest',
origin='lower',
vmin=0,
vmax=1,
extent=[0, w, 0, h],
cmap='gray')
brightblobs = set()
for nb in neighbors:
if nb.brickname == brickname:
# ignore myself!
continue
print('Neighbor:', nb.brickname)
mfn = insurvey.find_file('maskbits', brick=nb.brickname)
if not os.path.exists(mfn):
print('No maskbits file:', mfn)
continue
maskbits = fitsio.read(mfn)
bright = ((maskbits & MASKBITS['BRIGHT']) > 0)
print(np.sum(bright > 0), 'BRIGHT pixels set')
primary = (maskbits & MASKBITS['NPRIMARY'] == 0)
print(np.sum(primary), 'PRIMARY pixels set')
edge = binary_dilation(primary, structure=np.ones((3, 3), bool))
edge = edge * np.logical_not(primary)
brightedge = edge & bright
if plots:
plt.clf()
showbool(bright)
plt.title('bright: brick %s' % nb.brickname)
ps.savefig()
# plt.clf()
# showbool(primary)
# plt.title('PRIMARY, brick %s' % nb.brickname)
# ps.savefig()
#
# plt.clf()
# showbool(edge)
# plt.title('boundary, brick %s' % nb.brickname)
# ps.savefig()
plt.clf()
showbool(brightedge)
plt.title('bright at edge, brick %s' % nb.brickname)
ps.savefig()
nwcs = wcs_for_brick(nb)
yy, xx = np.nonzero(brightedge)
print(len(yy), 'bright edge pixels')
if len(yy) == 0:
continue
rr, dd = nwcs.pixelxy2radec(xx + 1, yy + 1)
print('RA range', rr.min(), rr.max(), 'vs brick', brick.ra1, brick.ra2)
print('Dec range', dd.min(), dd.max(), 'vs brick', brick.dec1,
brick.dec2)
# Find pixels that are within this brick's unique area
I, = np.nonzero((rr >= brick.ra1) * (rr <= brick.ra2) *
(dd >= brick.dec1) * (dd <= brick.dec2))
if plots:
plt.clf()
plt.plot(
[brick.ra1, brick.ra1, brick.ra2, brick.ra2, brick.ra1],
[brick.dec1, brick.dec2, brick.dec2, brick.dec1, brick.dec1],
'b-')
plt.plot(rr, dd, 'k.')
plt.plot(rr[I], dd[I], 'r.')
plt.title('Bright pixels from %s' % nb.brickname)
ps.savefig()
if len(I) == 0:
print('No edge pixels touch')
#plt.plot(br,bd, 'b-')
continue
#print('Edge pixels touch!')
#plt.plot(br,bd, 'r-', zorder=20)
ok, x, y = brickwcs.radec2pixelxy(rr[I], dd[I])
x = np.round(x).astype(int) - 1
y = np.round(y).astype(int) - 1
print('Pixel ranges X', x.min(), x.max(), 'Y', y.min(), y.max())
assert (np.all((x >= 0) * (x < w) * (y >= 0) * (y < h)))
print('Adding blobs:', np.unique(blobs[y, x]))
brightblobs.update(blobs[y, x])
print('Blobs touching bright pixels:', brightblobs)
print()
brightblobs.discard(-1)
if len(brightblobs) == 0:
print('No neighboring bright blobs to update!')
return
print('Updating', len(brightblobs), 'blobs:', brightblobs)
tmap = np.zeros(blobs.max() + 2, bool)
for b in brightblobs:
tmap[b + 1] = True
touching = tmap[blobs + 1]
if plots:
plt.clf()
showbool(touching)
plt.title('Blobs touching bright, brick %s' % brickname)
ps.savefig()
mfn = insurvey.find_file('maskbits', brick=brickname)
maskbits, hdr = fitsio.read(mfn, header=True)
updated = maskbits | (MASKBITS['BRIGHT'] * touching)
if np.all(maskbits == updated):
print('No bits updated! (Bright stars were already masked)')
return
maskbits = updated
if plots:
plt.clf()
showbool((maskbits & MASKBITS['BRIGHT']) > 0)
plt.title('New maskbits map for BRIGHT, brick %s' % brickname)
ps.savefig()
with outsurvey.write_output('maskbits', brick=brickname) as out:
out.fits.write(maskbits, hdr=hdr)
tfn = insurvey.find_file('tractor', brick=brickname)
phdr = fitsio.read_header(tfn, ext=0)
hdr = fitsio.read_header(tfn, ext=1)
T = fits_table(tfn)
print('Read', len(T), 'sources')
print('Bright:', Counter(T.brightstarinblob))
iby = np.clip(np.round(T.by), 0, h - 1).astype(int)
ibx = np.clip(np.round(T.bx), 0, w - 1).astype(int)
if plots:
before = np.flatnonzero(T.brightstarinblob)
T.brightstarinblob |= touching[iby, ibx]
print('Bright:', Counter(T.brightstarinblob))
# yuck -- copy the TUNIT headers from input to output.
units = [
hdr.get('TUNIT%i' % (i + 1), '') for i in range(len(T.get_columns()))
]
if plots:
plt.clf()
showbool((maskbits & MASKBITS['BRIGHT']) > 0)
ax = plt.axis()
after = np.flatnonzero(T.brightstarinblob)
plt.plot(T.bx[before], T.by[before], 'gx')
plt.plot(T.bx[after], T.by[after], 'r.')
plt.axis(ax)
plt.title('sources with brightstarinblob, brick %s' % brickname)
ps.savefig()
with outsurvey.write_output('tractor', brick=brickname) as out:
T.writeto(None,
fits_object=out.fits,
primheader=phdr,
header=hdr,
units=units)
def main():
# indir = '/global/cscratch1/sd/dstn/dr8test-1'
# name = 'dr8-test1'
# pretty = 'DR8 test1'
# indir = '/scratch1/scratchdirs/desiproc/dr8test002/'
# name = 'dr8-test2'
# pretty = 'DR8 test2 (outliers)'
# indir = '/scratch1/scratchdirs/desiproc/dr8test003/'
# name = 'dr8-test3'
# pretty = 'DR8 test3 (outliers)'
#
# indir = '/scratch1/scratchdirs/desiproc/dr8test004/'
# name = 'dr8-test4'
# pretty = 'DR8 test4 (large-galaxies)'
# indir = '/global/cscratch1/sd/dstn/dr8test005/'
# name = 'dr8-test5'
# pretty = 'DR8 test5 (trident)'
# indir = '/global/cscratch1/sd/dstn/dr8test006/'
# name = 'dr8-test6'
# pretty = 'DR8 test6 (sky)'
# indir = '/global/cscratch1/sd/dstn/dr8test007/'
# name = 'dr8-test7'
# pretty = 'DR8 test7 (outliers)'
#indir = '/global/cscratch1/sd/dstn/dr8test14/'
#name = 'dr8-test14'
#pretty = 'DR8 test14 (rc)'
#indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8a/'
#name = 'dr8a'
#pretty = 'DR8a (rc)'
if False:
indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-decam/'
name = 'dr8b-decam'
pretty = 'DR8b DECam'
survey_dir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-decam'
if True:
indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-90prime-mosaic/'
name = 'dr8b-90p-mos'
pretty = 'DR8b BASS+MzLS'
survey_dir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-90prime-mosaic'
# ln -s /global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-decam/coadds-only/coadd/ .
sublayers = ['', '-model', '-resid']
subpretty = {'': ' images', '-model': ' models', '-resid': ' residuals'}
# survey_dir = '/global/cscratch1/sd/desiproc/dr7'
# sublayers = ['']
# subpretty = {'':' images'}
#survey_dir = '/global/cscratch1/sd/dstn/dr8-depthcut'
#survey_dir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8a/'
rsync = False
datadir = 'data'
survey = LegacySurveyData(survey_dir=survey_dir)
fn = 'map/test_layers.py'
txt = open(fn).read()
for x in sublayers:
txt = txt + '\n' + 'test_layers.append(("%s%s", "%s%s"))\n' % (
name, x, pretty, subpretty[x])
open(fn, 'wb').write(txt.encode())
print('Wrote', fn)
basedir = os.path.join(datadir, name)
if rsync:
cmd = 'rsync -LRarv %s/./{coadd/*/*/*-{image-,model-,ccds}*.fits*,tractor} %s/%s' % (
indir, datadir, name)
print(cmd)
os.system(cmd)
# ...?
cmd = 'rsync -Rarv %s/./{images,survey-ccds*.fits} %s/%s' % (
survey_dir, datadir, name)
print(cmd)
os.system(cmd)
else:
# symlink
if os.path.exists(basedir):
print('Not symlinking', indir, 'to', basedir, ': already exists!')
else:
os.makedirs(basedir)
for subdir in ['coadd', 'tractor']:
os.symlink(os.path.join(indir, subdir),
os.path.join(basedir, subdir),
target_is_directory=True)
for fn in ['images', 'calib']:
os.symlink(os.path.join(indir, subdir),
os.path.join(basedir, subdir),
target_is_directory=False)
for pat in ['survey-ccds-*']:
for fn in [
os.path.basename(f)
for f in glob(os.path.join(indir, pat))
]:
os.symlink(os.path.join(indir, subdir),
os.path.join(basedir, subdir),
target_is_directory=False)
allbricks = survey.get_bricks_readonly()
imagefns = glob(os.path.join(basedir, 'coadd', '*', '*',
'*-image-*.fits*'))
extraimagefns = glob(
os.path.join(basedir, 'extra-images', 'coadd', '*', '*',
'*-image-*.fits*'))
print('Image filenames:', len(imagefns), 'plus', len(extraimagefns),
'extras')
imagefns += extraimagefns
brickset = set()
for fn in imagefns:
dirs = fn.split('/')
brickname = dirs[-2]
brickset.add(brickname)
print(len(brickset), 'bricks found')
I, = np.nonzero([b in brickset for b in allbricks.brickname])
bricks = allbricks[I]
brickfn = os.path.join(basedir, 'survey-bricks.fits.gz')
bricks.writeto(brickfn)
print('Wrote', brickfn)
threads = 8
tharg = '--threads %i ' % threads
#tharg = ''
for x in sublayers:
cmd = 'python -u render-tiles.py --kind %s%s --bricks' % (name, x)
print(cmd)
os.system(cmd)
# images
for scale in range(1, 8):
cmd = 'python -u render-tiles.py --kind %s --scale --zoom %i %s' % (
name, scale, tharg)
print(cmd)
os.system(cmd)
# models
for scale in range(1, 8):
cmd = 'python -u render-tiles.py --kind %s-model --scale --zoom %i %s' % (
name, scale, tharg)
print(cmd)
os.system(cmd)
# resids
for scale in range(1, 8):
cmd = 'python -u render-tiles.py --kind %s-resid --scale --zoom %i %s' % (
name, scale, tharg)
print(cmd)
os.system(cmd)
for x in sublayers:
cmd = 'python -u render-tiles.py --kind %s%s --top' % (name, x)
print(cmd)
os.system(cmd)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build-sample',
action='store_true',
help='Build the sample.')
parser.add_argument('--jpg-cutouts',
action='store_true',
help='Get jpg cutouts from the viewer.')
parser.add_argument('--ccd-cutouts',
action='store_true',
help='Get CCD cutouts of each galaxy.')
parser.add_argument('--runbrick',
action='store_true',
help='Run the pipeline.')
parser.add_argument('--build-webpage',
action='store_true',
help='(Re)build the web content.')
args = parser.parse_args()
# Top-level directory
key = 'LEGACY_SURVEY_LARGE_GALAXIES'
if key not in os.environ:
print('Required ${} environment variable not set'.format(key))
return 0
largedir = os.getenv(key)
samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
# --------------------------------------------------
# Build the sample of large galaxies based on the available imaging.
if args.build_sample:
# Read the parent catalog.
cat = read_rc3()
# Create a simple WCS object for each object and find all the CCDs
# touching that WCS footprint.
survey = LegacySurveyData(version='dr2') # hack!
allccds = survey.get_ccds()
keep = np.concatenate((survey.apply_blacklist(allccds),
survey.photometric_ccds(allccds)))
allccds.cut(keep)
ccdlist = []
outcat = []
for gal in cat:
galwcs = _simplewcs(gal)
ccds1 = allccds[ccds_touching_wcs(galwcs, allccds)]
ccds1 = ccds1[_uniqccds(ccds1)]
if len(
ccds1
) > 0 and 'g' in ccds1.filter and 'r' in ccds1.filter and 'z' in ccds1.filter:
print('Found {} CCDs for {}, D(25)={:.4f}'.format(
len(ccds1), gal['GALAXY'], gal['RADIUS']))
ccdsfile = os.path.join(
largedir, 'ccds',
'{}-ccds.fits'.format(gal['GALAXY'].strip().lower()))
print(' Writing {}'.format(ccdsfile))
if os.path.isfile(ccdsfile):
os.remove(ccdsfile)
ccds1.writeto(ccdsfile)
ccdlist.append(ccds1)
if len(outcat) == 0:
outcat = gal
else:
outcat = vstack((outcat, gal))
#if gal['GALAXY'] == 'MCG5-19-36':
# pdb.set_trace()
# Write out the final catalog.
samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
if os.path.isfile(samplefile):
os.remove(samplefile)
print('Writing {}'.format(samplefile))
outcat.write(samplefile)
print(outcat)
# Do we need to transfer any of the data to nyx?
_getfiles(merge_tables(ccdlist))
# --------------------------------------------------
# Get data, model, and residual cutouts from the legacysurvey viewer. Also
# get thumbnails that are lower resolution.
if args.jpg_cutouts:
thumbsize = 100
sample = fits.getdata(samplefile, 1)
for gal in sample:
size = np.ceil(10 * gal['RADIUS'] / PIXSCALE)
thumbpixscale = PIXSCALE * size / thumbsize
#imageurl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
# '&pixscale={:.3f}&size={:g}'.format(PIXSCALE, size)
#imagejpg = os.path.join(largedir, 'cutouts', gal['GALAXY'].strip().lower()+'-image.jpg')
#if os.path.isfile(imagejpg):
# os.remove(imagejpg)
#os.system('wget --continue -O {:s} "{:s}"' .format(imagejpg, imageurl))
thumburl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
'&pixscale={:.3f}&size={:g}'.format(thumbpixscale, thumbsize)
thumbjpg = os.path.join(
largedir, 'cutouts',
gal['GALAXY'].strip().lower() + '-image-thumb.jpg')
if os.path.isfile(thumbjpg):
os.remove(thumbjpg)
os.system('wget --continue -O {:s} "{:s}"'.format(
thumbjpg, thumburl))
# --------------------------------------------------
# (Re)build the webpage.
if args.build_webpage:
# index.html
html = open(os.path.join(largedir, 'index.html'), 'w')
html.write('<html><body>\n')
html.write('<h1>Sample of Large Galaxies</h1>\n')
html.write('<table border="2" width="30%">\n')
html.write('<tbody>\n')
sample = fits.getdata(samplefile, 1)
for gal in sample:
# Add coordinates and sizes here.
galaxy = gal['GALAXY'].strip().lower()
html.write('<tr>\n')
html.write('<td><a href="html/{}.html">{}</a></td>\n'.format(
galaxy, galaxy.upper()))
html.write(
'<td><a href="http://legacysurvey.org/viewer/?ra={:.6f}&dec={:.6f}" target="_blank"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'
.format(gal['RA'], gal['DEC'], galaxy, galaxy.upper()))
# html.write('<td><a href="html/{}.html"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'.format(galaxy, galaxy, galaxy.upper()))
html.write('</tr>\n')
html.write('</tbody>\n')
html.write('</table>\n')
html.write('</body></html>\n')
html.close()
sys.exit(1)
# individual galaxy pages
for gal in sample[:3]:
galaxy = gal['GALAXY'].strip().lower()
html = open(os.path.join(largedir, 'html/{}.html'.format(galaxy)),
'w')
html.write('<html><body>\n')
html.write(
'<a href=../cutouts/{}.jpg><img src=../cutouts/{}-image.jpg alt={} /></a>\n'
.format(galaxy, galaxy, galaxy, galaxy.upper()))
html.write('</body></html>\n')
html.close()
# --------------------------------------------------
# Get cutouts of all the CCDs for each galaxy.
if args.ccd_cutouts:
sample = fits.getdata(samplefile, 1)
for gal in sample[1:2]:
galaxy = gal['GALAXY'].strip().lower()
ccdsfile = os.path.join(largedir, 'ccds',
'{}-ccds.fits'.format(galaxy))
ccds = fits.getdata(ccdsfile)
pdb.set_trace()
# --------------------------------------------------
# Run the pipeline.
if args.runbrick:
sample = fits.getdata(samplefile, 1)
for gal in sample[1:2]:
galaxy = gal['GALAXY'].strip().lower()
diam = 10 * np.ceil(gal['RADIUS'] / PIXSCALE).astype(
'int16') # [pixels]
# Note: zoom is relative to the center of an imaginary brick with
# dimensions (0, 3600, 0, 3600).
survey = LegacySurveyData(version='dr2', output_dir=largedir)
run_brick(None,
survey,
radec=(gal['RA'], gal['DEC']),
blobxy=zip([diam / 2], [diam / 2]),
threads=1,
zoom=(1800 - diam / 2, 1800 + diam / 2, 1800 - diam / 2,
1800 + diam / 2),
wise=False,
forceAll=True,
writePickles=False,
do_calibs=False,
write_metrics=False,
pixPsf=True,
splinesky=True,
early_coadds=True,
stages=['writecat'],
ceres=False)
pdb.set_trace()
def main(survey=None, opt=None):
print(' '.join(sys.argv))
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = tlast = Time()
if opt.skip and os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing first arg as exposure number (otherwise, it's a filename)
try:
expnum = int(opt.expnum)
filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
filename = opt.expnum
# Try parsing HDU: "all" or HDU name or HDU number.
all_hdus = (opt.ccdname == 'all')
hdu = -1
ccdname = None
if not all_hdus:
try:
hdu = int(opt.ccdname)
except:
ccdname = opt.ccdname
if survey is None:
survey = LegacySurveyData(survey_dir=opt.survey_dir)
catsurvey_north = survey
catsurvey_south = None
if opt.catalog_dir_north is not None:
assert (opt.catalog_dir_south is not None)
assert (opt.catalog_resolve_dec_ngc is not None)
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
if opt.catalog_dir is not None:
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)
if filename is not None and hdu >= 0:
# FIXME -- try looking up in CCDs file?
# Read metadata from file
print('Warning: faking metadata from file contents')
T = exposure_metadata([filename], hdus=[hdu])
print('Metadata:')
T.about()
if not 'ccdzpt' in T.columns():
phdr = fitsio.read_header(filename)
T.ccdzpt = np.array([phdr['MAGZERO']])
print('WARNING: using header MAGZERO')
T.ccdraoff = np.array([0.])
T.ccddecoff = np.array([0.])
print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and ccdname', ccdname)
if hdu >= 0:
T.cut(T.image_hdu == hdu)
print(len(T), 'with HDU', hdu)
if filename is not None:
T.cut(np.array([f.strip() == filename for f in T.image_filename]))
print(len(T), 'with filename', filename)
if opt.camera is not None:
T.cut(T.camera == opt.camera)
print(len(T), 'with camera', opt.camera)
if not all_hdus:
assert (len(T) == 1)
args = []
for ccd in T:
args.append((survey, catsurvey_north, catsurvey_south,
opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))
if opt.threads:
from astrometry.util.multiproc import multiproc
from astrometry.util.timingpool import TimingPool, TimingPoolMeas
pool = TimingPool(opt.threads)
poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(poolmeas)
mp = multiproc(None, pool=pool)
tm = Time()
FF = mp.map(bounce_one_ccd, args)
print('Multi-processing forced-phot:', Time() - tm)
else:
FF = map(bounce_one_ccd, args)
FF = [F for F in FF if F is not None]
if len(FF) == 0:
print('No photometry results to write.')
return 0
# Keep only the first header
_, version_hdr = FF[0]
FF = [F for F, hdr in FF]
F = merge_tables(FF)
if all_hdus:
version_hdr.delete('CPHDU')
version_hdr.delete('CCDNAME')
units = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2',
'apflux': 'nanomaggy',
'apflux_ivar': '1/nanomaggy^2',
'psfdepth': '1/nanomaggy^2',
'galdepth': '1/nanomaggy^2',
'sky': 'nanomaggy/arcsec^2',
'psfsize': 'arcsec'
}
if opt.derivs:
units.update({
'dra': 'arcsec',
'ddec': 'arcsec',
'dra_ivar': '1/arcsec^2',
'ddec_ivar': '1/arcsec^2'
})
columns = F.get_columns()
order = [
'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'ccd_cuts',
'airmass', 'sky', 'psfdepth', 'galdepth', 'ra', 'dec', 'flux',
'flux_ivar', 'fracflux', 'rchisq', 'fracmasked', 'apflux',
'apflux_ivar', 'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar',
'ddec_ivar'
]
columns = [c for c in order if c in columns]
# Set units headers (must happen after column ordering is set!)
hdr = fitsio.FITSHDR()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.outfn))
fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
F.writeto(tmpfn, header=hdr, append=True, columns=columns)
os.rename(tmpfn, opt.outfn)
print('Wrote', opt.outfn)
tnow = Time()
print('Total:', tnow - t0)
return 0
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(
description=
"This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
)
parser.add_argument('--calibs',
action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper',
type=int,
default=None,
help='Batch N calibs per line')
parser.add_argument(
'--byexp',
action='store_true',
default=False,
help='Run one whole exposure per job (not one CCD per job)')
parser.add_argument('--forced',
action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb',
action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--stage', help='Stage image files to given directory')
parser.add_argument('--touching',
action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--near',
action='store_true',
help='Quick cut to only CCDs near selected bricks')
parser.add_argument('--check-coadd',
action='store_true',
help='Check which coadds actually need to run.')
parser.add_argument('--out',
help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command',
action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--opt', help='With --command, extra options to add')
parser.add_argument('--maxra', type=float, help='Maximum RA to run')
parser.add_argument('--minra', type=float, help='Minimum RA to run')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--ignore_cuts',
action='store_true',
default=False,
help='no photometric cuts')
parser.add_argument('--save_to_fits',
action='store_true',
default=False,
help='save cut brick,ccd to fits table')
parser.add_argument(
'--name',
action='store',
default='dr3',
help='save with this suffix, e.g. refers to ccds table')
parser.add_argument('--delete-sky',
action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
parser.add_argument('--nccds',
action='store_true',
default=False,
help='Prints number of CCDs per brick')
parser.add_argument('--bands',
default='g,r,z',
help='Set bands to keep: comma-separated list.')
opt = parser.parse_args()
want_ccds = (opt.calibs or opt.forced or opt.lsb)
want_bricks = not want_ccds
survey = LegacySurveyData()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = survey.get_bricks()
log('Bricks Dec range:', B.dec.min(), B.dec.max())
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = survey.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
if opt.ignore_cuts == False:
log('Applying CCD cuts...')
if 'ccd_cuts' in T.columns():
T.cut(T.ccd_cuts == 0)
log(len(T), 'CCDs survive cuts')
bands = opt.bands.split(',')
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
log('CCDs Dec range:', T.dec.min(), T.dec.max())
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo, dhi = -25, 40
rlo, rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(
test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo, dhi = -90, 90
rlo, rhi = 0, 360
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo, rhi = 240, 245
dlo, dhi = 5, 12
elif opt.region == 'dr8-decam':
rlo, rhi = 0, 360
dlo, dhi = -70, 40
log('DR8-DECam region')
elif opt.region == 'edrplus':
rlo, rhi = 235, 248
dlo, dhi = 5, 15
elif opt.region == 'edr-south':
rlo, rhi = 240, 245
dlo, dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo, rhi = 149.75, 150.75
dlo, dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo, rhi = 240, 250
dlo, dhi = 10, 15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo, rhi = 320., 360.
dlo, dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo, rhi = 240, 360
dlo, dhi = 20, 40
elif opt.region == 'north':
rlo, rhi = 120, 240
dlo, dhi = 20, 40
elif opt.region == 'northeast':
rlo, rhi = 0, 120
dlo, dhi = 20, 40
elif opt.region == 'southwest':
rlo, rhi = 240, 360
dlo, dhi = -20, 0
elif opt.region == 'south':
rlo, rhi = 120, 240
dlo, dhi = -20, 0
elif opt.region == 'southeast':
rlo, rhi = 0, 120
dlo, dhi = -20, 0
elif opt.region == 'southsoutheast':
rlo, rhi = 0, 120
dlo, dhi = -20, -10
elif opt.region == 'midwest':
rlo, rhi = 240, 360
dlo, dhi = 0, 20
elif opt.region == 'middle':
rlo, rhi = 120, 240
dlo, dhi = 0, 20
elif opt.region == 'mideast':
rlo, rhi = 0, 120
dlo, dhi = 0, 20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks survey-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo, rhi = 250, 260
dlo, dhi = 30, 35
elif opt.region == 'deep2f2':
rlo, rhi = 251.4, 254.4
dlo, dhi = 34.6, 35.3
elif opt.region == 'deep2f3':
rlo, rhi = 351.25, 353.75
dlo, dhi = 0, 0.5
elif opt.region == 'deep3':
rlo, rhi = 214, 216
dlo, dhi = 52.25, 53.25
elif opt.region == 'virgo':
rlo, rhi = 185, 190
dlo, dhi = 10, 15
elif opt.region == 'virgo2':
rlo, rhi = 182, 192
dlo, dhi = 8, 18
elif opt.region == 'coma':
# van Dokkum et al Coma cluster ultra-diffuse galaxies: 3x3 field centered on Coma cluster
rc, dc = 195., 28.
dd = 1.5
cosdec = np.cos(np.deg2rad(dc))
rlo, rhi = rc - dd / cosdec, rc + dd / cosdec
dlo, dhi = dc - dd, dc + dd
elif opt.region == 'lsb':
rlo, rhi = 147.2, 147.8
dlo, dhi = -0.4, 0.4
elif opt.region == 'eboss-sgc':
# generous boundaries to make sure get all relevant images
# RA -45 to +45
# Dec -5 to +7
rlo, rhi = 310., 50.
dlo, dhi = -6., 6.
elif opt.region == 'eboss-ngc':
# generous boundaries to make sure get all relevant images
# NGC ELGs
# RA 115 to 175
# Dec 15 to 30
# rlo,rhi = 122., 177.
# dlo,dhi = 12., 32.
rlo, rhi = 126., 168.
dlo, dhi = 18., 33.
elif opt.region == 'mzls':
dlo, dhi = -10., 90. # -10: pull in Stripe 82 data too
elif opt.region == 'dr4-bootes':
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
#dlo,dhi = 34., 35.
#rlo,rhi = 209.5, 210.5
dlo, dhi = 33., 36.
rlo, rhi = 216.5, 219.5
elif opt.region == 'des-sn-x3':
#rlo,rhi = 36., 37.
#dlo,dhi = -5., -4.
rlo, rhi = 36., 36.5
dlo, dhi = -4.5, -4.
elif opt.region == 'ngc2632':
# open cluster
rlo, rhi = 129.0, 131.0
dlo, dhi = 19.0, 20.5
elif opt.region == 'dr8sky':
rlo, rhi = 35.0, 37.0
dlo, dhi = -3.0, -1.0
# ADM DR8 test regions, see, e.g.:
# https://desi.lbl.gov/trac/wiki/DecamLegacy/DR8#Testregions
elif opt.region == 'dr8-test-s82':
rlo, rhi = 0, 45
dlo, dhi = -1.25, 1.25
elif opt.region == 'dr8-test-hsc-sgc':
rlo, rhi = 30, 40
dlo, dhi = -6.5, -1.25
elif opt.region == 'dr8-test-hsc-ngc':
rlo, rhi = 177.5, 182.5
dlo, dhi = -1, 1
elif opt.region == 'dr8-test-edr':
rlo, rhi = 240, 245
dlo, dhi = 5, 12
elif opt.region == 'dr8-test-hsc-north':
rlo, rhi = 240, 250
dlo, dhi = 42, 45
elif opt.region == 'dr8-test-deep2-egs':
rlo, rhi = 213, 216.5
dlo, dhi = 52, 54
elif opt.region == 'dr8-test-overlap':
rlo, rhi = 132, 140.5
dlo, dhi = 31.5, 35
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if opt.minra is not None:
rlo = opt.minra
if opt.maxra is not None:
rhi = opt.maxra
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(
np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
(B.dec <= dhi))
log(len(B), 'bricks in range; cut Dec range', B.dec.min(), B.dec.max())
#for name in B.get('brickname'):
# print(name)
#B.writeto('bricks-cut.fits')
bricksize = 0.25
# A bit more than 0.25-degree brick radius + Bok image radius ~ 0.57
search_radius = 1.05 * np.sqrt(2.) * (bricksize +
(0.455 * 4096 / 3600.)) / 2.
log(len(T), 'CCDs')
log(len(B), 'Bricks')
I, J, d = match_radec(B.ra,
B.dec,
T.ra,
T.dec,
search_radius,
nearest=True)
B.cut(I)
log('Cut to', len(B), 'bricks near CCDs')
log('Bricks Dec range:', B.dec.min(), B.dec.max())
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# plt.title('DR3 bricks')
# plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
# plt.savefig('bricks.png')
if opt.touching:
I, J, d = match_radec(T.ra,
T.dec,
B.ra,
B.dec,
search_radius,
nearest=True)
# list the ones that will be cut
# drop = np.ones(len(T))
# drop[I] = False
# for i in np.flatnonzero(drop):
# from astrometry.util.starutil_numpy import degrees_between
# dists = degrees_between(B.ra, B.dec, T.ra[i], T.dec[i])
# mindist = min(dists)
# print('Dropping:', T.ra[i], T.dec[i], 'min dist', mindist, 'search_radius', search_radius)
T.cut(I)
log('Cut to', len(T), 'CCDs near bricks')
# sort by RA increasing
B.cut(np.argsort(B.ra))
if opt.save_to_fits:
assert (opt.touching)
# Write cut tables to file
for tab, typ in zip([B, T], ['bricks', 'ccds']):
fn = '%s-%s-cut.fits' % (typ, opt.region)
if os.path.exists(fn):
os.remove(fn)
tab.writeto(fn)
log('Wrote %s' % fn)
# Write text files listing ccd and filename names
# nm1,nm2= 'ccds-%s.txt'% opt.region,'filenames-%s.txt' % opt.region
# if os.path.exists(nm1):
# os.remove(nm1)
# if os.path.exists(nm2):
# os.remove(nm2)
# f1,f2=open(nm1,'w'),open(nm2,'w')
# fns= list(set(T.get('image_filename')))
# for fn in fns:
# f2.write('%s\n' % fn.strip())
# for ti in T:
# f1.write('%s\n' % ti.get('image_filename').strip())
# f1.close()
# f2.close()
# log('Wrote *-names.txt')
if opt.touching:
if want_bricks:
# Shortcut the list of bricks that are definitely touching CCDs --
# a brick-ccd pair within this radius must be touching.
closest_radius = 0.95 * (bricksize + 0.262 * 2048 / 3600.) / 2.
J1, nil, nil = match_radec(B.ra,
B.dec,
T.ra,
T.dec,
closest_radius,
nearest=True)
log(len(J1), 'of', len(B), 'bricks definitely touch CCDs')
tocheck = np.ones(len(B), bool)
tocheck[J1] = False
J2 = []
for j in np.flatnonzero(tocheck):
b = B[j]
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickname)
if len(I) == 0:
continue
J2.append(j)
J = np.hstack((J1, J2))
J = np.sort(J).astype(int)
B.cut(J)
log('Cut to', len(B), 'bricks touching CCDs')
else:
J = []
allI = set()
for j, b in enumerate(B):
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickname)
if len(I) == 0:
continue
allI.update(I)
J.append(j)
allI = list(allI)
allI.sort()
B.cut(np.array(J))
log('Cut to', len(B), 'bricks touching CCDs')
elif opt.near:
# Find CCDs near bricks
allI, nil, nil = match_radec(T.ra,
T.dec,
B.ra,
B.dec,
search_radius,
nearest=True)
# Find bricks near CCDs
J, nil, nil = match_radec(B.ra,
B.dec,
T.ra,
T.dec,
search_radius,
nearest=True)
B.cut(J)
log('Cut to', len(B), 'bricks near CCDs')
else:
allI = np.arange(len(T))
if opt.byexp:
nil, eI = np.unique(T.expnum[allI], return_index=True)
allI = allI[eI]
print('Cut to', len(allI), 'expnums')
if opt.nccds:
from queue import Queue
from threading import Thread
log('Checking number of CCDs per brick')
def worker():
while True:
i = q.get()
if i is None:
break
b = B[i]
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(b.brickname, len(I))
q.task_done()
q = Queue()
num_threads = 24
threads = []
for i in range(num_threads):
t = Thread(target=worker)
t.start()
threads.append(t)
for i in range(len(B)):
q.put(i)
q.join()
for i in range(num_threads):
q.put(None)
for t in threads:
t.join()
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
if want_bricks:
# Print the list of bricks and exit.
for b in B:
print(b.brickname)
if opt.save_to_fits:
B.writeto('bricks-%s-touching.fits' % opt.region)
if not want_ccds:
sys.exit(0)
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.stage is not None:
cmd_pat = 'rsync -LRarv %s %s'
fns = set()
for iccd in allI:
im = survey.get_image_object(T[iccd])
fns.update([
im.imgfn, im.wtfn, im.dqfn, im.psffn, im.merged_psffn,
im.merged_splineskyfn, im.splineskyfn
])
for i, fn in enumerate(fns):
print('File', i + 1, 'of', len(fns), ':', fn)
if not os.path.exists(fn):
print('No such file:', fn)
continue
base = survey.get_survey_dir()
if base.endswith('/'):
base = base[:-1]
rel = os.path.relpath(fn, base)
dest = os.path.join(opt.stage, rel)
print('Dest:', dest)
if os.path.exists(dest):
print('Exists:', dest)
continue
cmd = cmd_pat % ('%s/./%s' % (base, rel), opt.stage)
print(cmd)
rtn = os.system(cmd)
assert (rtn == 0)
sys.exit(0)
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
expstr = '%08i' % T.expnum[i]
imgfn = os.path.join(survey.survey_dir, 'images',
T.image_filename[i].strip())
if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
and os.path.exists(imgfn[:-3])):
imgfn = imgfn[:-3]
outfn = os.path.join(
expstr[:5], expstr, 'forced-%s-%s-%s.fits' %
(T.camera[i].strip(), expstr, T.ccdname[i]))
f.write(
'python legacypipe/forced_photom.py --apphot --derivs --catalog-dir /project/projectdirs/cosmo/data/legacysurvey/dr7/ %i %s forced/%s\n'
% (T.expnum[i], T.ccdname[i], outfn))
f.close()
log('Wrote', opt.out)
fn = 'forced-ccds.fits'
T[allI].writeto(fn)
print('Wrote', fn)
sys.exit(0)
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write(
'python legacyanalysis/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
% (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
sys.exit(0)
log('Writing calibs to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if cmd is None:
cmd = ''
f.write(cmd + ' '.join(batch) + '\n')
cmd = None
if opt.command:
cmd = 'python legacypipe/run-calib.py '
if opt.opt is not None:
cmd += opt.opt + ' '
for j, i in enumerate(allI):
if opt.delete_sky:
log(j + 1, 'of', len(allI))
im = survey.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.command:
if opt.byexp:
s = '--expnum %i' % (T.expnum[i])
else:
s = '%i-%s' % (T.expnum[i], T.ccdname[i])
prefix = 'python legacypipe/run-calib.py '
if opt.opt is not None:
prefix = prefix + opt.opt
#('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
# (T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
prefix = ''
if j < 10:
print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname',
T.ccdname[i], 'filename', T.image_filename[i])
if not opt.nper:
f.write(prefix + s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, cmd)
batch = []
if len(batch):
write_batch(f, batch, cmd)
f.close()
log('Wrote', opt.out)
return 0
def run_one_brick(X):
brick, ibrick, nbricks, plots, kwargs = X
survey = LegacySurveyData()
print()
print()
print('Brick', (ibrick + 1), 'of', nbricks, ':', brick.brickname)
dirnm = os.path.join('depthcuts', brick.brickname[:3])
outfn = os.path.join(dirnm, 'ccds-%s.fits' % brick.brickname)
if os.path.exists(outfn):
print('Exists:', outfn)
return 0
H, W = 3600, 3600
pixscale = 0.262
bands = ['g', 'r', 'z']
# Get WCS object describing brick
targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
targetrd = np.array([
targetwcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
gitver = get_git_version()
ccds = survey.ccds_touching_wcs(targetwcs)
if ccds is None:
print('No CCDs actually touching brick')
return 0
print(len(ccds), 'CCDs actually touching brick')
ccds.cut(np.in1d(ccds.filter, bands))
print('Cut on filter:', len(ccds), 'CCDs remain.')
if 'ccd_cuts' in ccds.get_columns():
norig = len(ccds)
ccds.cut(ccds.ccd_cuts == 0)
print(len(ccds), 'of', norig, 'CCDs pass cuts')
else:
print('No CCD cuts')
if len(ccds) == 0:
print('No CCDs left')
return 0
ps = None
if plots:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('depth-%s' % brick.brickname)
splinesky = True
gaussPsf = False
pixPsf = True
do_calibs = False
normalizePsf = True
get_depth_maps = kwargs.pop('get_depth_maps', False)
try:
D = make_depth_cut(survey,
ccds,
bands,
targetrd,
brick,
W,
H,
pixscale,
plots,
ps,
splinesky,
gaussPsf,
pixPsf,
normalizePsf,
do_calibs,
gitver,
targetwcs,
get_depth_maps=get_depth_maps,
**kwargs)
if get_depth_maps:
keep, overlapping, depthmaps = D
else:
keep, overlapping = D
except:
print('Failed to make_depth_cut():')
import traceback
traceback.print_exc()
return -1
print(np.sum(overlapping), 'CCDs overlap the brick')
print(np.sum(keep), 'CCDs passed depth cut')
ccds.overlapping = overlapping
ccds.passed_depth_cut = keep
if not os.path.exists(dirnm):
try:
os.makedirs(dirnm)
except:
pass
if get_depth_maps:
for band, depthmap in depthmaps:
doutfn = os.path.join(dirnm,
'depth-%s-%s.fits' % (brick.brickname, band))
hdr = fitsio.FITSHDR()
# Plug the WCS header cards into these images
targetwcs.add_to_header(hdr)
hdr.delete('IMAGEW')
hdr.delete('IMAGEH')
hdr.add_record(dict(name='EQUINOX', value=2000.))
hdr.add_record(dict(name='FILTER', value=band))
fitsio.write(doutfn, depthmap, header=hdr)
print('Wrote', doutfn)
tmpfn = os.path.join(os.path.dirname(outfn),
'tmp-' + os.path.basename(outfn))
ccds.writeto(tmpfn)
os.rename(tmpfn, outfn)
print('Wrote', outfn)
return 0
'''
After running forced_photom.py on a set of CCDs, this script merges
the results back into a catalog.
'''
from astrometry.util.fits import *
import numpy as np
from glob import glob
from collections import Counter
from legacypipe.survey import LegacySurveyData
fns = glob('forced/*/*/forced-*.fits')
F = merge_tables([fits_table(fn) for fn in fns])
dr6 = LegacySurveyData('/project/projectdirs/cosmo/data/legacysurvey/dr6')
B = dr6.get_bricks_readonly()
I = np.flatnonzero((B.ra1 < F.ra.max()) * (B.ra2 > F.ra.min()) * (B.dec1 < F.dec.max()) * (B.dec2 > F.dec.min()))
print(len(I), 'bricks')
T = merge_tables([fits_table(dr6.find_file('tractor', brick=B.brickname[i])) for i in I])
print(len(T), 'sources')
T.cut(T.brick_primary)
print(len(T), 'primary')
# map from F to T index
imap = dict([((b,o),i) for i,(b,o) in enumerate(zip(T.brickid, T.objid))])
F.tindex = np.array([imap[(b,o)] for b,o in zip(F.brickid, F.objid)])
assert(np.all(T.brickid[F.tindex] == F.brickid))
assert(np.all(T.objid[F.tindex] == F.objid))
fcols = 'apflux apflux_ivar camera expnum ccdname exptime flux flux_ivar fracflux mask mjd rchi2 x y brickid objid'.split()
def main(args=None):
"""Main routine which parses the optional inputs."""
t0= Time()
# Command line options
if args is None:
# Read from cmd line
parser= get_parser()
args = parser.parse_args(args=args)
else:
# args is already a argparse.Namespace obj
pass
# Print calling sequence
print('Args:', args)
if args.do_more == 'yes':
assert(not args.minid is None)
# Setup loggers
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(level=lvl, stream=sys.stdout) #,format='%(message)s')
log = logging.getLogger('decals_sim')
# Sort through args
#log.info('decals_sim.py args={}'.format(args))
#max_nobj=500
#max_nchunk=1000
#if args.ith_chunk is not None: assert(args.ith_chunk <= max_nchunk-1)
#assert(args.nchunk <= max_nchunk)
#assert(args.nobj <= max_nobj)
#if args.ith_chunk is not None:
# assert(args.nchunk == 1) #if choose a chunk, only doing 1 chunk
if args.nobj is None:
parser.print_help()
sys.exit(1)
# Exit if expected output already exists
rsdir= get_outdir_runbrick(args.outdir,
args.brick,args.rowstart,
do_skipids=args.do_skipids,
do_more=args.do_more)
rsdir= os.path.basename(rsdir)
tractor_fn= os.path.join(args.outdir,
'tractor',args.brick[:3],args.brick,
rsdir,
'tractor-%s.fits' % args.brick)
if (os.path.exists(tractor_fn) &
(not args.overwrite_if_exists)):
print('Exiting, already finished %s' % tractor_fn)
return 0 #sys.exit(0)
brickname = args.brick
objtype = args.objtype
# Output dir
decals_sim_dir = args.outdir
#nchunk = args.nchunk
#rand = np.random.RandomState(args.seed) # determines seed for all chunks
#seeds = rand.random_integers(0,2**18, max_nchunk)
log.info('Object type = {}'.format(objtype))
#log.info('Number of objects = {}'.format(nobj))
#log.info('Number of chunks = {}'.format(nchunk))
# Optionally zoom into a portion of the brick
survey = LegacySurveyData(survey_dir=args.survey_dir)
brickinfo= get_brickinfo_hack(survey,brickname)
#brickinfo = survey.get_brick_by_name(brickname)
#print(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()
log.info('Brick = {}'.format(brickname))
if args.zoom is not None: # See also runbrick.stage_tims()
(x0, x1, y0, y1) = args.zoom
W = x1 - x0
H = y1 - y0
brickwcs = brickwcs.get_subimage(x0, y0, W, H)
log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
targetrd = np.array([brickwcs.pixelxy2radec(x, y) for x, y in
[(1,1), (W,1), (W,H), (1,H), (1,1)]])
radec_center = brickwcs.radec_center()
log.info('RA, Dec center = {}'.format(radec_center))
log.info('Brick = {}'.format(brickname))
t0= ptime('First part of Main()',t0)
# SAMPLE table
sample_kwargs= {"objtype":args.objtype,
"brick":args.brick,
"outdir":args.outdir,
"randoms_db":args.randoms_db,
"minid":args.minid,
"do_skipids":args.do_skipids,
"randoms_from_fits":args.randoms_from_fits,
"dont_sort_sampleid":args.dont_sort_sampleid}
Samp,seed= get_sample(**sample_kwargs)
Samp= Samp[args.rowstart:args.rowstart + args.nobj]
# Performance
#if objtype in ['elg','lrg']:
# Samp=Samp[np.argsort( Samp.get('%s_n' % objtype) )]
print('Max sample size=%d, actual sample size=%d' % (args.nobj,len(Samp)))
assert(len(Samp) <= args.nobj)
t0= ptime('Got randoms sample',t0)
# Store args in dict for easy func passing
kwargs=dict(Samp=Samp,\
brickname=brickname, \
checkpoint=args.checkpoint, \
seed= seed,
decals_sim_dir= decals_sim_dir,\
brickwcs= brickwcs, \
objtype=objtype,\
nobj=len(Samp),\
maxobjs=args.nobj,\
rowst=args.rowstart,\
do_skipids=args.do_skipids,\
do_more=args.do_more,\
minid=args.minid,\
survey_dir=args.survey_dir,\
args=args)
# Stop if starting row exceeds length of radec,color table
if len(Samp) == 0:
fn= get_outdir_runbrick(kwargs['decals_sim_dir'],
kwargs['brickname'],kwargs['rowst'],
do_skipids=kwargs['do_skipids'],do_more=kwargs['do_more'])
fn+= '_exceeded.txt'
junk= os.system('touch %s' % fn)
print('Wrote %s' % fn)
#we want not to add any sample -- obiwan
#raise ValueError('starting row=%d exceeds number of artificial sources, quit' % args.rowstart)
# Create simulated catalogues and run Tractor
create_metadata(kwargs=kwargs)
t0= ptime('create_metadata',t0)
# do chunks
#for ith_chunk in chunk_list:
#log.info('Working on chunk {:02d}/{:02d}'.format(ith_chunk,kwargs['nchunk']-1))
# Random ra,dec and source properties
create_ith_simcat(d=kwargs)
#log.info('HUI-TEST:::out of create_ith_simcat')
t0= ptime('create_ith_simcat',t0)
# Run tractor
#log.info('HUI-TEST:::running tractor')
do_one_chunk(d=kwargs)
#log.info('HUI-TEST::: checkpoint3i')
t0= ptime('do_one_chunk',t0)
# Clean up output
if args.no_cleanup == False:
do_ith_cleanup(d=kwargs)
#log.info('HUI-TEST::: checkpoint3j')
t0= ptime('do_ith_cleanup',t0)
log.info('All done!')
#log.info('HUI-TEST::: checkpoint3k')
return 0
def main():
survey = LegacySurveyData()
brickname = '2351p137'
# RA,Dec = 235.0442, 13.7125
bx, by = 3300, 1285
sz = 50
bbox = [bx - sz, bx + sz, by - sz, by + sz]
objid = 1394
bands = ['g', 'r', 'z']
from legacypipe.runbrick import stage_tims, _get_mod, rgbkwargs, rgbkwargs_resid
from legacypipe.survey import get_rgb, imsave_jpeg
from legacypipe.coadds import make_coadds
from astrometry.util.multiproc import multiproc
from astrometry.util.fits import fits_table
from legacypipe.catalog import read_fits_catalog
# brick = survey.get_brick_by_name(brickname)
# # Get WCS object describing brick
# targetwcs = wcs_for_brick(brick)
# (x0,x1,y0,y1) = bbox
# W = x1-x0
# H = y1-y0
# targetwcs = targetwcs.get_subimage(x0, y0, W, H)
# H,W = targetwcs.shape
mp = multiproc()
P = stage_tims(brickname=brickname,
survey=survey,
target_extent=bbox,
pixPsf=True,
hybridPsf=True,
depth_cut=False,
mp=mp)
print('Got', P.keys())
tims = P['tims']
targetwcs = P['targetwcs']
H, W = targetwcs.shape
# Read Tractor catalog
fn = survey.find_file('tractor', brick=brickname)
print('Trying to read catalog', fn)
cat = fits_table(fn)
print('Read', len(cat), 'sources')
ok, xx, yy = targetwcs.radec2pixelxy(cat.ra, cat.dec)
I = np.flatnonzero((xx > 0) * (xx < W) * (yy > 0) * (yy < H))
cat.cut(I)
print('Cut to', len(cat), 'sources within box')
I = np.flatnonzero(cat.objid != objid)
cat.cut(I)
print('Cut to', len(cat), 'sources with objid !=', objid)
#cat.about()
# Convert FITS catalog into tractor source objects
print('Creating tractor sources...')
srcs = read_fits_catalog(cat, fluxPrefix='')
print('Sources:')
for src in srcs:
print(' ', src)
print('Rendering model images...')
mods = [_get_mod((tim, srcs)) for tim in tims]
print('Producing coadds...')
C = make_coadds(tims, bands, targetwcs, mods=mods, mp=mp)
print('Coadds:', dir(C))
coadd_list = [('image', C.coimgs, rgbkwargs),
('model', C.comods, rgbkwargs),
('resid', C.coresids, rgbkwargs_resid)]
#C.coimgs, C.comods, C.coresids
for name, ims, rgbkw in coadd_list:
rgb = get_rgb(ims, bands, **rgbkw)
kwa = {}
#with survey.write_output(name + '-jpeg', brick=brickname) as out:
# imsave_jpeg(out.fn, rgb, origin='lower', **kwa)
# print('Wrote', out.fn)
outfn = name + '.jpg'
imsave_jpeg(outfn, rgb, origin='lower', **kwa)
del rgb
def main():
ps = PlotSequence('cov')
survey = LegacySurveyData()
ra, dec = 242.0, 10.2
fn = 'coverage-ccds.fits'
if not os.path.exists(fn):
ccds = survey.get_ccds()
ccds.cut(ccds.filter == 'r')
ccds.cut(ccds.propid == '2014B-0404')
ccds.cut(np.hypot(ccds.ra_bore - ra, ccds.dec_bore - dec) < 2.5)
print(np.unique(ccds.expnum), 'unique exposures')
print('propids', np.unique(ccds.propid))
ccds.writeto(fn)
else:
ccds = fits_table(fn)
plt.clf()
for e in np.unique(ccds.expnum):
I = np.flatnonzero(ccds.expnum == e)
plt.plot(ccds.ra[I], ccds.dec[I], '.')
ps.savefig()
degw = 3.0
pixscale = 10.
W = degw * 3600 / 10.
H = W
hi = 6
cmap = cmap_discretize('jet', hi + 1)
wcs = Tan(ra, dec, W / 2. + 0.5, H / 2. + 0.5, -pixscale / 3600., 0., 0.,
pixscale / 3600., float(W), float(H))
r0, d0 = wcs.pixelxy2radec(1, 1)
r1, d1 = wcs.pixelxy2radec(W, H)
extent = [min(r0, r1), max(r0, r1), min(d0, d1), max(d0, d1)]
for expnums in [
[348666],
[348666, 348710, 348686],
[348659, 348667, 348658, 348666, 348665, 348669, 348668],
None,
[
348683, 348687, 347333, 348686, 348685, 348692, 348694, 348659,
348667, 348658, 348666, 348665, 348669, 348668, 348707, 348709,
348708, 348710, 348711, 348716, 348717
],
]:
nexp = np.zeros((H, W), np.uint8)
for ccd in ccds:
if expnums is not None and not ccd.expnum in expnums:
continue
ccdwcs = survey.get_approx_wcs(ccd)
r, d = ccdwcs.pixelxy2radec(1, 1)
ok, x0, y0 = wcs.radec2pixelxy(r, d)
r, d = ccdwcs.pixelxy2radec(ccd.width, ccd.height)
ok, x1, y1 = wcs.radec2pixelxy(r, d)
xlo = np.clip(int(np.round(min(x0, x1))) - 1, 0, W - 1)
xhi = np.clip(int(np.round(max(x0, x1))) - 1, 0, W - 1)
ylo = np.clip(int(np.round(min(y0, y1))) - 1, 0, H - 1)
yhi = np.clip(int(np.round(max(y0, y1))) - 1, 0, H - 1)
nexp[ylo:yhi + 1, xlo:xhi + 1] += 1
plt.clf()
plt.imshow(nexp,
interpolation='nearest',
origin='lower',
vmin=-0.5,
vmax=hi + 0.5,
cmap=cmap,
extent=extent)
plt.colorbar(ticks=np.arange(hi + 1))
ps.savefig()
O = fits_table('obstatus/decam-tiles_obstatus.fits')
O.cut(np.hypot(O.ra - ra, O.dec - dec) < 2.5)
for p in [1, 2, 3]:
print('Pass', p, 'exposures:', O.r_expnum[O.get('pass') == p])
O.cut(O.get('pass') == 2)
print(len(O), 'pass 2 nearby')
d = np.hypot(O.ra - ra, O.dec - dec)
print('Dists:', d)
I = np.flatnonzero(d < 0.5)
assert (len(I) == 1)
ocenter = O[I[0]]
print('Center expnum', ocenter.r_expnum)
I = np.flatnonzero(d >= 0.5)
O.cut(I)
#center = ccds[ccds.expnum == ocenter.r_expnum]
#p2 = ccds[ccds.
ok, xc, yc = wcs.radec2pixelxy(ocenter.ra, ocenter.dec)
xx, yy = np.meshgrid(np.arange(W) + 1, np.arange(H) + 1)
c_d2 = (xc - xx)**2 + (yc - yy)**2
best = np.ones((H, W), bool)
for o in O:
ok, x, y = wcs.radec2pixelxy(o.ra, o.dec)
d2 = (x - xx)**2 + (y - yy)**2
best[d2 < c_d2] = False
del d2
del c_d2, xx, yy
# plt.clf()
# plt.imshow(best, interpolation='nearest', origin='lower', cmap='gray',
# vmin=0, vmax=1)
# ps.savefig()
plt.clf()
plt.imshow(nexp * best,
interpolation='nearest',
origin='lower',
vmin=-0.5,
vmax=hi + 0.5,
cmap=cmap,
extent=extent)
plt.colorbar(ticks=np.arange(hi + 1))
ps.savefig()
plt.clf()
n, b, p = plt.hist(np.clip(nexp[best], 0, hi),
range=(-0.5, hi + 0.5),
bins=hi + 1)
plt.xlim(-0.5, hi + 0.5)
ps.savefig()
print('b', b)
print('n', n)
print('fracs', np.array(n) / np.sum(n))
print('pcts',
', '.join(['%.1f' % f for f in 100. * np.array(n) / np.sum(n)]))
def main():
indir = '/global/cscratch1/sd/dstn/dr8test-1'
name = 'dr8-test1'
pretty = 'DR8 test1'
sublayers = ['', '-model', '-resid']
subpretty = {'': ' images', '-model': ' models', '-resid': ' residuals'}
survey_dir = '/global/cscratch1/sd/desiproc/dr7'
datadir = 'data'
survey = LegacySurveyData(survey_dir=survey_dir)
fn = 'map/test_layers.py'
txt = open(fn).read()
for x in sublayers:
txt = txt + '\n' + 'test_layers.append(("%s%s", "%s%s"))\n' % (
name, x, pretty, subpretty[x])
open(fn, 'wb').write(txt.encode())
print('Wrote', fn)
cmd = 'rsync -LRarv %s/./{coadd/*/*/*-{image,model}-*.fits*,tractor} %s/%s' % (
indir, datadir, name)
print(cmd)
os.system(cmd)
basedir = os.path.join(datadir, name)
allbricks = survey.get_bricks_readonly()
imagefns = glob(os.path.join(basedir, 'coadd', '*', '*',
'*-image-*.fits*'))
print('Image filenames:', len(imagefns))
brickset = set()
for fn in imagefns:
dirs = fn.split('/')
brickname = dirs[-2]
brickset.add(brickname)
print(len(brickset), 'bricks found')
I, = np.nonzero([b in brickset for b in allbricks.brickname])
bricks = allbricks[I]
brickfn = os.path.join(basedir, 'survey-bricks.fits.gz')
bricks.writeto(brickfn)
print('Wrote', brickfn)
threads = 8
tharg = '--threads %i ' % threads
# images
for scale in range(1, 8):
cmd = 'python -u render-tiles.py --kind %s --scale --zoom %i %s' % (
name, scale, tharg)
print(cmd)
os.system(cmd)
# models
for scale in range(1, 8):
cmd = 'python -u render-tiles.py --kind %s-model --scale --zoom %i %s' % (
name, scale, tharg)
print(cmd)
os.system(cmd)
for x in sublayers:
cmd = 'python -u render-tiles.py --kind %s%s --top' % (name, x)
print(cmd)
os.system(cmd)
alldec = []
allstate = []
alltasks = []
# allra.append(ra)
# alldec.append(dec)
# allstate.append([state] * len(ra))
# alltasks.append(tasks)
ra = np.hstack(allra)
dec = np.hstack(alldec)
state = np.hstack(allstate)
tasks = np.hstack(alltasks)
# Match to actual table of bricks to get brickq.
survey = LegacySurveyData()
bricks = survey.get_bricks_readonly()
I,J,d = match_radec(ra, dec, bricks.ra, bricks.dec, 0.2, nearest=True)
print(len(ra), 'jobs')
print(len(I), 'matches')
ra = ra[I]
dec = dec[I]
state = state[I]
tasks = tasks[I]
brickq = bricks.brickq[J]
for q in [0,1,2,3]:
print()
print('Brickq', q)
plt.clf()
def psf_residuals(expnum,
ccdname,
stampsize=35,
nstar=30,
magrange=(13, 17),
verbose=0,
splinesky=False):
# Set the debugging level.
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
pngprefix = 'qapsf-{}-{}'.format(expnum, ccdname)
# Gather all the info we need about this CCD.
survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
band = ccd.filter
ps1band = dict(g=0, r=1, i=2, z=3, Y=4)
print('Band {}'.format(band))
#scales = dict(g=0.0066, r=0.01, z=0.025)
#vmin, vmax = np.arcsinh(-1), np.arcsinh(100)
#print(scales[band])
im = survey.get_image_object(ccd)
iminfo = im.get_image_info()
H, W = iminfo['dims']
wcs = im.get_wcs()
# Choose a uniformly selected subset of PS1 stars on this CCD.
ps1 = ps1cat(ccdwcs=wcs)
cat = ps1.get_stars(band=band, magrange=magrange)
rand = np.random.RandomState(seed=expnum * ccd.ccdnum)
these = rand.choice(len(cat) - 1, nstar, replace=False)
#these = rand.random_integers(0,len(cat)-1,nstar)
cat = cat[these]
cat = cat[np.argsort(cat.median[:, ps1band[band]])] # sort by magnitude
#print(cat.nmag_ok)
get_tim_kwargs = dict(pixPsf=True, splinesky=splinesky)
# Make a QAplot of the positions of all the stars.
tim = im.get_tractor_image(**get_tim_kwargs)
img = tim.getImage()
#img = tim.getImage()/scales[band]
fig = plt.figure(figsize=(5, 10))
ax = fig.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
#ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest',
# origin='lower',vmin=vmax,vmax=vmax)
ax.imshow(img, **tim.ima)
ax.axis('off')
ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.format(
band, expnum, ccdname, ccd.airmass, ccd.seeing))
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ax.text(xpos,
ypos,
'{:2d}'.format(istar + 1),
color='red',
horizontalalignment='left')
circ = plt.Circle((xpos, ypos), radius=30, color='g', fill=False, lw=1)
ax.add_patch(circ)
#radec = wcs.radec_bounds()
#ax.scatter(cat.ra,cat.dec)
#ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998])
#ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015])
#ax.set_xlabel('$RA\ (deg)$',fontsize=18)
#ax.set_ylabel('$Dec\ (deg)$',fontsize=18)
fig.savefig(pngprefix + '-ccd.png', bbox_inches='tight')
# Initialize the many-stamp QAplot
ncols = 3
nrows = np.ceil(nstar / ncols).astype('int')
inchperstamp = 2.0
fig = plt.figure(figsize=(inchperstamp * 3 * ncols, inchperstamp * nrows))
irow = 0
icol = 0
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
mag = ps1star.median[ps1band[band]] # r-band
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ix, iy = int(xpos), int(ypos)
# create a little tractor Image object around the star
slc = (slice(max(iy - stampsize, 0), min(iy + stampsize + 1, H)),
slice(max(ix - stampsize, 0), min(ix + stampsize + 1, W)))
# The PSF model 'const2Psf' is the one used in DR1: a 2-component
# Gaussian fit to PsfEx instantiated in the image center.
tim = im.get_tractor_image(slc=slc, **get_tim_kwargs)
stamp = tim.getImage()
ivarstamp = tim.getInvvar()
# Initialize a tractor PointSource from PS1 measurements
flux = NanoMaggies.magToNanomaggies(mag)
star = PointSource(RaDecPos(ra, dec), NanoMaggies(**{band: flux}))
# Fit just the source RA,Dec,flux.
tractor = Tractor([tim], [star])
tractor.freezeParam('images')
print('2-component MOG:', tim.psf)
tractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_mog = tractor.getModelImage(0)
chi2_mog = -2.0 * tractor.getLogLikelihood()
mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
# Now change the PSF model to a pixelized PSF model from PsfEx instantiated
# at this place in the image.
psf = PixelizedPsfEx(im.psffn)
tim.psf = psf.constantPsfAt(xpos, ypos)
#print('PSF model:', tim.psf)
#tractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_psfex = tractor.getModelImage(0)
chi2_psfex = -2.0 * tractor.getLogLikelihood()
mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
#mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95])
sig = np.std((stamp - model_psfex)[ivarstamp > 0])
mn, mx = [-2.0 * sig, 5 * sig]
# Generate a QAplot.
if (istar > 0) and (istar % (ncols) == 0):
irow = irow + 1
icol = 3 * istar - 3 * ncols * irow
#print(istar, irow, icol, icol+1, icol+2)
ax1 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol),
aspect='equal')
ax1.axis('off')
#ax1.imshow(stamp, **tim.ima)
ax1.imshow(stamp,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax1.text(0.1,
0.9,
'{:2d}'.format(istar + 1),
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax1.transAxes)
ax2 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 1),
aspect='equal')
ax2.axis('off')
#ax2.imshow(stamp-model_mog, **tim.ima)
ax2.imshow(stamp - model_mog,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax2.text(0.1,
0.9,
'MoG',
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax2.transAxes)
ax2.text(0.08,
0.08,
'{:.3f}'.format(mag_mog),
color='white',
horizontalalignment='left',
verticalalignment='bottom',
transform=ax2.transAxes)
#ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14)
#ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex))
ax3 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 2),
aspect='equal')
ax3.axis('off')
#ax3.imshow(stamp-model_psfex, **tim.ima)
ax3.imshow(stamp - model_psfex,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax3.text(0.1,
0.9,
'PSFEx',
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax3.transAxes)
ax3.text(0.08,
0.08,
'{:.3f}'.format(mag_psfex),
color='white',
horizontalalignment='left',
verticalalignment='bottom',
transform=ax3.transAxes)
if istar == (nstar - 1):
break
fig.savefig(pngprefix + '-stargrid.png', bbox_inches='tight')
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'--expnum',
type=str,
help='Run specified exposure numbers (can be comma-separated list')
parser.add_argument(
'--all-found',
action='store_true',
default=False,
help='Only write output if all required input files are found')
parser.add_argument('--ccds',
help='Set ccds.fits file to load, default is all')
parser.add_argument('--continue',
dest='con',
help='Continue even if one exposure is bad',
action='store_true',
default=False)
parser.add_argument('--outdir',
help='Output directory, default %(default)s',
default='calib')
opt = parser.parse_args()
survey = LegacySurveyData()
if opt.ccds:
ccds = fits_table(opt.ccds)
ccds = survey.cleanup_ccds_table(ccds)
survey.ccds = ccds
if opt.expnum is not None:
expnums = [(None, int(x, 10)) for x in opt.expnum.split(',')]
else:
ccds = survey.get_ccds()
expnums = set(zip(ccds.camera, ccds.expnum))
print(len(expnums), 'unique camera+expnums')
for i, (camera, expnum) in enumerate(expnums):
print()
print('Exposure', i + 1, 'of', len(expnums), ':', camera, 'expnum',
expnum)
if camera is None:
C = survey.find_ccds(expnum=expnum)
print(len(C), 'CCDs with expnum', expnum)
camera = C.camera[0]
print('Set camera to', camera)
C = survey.find_ccds(expnum=expnum, camera=camera)
print(len(C), 'CCDs with expnum', expnum, 'and camera', camera)
im0 = survey.get_image_object(C[0])
skyoutfn = im0.merged_skyfn
psfoutfn = im0.merged_psffn
print('Checking for', skyoutfn)
print('Checking for', psfoutfn)
if os.path.exists(skyoutfn) and os.path.exists(psfoutfn):
print('Exposure', expnum, 'is done already')
continue
if not os.path.exists(skyoutfn):
try:
merge_splinesky(survey, expnum, C, skyoutfn, opt)
except:
if not opt.con:
raise
import traceback
traceback.print_exc()
print('Exposure failed:', expnum, '. Continuing...')
if not os.path.exists(psfoutfn):
try:
merge_psfex(survey, expnum, C, psfoutfn, opt)
except:
if not opt.con:
raise
import traceback
traceback.print_exc()
print('Exposure failed:', expnum, '. Continuing...')
type=str,
default=None,
help='Override the $LEGACY_SURVEY_DIR environment variable')
parser.add_argument('--out',
'-o',
default='skyfibers.fits',
help='Output filename')
parser.add_argument('--plots',
'-p',
default=None,
help='Plots base filename')
parser.add_argument('--brick', default=None, help='Brick name')
opt = parser.parse_args()
if not opt.brick:
parser.print_help()
sys.exit(-1)
from legacypipe.survey import LegacySurveyData
survey = LegacySurveyData(survey_dir=opt.survey_dir)
skyfibers = sky_fibers_for_brick(survey, opt.brick)
skyfibers.writeto(opt.out, header=skyfibers._header)
print('Wrote', opt.out)
if opt.plots:
import matplotlib
matplotlib.use('Agg')
sky_fiber_plots(survey, opt.brick, skyfibers, opt.plots)