def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--mzls',
action='store_true',
help='Set MzLS (default: DECaLS)')
parser.add_argument('--ann', help='Set annotated-CCDs file')
opt = parser.parse_args()
if opt.mzls:
from mosaic import MosaicNominalCalibration
from camera_mosaic import database_filename, camera_name
nom = MosaicNominalCalibration()
obstatus_fn = 'obstatus/mosaic-tiles_obstatus.fits'
out_fn = 'mosaic-obstatus-depth.fits'
bands = 'z'
declo, dechi = -5, 90
bad_expid_fn = 'obstatus/bad_expid.txt'
else:
from decam import DecamNominalCalibration
from camera_decam import database_filename, camera_name
nom = DecamNominalCalibration()
# ln -s ~/observing/obstatus/bad_expid.txt obstatus/decam-bad_expid.txt
obstatus_fn = 'obstatus/decam-tiles_obstatus.fits'
out_fn = 'decam-obstatus-depth.fits'
bad_expid_fn = 'obstatus/decam-bad_expid.txt'
bands = 'grz'
declo, dechi = -20, 35
f = open(bad_expid_fn)
bad_expids = set()
for line in f:
line = line.strip()
if len(line) == 0:
continue
if line[0] == '#':
continue
words = line.split()
try:
expnum = int(words[0])
except:
print('Skipping line:', line)
continue
bad_expids.add(expnum)
print('Read', len(bad_expids), 'bad exposure numbers')
# Convert copilot db to fits.
import obsdb
from copilot import db_to_fits
obsdb.django_setup(database_filename=database_filename)
ccds = obsdb.MeasuredCCD.objects.all()
copilot = db_to_fits(ccds)
all_copilot = copilot.copy()
fn = 'copilot.fits'
copilot.writeto(fn)
print('Wrote', fn)
print(len(copilot), 'measured CCDs in copilot database')
copilot.cut(np.array([c.strip() == camera_name for c in copilot.camera]))
print(len(copilot), 'copilot CCDs with camera = "%s"' % camera_name)
copilot.cut(copilot.expnum > 0)
print(len(copilot), 'measured CCDs in copilot database with EXPNUM')
print('Copilot expfactor extremes:',
np.percentile(copilot.expfactor[copilot.expfactor != 0], [1, 99]))
survey = LegacySurveyData()
if opt.ann:
ccds = fits_table(opt.ann)
else:
print('Reading annotated CCDs files...')
ccds = survey.get_annotated_ccds()
print(len(ccds), 'CCDs')
# Fix parsing of OBJECT field to tileid...
from obsbot import get_tile_id_from_name
tileids = []
for o in ccds.object:
tid = get_tile_id_from_name(o.strip())
if tid is None:
tid = 0
tileids.append(tid)
tileids = np.array(tileids)
print(len(np.unique(tileids)),
'unique tile ids in annotated file, from OBJECT')
print(len(np.unique(ccds.tileid)),
'unique tile ids in ann file from TILEID')
D = np.flatnonzero(tileids != ccds.tileid)
print(len(D), 'different tileids')
print('From OBJECT:', tileids[D])
print('From TILEID:', ccds.tileid[D])
ccds.tileid = tileids
O = fits_table(obstatus_fn)
print(len(O), 'tiles')
if opt.mzls:
from camera_mosaic import fix_expnums
# Fix MzLS exposure numbers with wrong leading "3".
fix_expnums(ccds.expnum)
# Also fix leading "3" in expnums in OBSTATUS file
fix_expnums(O.z_expnum)
# And copilot database
fix_expnums(copilot.expnum)
print('Z_EXPNUM range:', O.z_expnum.min(), 'min >0:',
O.z_expnum[O.z_expnum > 0].min(), O.z_expnum.max())
print('Pass numbers:', np.unique(O.get('pass')))
if opt.mzls:
goodtiles = (O.in_desi * (O.dec > 30) * (O.get('pass') <= 3))
print(sum(goodtiles), 'tiles of interest')
else:
goodtiles = (O.in_desi * (O.get('pass') <= 3))
print(sum(goodtiles), 'tiles in the footprint')
#O.cut(goodtiles)
#print('Cut to', len(O), 'tiles of interest')
# *after* fixing tileids
allccds = ccds.copy()
# Map tile IDs back to index in the obstatus file.
tileid_to_index = np.empty(max(O.tileid) + 1, int)
tileid_to_index[:] = -1
tileid_to_index[O.tileid] = np.arange(len(O))
assert (len(np.unique(O.tileid)) == len(O))
I = tileid_to_index[O.tileid]
assert (np.all(I == np.arange(len(O))))
# Look at whether exposures from other programs are near our tile centers.
# Basically nope.
# plt.clf()
# e,K = np.unique(ccds.expnum, return_index=True)
# I,J,d = match_radec(O.ra, O.dec, ccds.ra_bore[K], ccds.dec_bore[K],
# 1./60., nearest=True)
# KK = K[np.flatnonzero(ccds.tileid[K] > 0)]
# I,J,d2 = match_radec(O.ra, O.dec, ccds.ra_bore[KK], ccds.dec_bore[KK],
# 1./60., nearest=True)
# ha = dict(range=(0., 60.), bins=60, histtype='step')
# plt.hist(d * 3600., color='b', **ha)
# plt.hist(d2 * 3600., color='r', **ha)
# plt.xlabel('Distance from tile to nearest DECam boresight (arcsec)')
# plt.savefig('dists.png')
notileids = ccds[ccds.tileid <= 0]
print(len(notileids), 'CCDs have no tileid')
I, J, d = match_radec(notileids.ra_bore,
notileids.dec_bore,
O.ra,
O.dec,
0.5,
nearest=True)
plt.clf()
plt.hist(d, bins=50)
plt.xlabel('Distance to nearest tile center (deg)')
plt.savefig('tiledist.png')
plt.clf()
plt.hist(d * 3600, bins=50, range=(0, 30))
plt.xlabel('Distance to nearest tile center (arcsec)')
plt.savefig('tiledist2.png')
ccds.cut(ccds.tileid > 0)
print(len(ccds), 'CCDs with tileid')
expnums, I = np.unique(ccds.expnum, return_index=True)
print(len(expnums), 'unique exposures (with tileids)')
ccds.photometric = (ccds.ccd_cuts == 0)
# Compute the mean depth per exposure
E = ccds[I]
for expnum in expnums:
I = np.flatnonzero(ccds.expnum == expnum)
j = np.flatnonzero(E.expnum == expnum)
assert (len(j) == 1)
j = j[0]
E.photometric[j] = np.all(ccds.photometric[I])
#E.photometric[j] = np.all(ccds.ccd_cuts[I] == 0)
if len(np.unique(ccds.photometric[I])) == 2:
print('Exposure', expnum,
'has photometric and non-photometric CCDs')
non = I[ccds.photometric[I] == False]
phot = I[ccds.photometric[I]]
if opt.mzls and len(phot) == 3:
print('Accepting an exposure with 3 good CCDs')
E.photometric[j] = True
# And remove this exposure from the bad_expid list.
if expnum in bad_expids:
bad_expids.remove(expnum)
print('Removing exposure', expnum, 'from bad_expid file')
continue
for ii in non:
print(
' http://legacysurvey.org/viewer-dev/?ra=%.3f&dec=%.3f&zoom=11&ccds3&bad=%i-%s'
% (ccds.ra_center[ii], ccds.dec_center[ii], expnum,
ccds.ccdname[ii]))
print(
' http://legacysurvey.org/viewer-dev/ccd/decals-dr5/decam-%s-%s-%s/'
% (ccds.expnum[ii], ccds.ccdname[ii], ccds.filter[ii]))
print(' image:', ccds.image_filename[I][0])
print(' boresight:', ccds.ra_bore[I][0], ccds.dec_bore[I][0])
#print(' ccdnames:', ccds.ccdname[I])
print(' photometric:', len(phot), ', non-photometric:', len(non))
print(' median phot depth:', np.median(ccds.galdepth[phot]))
#print(' depth:', ccds.galdepth[I])
print(' non-photometric CCDs:', ccds.ccdname[non])
print(' depths:', ccds.galdepth[non])
print(' ccdnmatch', ccds.ccdnmatch[non], 'vs',
ccds.ccdnmatch[phot])
print(' ccdtransp:', ccds.ccdtransp[non], 'vs',
ccds.ccdtransp[phot])
print(' ccd zpt vs frame zpt:',
ccds.ccdzpt[non] - ccds.zpt[non])
dp = ccds.ccdzpt[phot] - ccds.zpt[phot]
print(' phot ccds zpt vs frame: range', dp.min(), dp.max(),
'mean', dp.mean())
whitelist = [
346662,
346664,
346665, # S3/S29 striping
346754, # one bad chip, wispy
346967,
347304, # M5 globular
347664, # zpt scatter
347744, # weird eye-shaped ghost; but lots of cov.
347755,
347768,
347769,
347782, # shallow, zpt scatter -- wispy pattern on focal plane
347918,
347920, # straddling transparency cut
347934,
347936,
347941,
347945,
347947, # zpt scatter
392377,
392380,
393173, # bright star
393671, # bright star
393672,
393673, # scatter
425339,
425340, # strangely low ccdnmatch
426225,
430808, # globular cluster
431640, # bright star
431644, # globular
432154, # one amp high bias
432159, # transp. on boundary
432179, # one amp high bias, +
432747,
432748,
432751, # scatter
433305,
433306, # bright star
497062,
497064,
497065, # low ccdnmatch
509516,
509517, # bright star
511247,
511263, # low ccdnmatch
511513,
511514, # bright star
512303, # bright star
520560, # bright star
521782, # scatter
522212, # bright star
535138, # bright stars, satellite hits?
535141,
535142,
535143,
535149, # low ccdnmatch
535210, # bright star
535695, # globular
536065, # globular
536385, # strangely zero ccdnmatch
547761, # nice galaxy
548257, # bright star
553779, # scatter
553795, # shallow
554284, # marginal zpt
563659, # zpt scatter
563850, # mild striping
563852, # ??
583118, # bright stars?
592621, # marginal zpt, scatter
592859, # some pattern noise
605068, # ??
625710, # strangely low ccdnmatch
631005, # bright star
634493, # globular
634786, # strangely low ccdnmatch
634877, # globular
635535, # bright star, glob
635962, # low ccdnmatch
635973, # bias level?
636018, # bias level?
637688, # bright star
]
blacklist = [
425328, # 2.7" seeing
488244,
488256,
488260,
488261,
488263,
488268, # weird striping
488270, # weird striping
496913,
496917,
496918,
496919,
496920,
496921,
496922, # 3" seeing
496923,
496925,
496926,
496927,
496928,
496930, # 3" seeing
509162,
509163,
509166,
509172,
509176,
509182,
509202, # 3" seeing
535471, # 4" seeing!
535498, # 3" seeing
548218, # double PSF -- telescope moved?
563835, # striping
563842, # striping
]
if expnum in whitelist:
print('** Exposure', expnum,
'in whitelist -- marking as photometric')
E.photometric[j] = True
# Don't include zeros in computing average depths!
Igood = I[(ccds.galdepth[I] > 0) * (ccds.ccdzpt[I] < 30)]
if len(Igood) > 0:
E.galdepth[j] = np.mean(ccds.galdepth[Igood])
else:
E.galdepth[j] = 0.
del expnums
keep = np.array([not (expnum in bad_expids) for expnum in ccds.expnum])
ccds.cut(keep)
print(len(ccds), 'CCDs NOT in the bad_expids file')
keep = np.array([not (expnum in bad_expids) for expnum in copilot.expnum])
copilot.cut(keep)
print(len(copilot), 'copilot exposures NOT in the bad_expids file')
keep = np.array([not (expnum in bad_expids) for expnum in E.expnum])
E.cut(keep)
print(len(E), 'CCD Exposures NOT in the bad_expids file')
# plt.clf()
# plt.plot(O.ra, O.dec, 'k.')
# plt.axis([360,0,-25,35])
# plt.title('All tiles')
# plt.savefig('tiles-all.png')
#
# print('in_desi:', np.unique(O.in_desi))
# plt.clf()
# J = np.flatnonzero(O.in_desi == 1)
# plt.plot(O.ra[J], O.dec[J], 'k.')
# plt.axis([360,0,-25,35])
# plt.title('In DESI')
# plt.savefig('tiles-desi.png')
#
# print('in_des:', np.unique(O.in_des))
# plt.clf()
# J = np.flatnonzero(O.in_des == 1)
# plt.plot(O.ra[J], O.dec[J], 'k.')
# plt.axis([360,0,-25,35])
# plt.title('IN DES')
# plt.savefig('tiles-des.png')
#print('Number of exposures of each tile:')
#print(Counter(E.tileid).most_common())
print()
print()
print('Number of exposures of tiles:')
for band in bands:
I = np.flatnonzero(E.filter == band)
c = Counter(E.tileid[I])
c2 = Counter([v for k, v in c.most_common()])
print(' ', band, 'band:', c2.most_common())
# Detection inverse-variance is the quantity that adds when there are
# multiple exposures.
# detsig1 = ccds.sig1 / ccds.galnorm_mean
# depth = 5. * detsig1
# # that's flux in nanomaggies -- convert to mag
# ccds.galdepth = -2.5 * (np.log10(depth) - 9)
with np.errstate(divide='ignore', over='ignore'):
# actually 5*detsig1...
detsig = 10.**((E.galdepth - 22.5) / -2.5)
E.detiv = 1. / detsig**2
E.detiv[E.galdepth == 0] = 0.
print('Smallest detivs:', E.detiv[np.argsort(E.detiv)[:10]])
print('w/ galdepths:', E.galdepth[np.argsort(E.detiv)[:10]])
print('Smallest positive detivs:',
E.detiv[np.argsort(E.detiv + 1e12 * (E.detiv == 0))[:10]])
print('w/ galdepths:',
E.galdepth[np.argsort(E.detiv + 1e12 * (E.detiv == 0))[:10]])
for band in bands:
print()
print('------------------')
print(band, 'band.')
# "I" indexes into exposures E.
I = np.flatnonzero(
(E.filter == band) * E.photometric * np.isfinite(E.detiv))
print(len(I), 'photometric exposures in', band)
# "iv" is parallel to O; will be converted to "galdepth".
iv = np.zeros(len(O), np.float32)
# "J" indexes into obstatus tiles O.
J = tileid_to_index[E.tileid[I]]
assert (np.all((J >= 0) * (J < len(O))))
assert (np.all(O.tileid[J] == E.tileid[I]))
#print('tileid range', E.tileid[I].min(), E.tileid[I].max())
# d = np.array([degrees_between(*a) for a in
# zip(E.ra_bore[I], E.dec_bore[I], O.ra[J], O.dec[J])])
# print('Degrees between tiles & exposures:', d)
np.add.at(iv, J, E.detiv[I])
print('galdepth range:', E.galdepth[I].min(), E.galdepth[I].max())
print('detiv range:', E.detiv[I].min(), E.detiv[I].max())
#print('index range:', J.min(), J.max())
nexp = np.zeros(len(O), int)
np.add.at(nexp, J, 1)
print('tile exposure counts:', Counter(nexp))
# convert iv back to galdepth in mags
with np.errstate(divide='ignore'):
galdepth = -2.5 * (np.log10(np.sqrt(1. / iv)) - 9)
galdepth[iv == 0] = 0.
# Shallowest before extinction correction
#I = np.argsort(iv + 1e6*(iv == 0))
I = np.argsort(galdepth + 50. * (galdepth == 0))
print(
'Shallowest depth estimates from annotated CCDs file, before extinction:'
)
for i in I[:10]:
print(' ', galdepth[i], 'iv', iv[i], 'tile', O.tileid[i],
'expnum',
O.get('%s_expnum' % band)[i])
e = O.get('%s_expnum' % band)[i]
j = np.flatnonzero(E.expnum == e)
print(' galdepth', E.galdepth[j])
fid = nom.fiducial_exptime(band)
extinction = O.ebv_med * fid.A_co
#print('Extinction range:', extinction.min(), extinction.max())
galdepth -= extinction
galdepth[iv == 0] = 0.
# Shallowest galdepth > 0
I = np.argsort(galdepth + 50. * (galdepth == 0))
print('Shallowest depth estimates from annotated CCDs file:')
for i in I[:10]:
print(' ', galdepth[i], 'tile', O.tileid[i], 'expnum',
O.get('%s_expnum' % band)[i])
#print('galdepth deciles:', np.percentile(galdepth, [0,10,20,30,40,50,60,70,80,90,100]))
# Z_DONE, Z_EXPNUM but no Z_DEPTH
missing_depth = np.flatnonzero(
(O.get('%s_expnum' % band) > 0) * (O.get('%s_done' % band) == 1) *
(galdepth == 0))
print('Found', len(missing_depth), 'tiles with', band,
'DONE and EXPNUM but no DEPTH; setting to DEPTH=30')
print(' eg, EXPNUMs',
O.get('%s_expnum' % band)[missing_depth[:10]], 'DATE',
O.get('%s_date' % band)[missing_depth[:10]])
# Don't actually update 'galdepth[missing_depth]' until after this next check...
# Flag tiles that have *only* non-photometric exposures with depth = 1.
I = np.flatnonzero((E.filter == band) * np.logical_not(E.photometric))
print(len(I), 'exposures are non-photometric in', band, 'band')
J = tileid_to_index[E.tileid[I]]
only_nonphot = J[galdepth[J] == 0.]
print(len(only_nonphot), 'tiles have only non-photometric exposures')
print('Marking', len(only_nonphot), 'non-photometric tiles in', band,
'with depth=1')
orig_galdepth = galdepth.copy()
galdepth[missing_depth] = 30.
galdepth[only_nonphot] = 1.
J = tileid_to_index[E.tileid[I]]
nonphot = (galdepth[J] == 1.)
print('Non-photometric galdepths:', E.galdepth[I])
print('Non-photometric galdepths:', E.galdepth[I[nonphot]])
plt.clf()
phot = np.flatnonzero((E.filter == band) * E.photometric)
plt.hist(E.galdepth[phot],
range=(18, 26),
bins=50,
histtype='step',
color='b',
label='Photometric')
plt.hist(E.galdepth[I[nonphot]],
range=(18, 26),
bins=50,
histtype='step',
color='r',
label='Non-phot')
plt.legend()
plt.savefig('nonphot-%s.png' % band)
# expnum_to_copilot = np.empty(expnums.max()+1, int)
# expnum_to_copilot[:] = -1
# expnum_to_copilot[copilot.expnum] = np.arange(len(copilot))
expnum_to_copilot = dict([(e, i)
for i, e in enumerate(copilot.expnum)])
if False:
# Let's check the accuracy of the copilot's depth estimates...
target_exptime = copilot.expfactor * fid.exptime
# What fraction of the target exposure time did we take?
depth_factor = copilot.exptime / target_exptime
nomdepth = fid.single_exposure_depth
depth = nomdepth + 2.5 * np.log10(np.sqrt(depth_factor))
#print('Copilot predicted depths:', depth)
IC = np.array(
[expnum_to_copilot.get(e, -1) for e in allccds.expnum])
K = np.flatnonzero(IC >= 0)
ext = np.array([
e['ugrizY'.index(f)]
for e, f in zip(allccds.decam_extinction, allccds.filter)
])
dd = allccds.galdepth - ext
print('Making scatterplot...', len(K), 'points')
plt.clf()
#plt.plot(dd[K], depth[IC[K]], 'b.', alpha=0.2, mec='none')
plt.scatter(dd[K],
depth[IC[K]],
c=np.clip(copilot.expfactor[IC[K]], 0, 2),
s=10,
alpha=0.2,
edgecolors='none')
plt.colorbar()
plt.xlabel('Pipeline depth')
plt.ylabel('Copilot depth proxy')
plt.plot([20, 25], [20, 25], 'k-', alpha=0.25)
plt.plot([20, 25], [20 + 0.1, 25 + 0.1], 'k--', alpha=0.25)
plt.plot([20, 25], [20 - 0.1, 25 - 0.1], 'k--', alpha=0.25)
plt.axis([20.5, 23, 21, 23.5])
plt.title(
'Copilot vs Pipeline depth estimates. (color = exp.factor)')
plt.savefig('depth-copilot-%s.png' % band)
print('Made scatterplot')
plt.clf()
ha = dict(bins=60, range=(0, 30), log=True, histtype='step')
plt.hist(O.get('%s_depth' % band), color='k', label='Before', **ha)
plt.hist(orig_galdepth, color='b', label='Annotated CCDs', **ha)
# Do we have measurements for any of these missing tiles in the copilot db?
for code in [30, 0]:
Igal = np.flatnonzero(
(O.get('%s_expnum' % band) > 0) *
(O.get('%s_done' % band) == 1) * (galdepth == code))
expnums = O.get('%s_expnum' % band)[Igal]
print(
len(expnums),
'still marked DONE, with EXPNUM, but missing DEPTH, with code =',
code)
Ihuh = np.flatnonzero(
(O.get('%s_done' % band) == 1) * (galdepth == code))
print(len(Ihuh), 'tiles marked DONE, without EXPNUM, and DEPTH =',
code)
if len(Ihuh):
print('Tile ids:', O.tileid[Ihuh])
for t in O.tileid[Ihuh]:
I = np.flatnonzero(E.tileid == t)
print(' tile', t, ': exposure numbers:', E.expnum[I])
print(' with depths', E.galdepth[I])
i = tileid_to_index[t]
if i >= 0:
print(' depth', galdepth[i])
else:
print(' no depth')
# Within an arcmin?
I, J, d = match_radec(O.ra[Ihuh],
O.dec[Ihuh],
all_copilot.rabore,
all_copilot.decbore,
1. / 60.,
nearest=True)
print('For', len(Ihuh), 'weird tiles,')
print(len(I), 'matches within an arcmin in the copilot db')
print('Smallest distances:', d[np.argsort(d)[:10]])
I, J, d = match_radec(O.ra[Ihuh],
O.dec[Ihuh],
allccds.ra_bore,
allccds.dec_bore,
1. / 60.,
nearest=True)
print(len(I), 'matches within an arcmin in the CCDs table')
print('Smallest distances:', d[np.argsort(d)[:10]])
O[Ihuh].writeto('weird-%s-%i.fits' % (band, code))
if len(expnums) == 0:
continue
IC = np.array([expnum_to_copilot.get(e, -1) for e in expnums])
K = np.flatnonzero(IC >= 0)
expnums = expnums[K]
# these are the indices into O / galdepth
Igal = Igal[K]
co = copilot[IC[K]]
print(len(expnums), 'matched to copilot database')
target_exptime = co.expfactor * fid.exptime
# What fraction of the target exposure time did we take?
depth_factor = co.exptime / target_exptime
nomdepth = fid.single_exposure_depth
print('Nominal single-exposure depth:', nomdepth)
co.depth = nomdepth + 2.5 * np.log10(np.sqrt(depth_factor))
print('Copilot predicted depths:', co.depth)
J = np.flatnonzero(np.isfinite(co.depth))
co = co[J]
# indices into O
Igal = Igal[J]
print(len(Igal), 'good copilot depth estimates')
pcts = [0, 1, 5, 25, 50, 75, 95, 99, 100]
print('Copilot depth percentiles:', np.percentile(co.depth, pcts))
print('Shallowest exposures:')
I = np.argsort(co.depth)
for i in I[:10]:
print(' Expnum', co.expnum[i], 'depth', co.depth[i],
'exptime', co.exptime[i])
co[I].writeto('depths.fits')
from astrometry.util.starutil_numpy import mjdtodate
print('Copilot-matched entries:')
I = np.argsort(co.expnum)
for i in I:
print(' EXPNUM', co.expnum[i], 'date',
mjdtodate(co.mjd_obs[i]), ' copilot name',
co.filename[i])
# e = co.expnum[i]
# I = np.flatnonzero(allccds.expnum == e-1)
# fn1 = None
# fn2 = None
# if len(I):
# print(' CCDs file contains', len(I), 'entries for expnum', e-1)
# print(' filename', allccds.image_filename[I[0]])
# fn1 = allccds.image_filename[I[0]]
# else:
# print(' No CCDs file entries for expnum', e-1)
# I = np.flatnonzero(allccds.expnum == e+1)
# if len(I):
# print(' CCDs file contains', len(I), 'entries for expnum', e+1)
# print(' filename', allccds.image_filename[I[0]])
# fn2 = allccds.image_filename[I[0]]
# else:
# print(' No CCDs file entries for expnum', e+1)
#
# if fn1 is not None and fn2 is not None:
# full1 = os.path.join(survey.get_image_dir(), fn1)
# #print('Full path 1:', full1)
# if os.path.exists(full1):
# print('exists')
# full2 = os.path.join(survey.get_image_dir(), fn2)
# #print('Full path 2:', full2)
# if os.path.exists(full2):
# print('exists')
# if os.path.exists(full1) and os.path.exists(full2):
# dir1 = os.path.dirname(full1)
# dir2 = os.path.dirname(full2)
# if dir1 == dir2:
# #print('dir:', dir1)
# fns = os.listdir(dir1)
# fns.sort()
# fns = [fn for fn in fns if ('oki' in fn or 'ooi' in fn)]
# base1 = os.path.basename(full1)
# base2 = os.path.basename(full2)
# i1 = fns.index(base1)
# i2 = fns.index(base2)
# print('Files found at list elements', i1, i2)
# #print(fns[i1:i2+1])
# for fn in fns[i1:i2+1]:
# print('EXPNUM', e, 'range', os.path.join(os.path.dirname(fn1), fn))
# if i1 + 4 == i2:
# print('EXPNUM', e, 'expected', os.path.join(os.path.dirname(fn1), fns[i1+2]))
# if i1 + 2 == i2:
# print('EXPNUM', e, 'expected', os.path.join(os.path.dirname(fn1), fns[i1+1]))
#print('Before:', galdepth[Igal])
galdepth[Igal] = co.depth
#print('After:', galdepth[Igal])
Igal = np.flatnonzero(
(O.get('%s_expnum' % band) > 0) *
(O.get('%s_done' % band) == 1) * (galdepth == code))
expnums = O.get('%s_expnum' % band)[Igal]
print(
len(expnums),
'still marked DONE, with EXPNUM, but missing DEPTH with code =',
code, 'after copilot patching')
print('Exposure numbers:', expnums)
print('Exposure dates:', O.get('%s_date' % band)[Igal])
print('Date counter:',
Counter(O.get('%s_date' % band)[Igal]).most_common())
O.set('%s_depth' % band, galdepth)
plt.hist(O.get('%s_depth' % band), color='r', label='After', **ha)
plt.savefig('depth-hist-%s.png' % band)
#print('Depth deciles: [', ', '.join(['%.3f' % f for f in np.percentile(O.get('%s_depth' % band), [0,10,20,30,40,50,60,70,80,90,100])]) + ']')
rlo, rhi = 0, 360
dlo, dhi = declo, dechi
J = np.flatnonzero(
(O.in_desi == 1) * (O.in_des == 0) * (O.dec > dlo) * (O.dec < dhi))
print('Median E(B-V) in DECaLS area:', np.median(O.ebv_med[J]))
print('Median extinction in DECaLS area, %s band:' % band,
np.median(extinction[J]))
I2 = np.flatnonzero((O.get('%s_expnum' % band) > 0) *
(O.get('%s_depth' % band) == 30) *
(O.get('%s_done' % band) == 1) * (O.in_desi == 1))
print(len(I2), 'with EXPNUM and DONE and IN_DESI, but no DEPTH')
# Sort by expnum
I2 = I2[np.argsort(O.get('%s_expnum' % band)[I2])]
print('Exposure numbers:', sorted(O.get('%s_expnum' % band)[I2]))
print('Dates:', sorted(O.get('%s_date' % band)[I2]))
print('Dates:', np.unique(O.get('%s_date' % band)[I2]))
for i in I2:
print(' date',
O.get('%s_date' % band)[i], 'expnum',
O.get('%s_expnum' % band)[i])
# for i2,o in zip(I2, O[I2]):
# print()
# e = o.get('%s_expnum' % band)
# print(' Expnum', e, 'orig galdepth', orig_galdepth[i2])
# date = o.get('%s_date' % band)
# print(' Date', date)
# print(' Pass', o.get('pass'), 'Tile', o.tileid)
# jj = np.flatnonzero(allccds.expnum == e)
# print(' In DESI:', o.in_desi, 'In DES:', o.in_des)
# print(' ', len(jj), 'matching CCDs')
# if len(jj) == 0:
# continue
# print(' CCDs OBJECT', [ob.strip() for ob in allccds.object[jj]])
# print(' CCDs Tileid', allccds.tileid[jj])
# print(' CCDs galdepth', allccds.galdepth[jj])
# print(' CCDs photometric', allccds.photometric[jj])
#
# ii = np.flatnonzero(E.expnum == e)
# print(' ', len(ii), 'Exposures matching')
# if len(ii):
# ee = E[ii[0]]
# print(' exposure tileid', ee.tileid)
# print(' index', tileid_to_index[ee.tileid])
# print(' vs i2=', i2)
# print(' only_nonphot', only_nonphot[i2], 'missing_depth', missing_depth[i2])
#
# kk = np.flatnonzero(allccds.tileid == o.tileid)
# kk = np.array(sorted(set(kk) - set(jj)))
# print(' ', len(kk), 'other CCDs of this tile')
# #print('Dates:', O.get('%s_date' % band)[I])
from astrometry.util.plotutils import antigray
rr, dd = np.meshgrid(np.linspace(rlo, rhi, 720),
np.linspace(dlo, dhi, 360))
JJ, II, d = match_radec(rr.ravel(),
dd.ravel(),
O.ra,
O.dec,
1.5,
nearest=True)
indesi = np.zeros(rr.shape, bool)
indesi.flat[JJ] = ((O.in_desi[II] == 1) * (O.in_des[II] == 0))
plt.figure(figsize=(14, 6))
plt.subplots_adjust(left=0.1, right=0.99)
for passnum in [1, 2, 3]:
print('Pass', passnum)
plt.clf()
J = np.flatnonzero(
(O.in_desi == 1) * (O.in_des == 0) * (O.dec > dlo) *
(O.dec < dhi) * (O.get('pass') == passnum))
#plt.plot(O.ra[J], O.dec[J], 'k.', alpha=0.5)
# Plot the gray background showing the in_desi footprint
plt.imshow(indesi,
extent=[rlo, rhi, dlo, dhi],
vmin=0,
vmax=4,
cmap=antigray,
aspect='auto',
interpolation='nearest',
origin='lower')
depth = O.get('%s_depth' % band)
#J = np.flatnonzero((O.get('pass') == passnum) * (depth > 0))
J = np.flatnonzero(
(O.get('pass') == passnum) * (depth > 1) * (depth < 30))
# print('Depths:', depth[J])
pct = np.percentile(depth[J],
[0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100])
#print('Depth deciles:', np.percentile(depth[J], [0,10,20,30,40,50,60,70,80,90,100]))
print('Depth deciles: [',
', '.join(['%.3f' % f for f in pct]) + ']')
if len(J) == 0:
sys.exit(0)
target = fid.single_exposure_depth
print('Target depth:', target)
cmap = cmap_discretize('RdBu', 11)
dm = 0.275
plt.scatter(O.ra[J],
O.dec[J],
c=depth[J] - target,
linewidths=0,
cmap=cmap,
vmin=-dm,
vmax=+dm,
zorder=-10,
s=1)
plt.colorbar(ticks=np.arange(-0.25, 0.251, 0.05))
hh, ww = rr.shape
rgba = np.zeros((hh, ww, 4), np.float32)
JJ, II, d = match_radec(rr.ravel(),
dd.ravel(),
O.ra[J],
O.dec[J],
1.,
nearest=True)
Jy, Jx = np.unravel_index(JJ, rr.shape)
rgba[Jy, Jx, :] = cmap((np.clip(depth[J[II]] - target, -dm, dm) -
(-dm)) / (dm - (-dm)))
plt.imshow(rgba,
extent=[rlo, rhi, dlo, dhi],
aspect='auto',
interpolation='nearest',
origin='lower')
I = np.flatnonzero((depth == 0) * (O.get('%s_done' % band) == 1) *
(O.get('pass') == passnum))
plt.plot(O.ra[I], O.dec[I], 'g.')
plt.title('Band %s, Pass %i' % (band, passnum))
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.axis([rhi, rlo, dlo, dhi])
plt.savefig('depth-%s-%i.png' % (band, passnum))
plt.clf()
print('Fiducial single-exposure-depth:', fid.single_exposure_depth)
for passnum in [1, 2, 3]:
depth = O.get('%s_depth' % band)
J = np.flatnonzero(
(O.get('pass') == passnum) * (depth > 1) * (depth < 30))
depth = depth[J]
print('Pass', passnum)
print(sum(depth < fid.single_exposure_depth - 0.25), 'of',
len(depth), 'tiles are more than 0.25 mag shallow')
odepth = O.get('%s_depth' % band)
K = np.flatnonzero(
(O.get('%s_done' % band) == 0) * (O.get('pass') == passnum) *
(odepth > 1) * (odepth < 30))
print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
len(odepth[K]),
'DONE=0 tiles are more than 0.25 mag shallow')
for k in K:
print(' EXPNUM',
O.get('%s_expnum' % band)[k], 'DATE',
O.get('%s_date' % band)[k], 'DEPTH',
O.get('%s_depth' % band)[k])
K = np.flatnonzero(
(O.get('%s_done' % band) == 1) * (O.get('pass') == passnum) *
(odepth > 1) * (odepth < 30))
print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
len(odepth[K]),
'DONE=1 tiles are more than 0.25 mag shallow')
K = np.flatnonzero((O.get('%s_done' % band) == 1) *
(O.get('pass') == passnum) * (odepth == 1))
print(len(K), 'DONE=1 tiles have DEPTH=1 (non-photometric)')
K = np.flatnonzero((O.get('%s_done' % band) == 1) *
(O.get('pass') == passnum) * (odepth == 30))
print(len(K), 'DONE=1 tiles have DEPTH=30 (unknown depth)')
K = np.flatnonzero((O.get('%s_done' % band) == 1) *
(O.get('pass') == passnum) * (odepth == 0))
print(len(K), 'DONE=1 tiles have DEPTH=0')
K = np.flatnonzero((O.get('%s_done' % band) == 0) *
(O.get('pass') == passnum) * (odepth != 0))
print(len(K), 'tiles have DONE=0 but DEPTH != 0')
mlo, mhi = 21, 24
plt.hist(np.clip(depth, mlo, mhi),
bins=100,
range=(mlo, mhi),
histtype='step',
color=' bgr'[passnum],
label='Pass %i' % passnum)
plt.axvline(fid.single_exposure_depth, color='k')
plt.axvline(fid.single_exposure_depth - 0.25,
color='k',
linestyle='--')
plt.xlabel('Depth (mag)')
plt.legend(loc='upper left')
plt.title('Depth: %s' % band)
plt.savefig('depth-%s.png' % band)
for passnum in [1, 2, 3]:
depth = O.get('%s_depth' % band)
roi = ((O.in_desi == 1) * (O.in_des == 0) * (O.dec > dlo) *
(O.dec < dhi) * (O.get('pass') == passnum))
J = np.flatnonzero(roi)
done = np.flatnonzero(roi * (O.get('%s_done' % band) == 1))
redo = np.flatnonzero(roi * np.logical_or(
(depth > 1) * (depth < 30) *
(depth < fid.single_exposure_depth - 0.25), depth == 1))
print(
'Band %s, pass %i: total tiles %i, done %i, redo %i, keep %i' %
(band, passnum, len(J), len(done), len(redo),
len(done) - len(redo)))
A = np.flatnonzero(roi * (depth > 1) * (depth < 30) *
(depth > fid.single_exposure_depth - 0.25))
B = np.flatnonzero(roi * (depth > 1) * (depth < 30) *
(depth <= fid.single_exposure_depth - 0.25))
C = np.flatnonzero(roi * (depth == 1))
D = np.flatnonzero(roi * (depth == 30))
print(
'Band %s, pass %i: total tiles: %i, A: %i, B: %i, C: %i, D: %i'
% (band, passnum, len(J), len(A), len(B), len(C), len(D)))
plt.clf()
plt.plot(O.ra[J], O.dec[J], 'ko', alpha=0.1)
plt.plot(O.ra[done], O.dec[done], 'k.')
plt.plot(O.ra[redo], O.dec[redo], 'r.')
plt.axis([360, 0, -20, 38])
plt.title('Tiles to redo: %s band, pass %i: %i of %i' %
(band, passnum, len(redo), len(done)))
plt.savefig('redo-%s-%i.png' % (band, passnum))
if band == 'z':
redo = np.flatnonzero(
(O.get('pass') == passnum) * np.logical_or(
(depth > 1) * (depth < 30) *
(depth < fid.single_exposure_depth - 0.5), depth == 1))
A = np.flatnonzero(roi * (depth > 1) * (depth < 30) *
(depth > fid.single_exposure_depth - 0.5))
B = np.flatnonzero(roi * (depth > 1) * (depth < 30) *
(depth <= fid.single_exposure_depth - 0.5))
print(
'Band %s, pass %i: total tiles: %i, A: %i, B: %i, C: %i, D: %i (shallow = 0.5 mag less than target)'
% (band, passnum, len(J), len(A), len(B), len(C), len(D)))
plt.clf()
plt.plot(O.ra[J], O.dec[J], 'ko', alpha=0.1)
plt.plot(O.ra[done], O.dec[done], 'k.')
plt.plot(O.ra[redo], O.dec[redo], 'r.')
plt.axis([360, 0, -20, 38])
plt.title(
'Tiles to redo (> 0.5 mag shallow): %s band, pass %i: %i of %i'
% (band, passnum, len(redo), len(done)))
plt.savefig('redo2-%s-%i.png' % (band, passnum))
print('Passes 1-3 combined:')
depth = O.get('%s_depth' % band)
J = np.flatnonzero((depth > 1) * (depth < 30))
depth = depth[J]
print(sum(depth < fid.single_exposure_depth - 0.25), 'of', len(depth),
'tiles are more than 0.25 mag shallow')
odepth = O.get('%s_depth' % band)
K = np.flatnonzero(
(O.get('%s_done' % band) == 0) * (odepth > 1) * (odepth < 30))
print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
len(odepth[K]), 'DONE=0 tiles are more than 0.25 mag shallow')
for k in K:
print(' EXPNUM',
O.get('%s_expnum' % band)[k], 'DATE',
O.get('%s_date' % band)[k], 'DEPTH',
O.get('%s_depth' % band)[k])
K = np.flatnonzero(
(O.get('%s_done' % band) == 1) * (odepth > 1) * (odepth < 30))
print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
len(odepth[K]), 'DONE=1 tiles are more than 0.25 mag shallow')
K = np.flatnonzero((O.get('%s_done' % band) == 1) * (odepth > 1) *
(odepth < 30) * goodtiles)
print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
len(odepth[K]),
'interesting DONE=1 tiles are more than 0.25 mag shallow')
K = np.flatnonzero((O.get('%s_done' % band) == 1) * (odepth == 1))
print(len(K), 'DONE=1 tiles have DEPTH=1 (non-photometric)')
K = np.flatnonzero(
(O.get('%s_done' % band) == 1) * (odepth == 1) * goodtiles)
print(len(K),
'interesting DONE=1 tiles have DEPTH=1 (non-photometric)')
K = np.flatnonzero((O.get('%s_done' % band) == 1) * (odepth == 30))
print(len(K), 'DONE=1 tiles have DEPTH=30 (unknown depth)')
K = np.flatnonzero(
(O.get('%s_done' % band) == 1) * (odepth == 30) * goodtiles)
print(len(K), 'interesting DONE=1 tiles have DEPTH=30 (unknown depth)')
K = np.flatnonzero((O.get('%s_done' % band) == 1) * (odepth == 0))
print(len(K), 'DONE=1 tiles have DEPTH=0')
K = np.flatnonzero(
(O.get('%s_done' % band) == 1) * (odepth == 0) * goodtiles)
print(len(K), 'interesting DONE=1 tiles have DEPTH=0')
K = np.flatnonzero((O.get('%s_done' % band) == 0) * (odepth != 0))
print(len(K), 'tiles have DONE=0 but DEPTH != 0')
O.writeto(out_fn)
def main(passnum, threads):
global udecs
global P3
global T
global tilewcs
global exps
global bad_expids
global tileid_to_depth
ps = PlotSequence('covfill-p%i' % passnum)
retirablefn = 'retirable-p%i.fits' % passnum
depthsfn = 'all-depths-p%i.fits' % passnum
if os.path.exists(retirablefn):
R = fits_table(retirablefn)
pcts = np.arange(0, 101)
target = 22.5
req_pcts = [0, 2, 2, 5, 5, 10, 10, 100]
req_depths = [0, 0, target-0.6, target-0.6,
target-0.3, target-0.3, target, target]
maglo, maghi = 21,23
plt.clf()
for depths in R.depths:
plt.plot(pcts, np.clip(depths, maglo, maghi), 'b-',
alpha=0.1)
plt.plot(req_pcts, np.clip(req_depths, maglo, maghi), 'k-',
lw=2, alpha=0.5)
plt.ylim(maglo, maghi)
plt.xlim(0, 100)
plt.xlabel('Coverage fraction')
plt.ylabel('Existing depth')
plt.suptitle('MzLS: retirable pass-%i tiles: %i' % (passnum,len(R)))
ps.savefig()
# Where are they on the sky?
T = fits_table('obstatus/mosaic-tiles_obstatus.fits')
T.cut(T.in_desi == 1)
T.cut(T.get('pass') <= 3)
tileid_to_index = np.zeros(T.tileid.max()+1, int)
tileid_to_index[T.tileid] = np.arange(len(T))
R.ra = T.ra [tileid_to_index[R.tileid]]
R.dec = T.dec[tileid_to_index[R.tileid]]
plt.clf()
plt.plot(T.ra, T.dec, 'k.', alpha=0.02)
I = (T.z_done == 1)
plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.1)
plt.plot(R.ra, R.dec, 'b.')
ax = [310,80,30,85]
#xl,xh = plt.xlim()
#plt.xlim(xh,xl)
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.title('MzLS: retirable pass-%i tiles' % passnum)
plt.axis(ax)
ps.savefig()
for p in [1,2,3]:
plt.clf()
plt.plot(T.ra, T.dec, 'k.', alpha=0.02)
#plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.1)
I = np.flatnonzero((T.get('pass') == p) * (T.z_done == 1)
* (T.z_depth > 1) * (T.z_depth < 30))
plt.scatter(T.ra[I], T.dec[I], c=T.z_depth[I],
vmin=20, vmax=23, s=4)
I = np.flatnonzero((T.get('pass') == p) * (T.z_done == 1)
* (T.z_depth == 30))
plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.5)
plt.colorbar()
plt.title('MzLS: Finished tiles in pass %i' % p)
plt.axis(ax)
ps.savefig()
sys.exit(0)
# NERSC: export LEGACY_SURVEY_DIR=/global/cscratch1/sd/dstn/dr4plus
# (dstn laptop: export LEGACY_SURVEY_DIR=~/legacypipe-dir-mzls/)
survey = LegacySurveyData()
ccds = survey.get_annotated_ccds()
print('Annotated CCDs:', len(ccds))
ccds.cut(ccds.camera == 'mosaic')
print(len(ccds), 'Mosaic')
print('Unique exposures:', len(np.unique(ccds.expnum)))
ccds.cut(ccds.exptime > 60)
print('Exptime > 60 sec:', len(ccds))
nccds = Counter(ccds.expnum)
for k,v in nccds.most_common():
if v <= 4:
break
print('Expnum', k, 'appears', v, 'times')
print('Tile pass numbers:', Counter(ccds.tilepass).most_common())
# Fix parsing of OBJECT field to tileid...
from obsbot import get_tile_id_from_name
tileids = []
for o in ccds.object:
tid = get_tile_id_from_name(o.strip())
if tid is None:
tid = 0
tileids.append(tid)
tileids = np.array(tileids)
print(len(np.unique(tileids)), 'unique tile ids in annotated file, from OBJECT')
print(len(np.unique(ccds.tileid)), 'unique tile ids in ann file from TILEID')
D = np.flatnonzero(tileids != ccds.tileid)
print(len(D), 'different tileids')
print('From OBJECT:', tileids[D])
print('From TILEID:', ccds.tileid[D])
ccds.tileid = tileids
T = fits_table('obstatus/mosaic-tiles_obstatus.fits')
f = open('obstatus/bad_expid.txt')
bad_expids = []
for line in f:
line = line.strip()
if len(line) == 0:
continue
if line[0] == '#':
continue
words = line.split()
try:
expnum = int(words[0])
except:
print('Skipping line:', line)
continue
bad_expids.append(expnum)
print('Read', len(bad_expids), 'bad exposure numbers')
# Update ccds.tilepass from ccds.tileid
tileidtopass = dict(zip(T.tileid, T.get('pass')))
tileidtoebv = dict(zip(T.tileid, T.ebv_med))
ccds.tilepass = np.array([tileidtopass.get(tileid, 0)
for tileid in ccds.tileid])
ccds.tileebv = np.array([tileidtoebv.get(tileid, 0)
for tileid in ccds.tileid])
print('Tile pass numbers after update:', Counter(ccds.tilepass).most_common())
e,I = np.unique(ccds.expnum, return_index=True)
exps = ccds[I]
#print('Object names,exptimes for tilepass==0:', zip(exps.object[exps.tilepass == 0], exps.exptime[exps.tilepass == 0]))
# Average the depth per exposure
for j,expnum in enumerate(exps.expnum):
I = np.flatnonzero(ccds.expnum == expnum)
if len(I) != 4:
print('Exposure', expnum, 'has', len(I), 'CCD entries')
continue
# Don't include zeros in computing average depths!
Igood = I[(ccds.galdepth[I] > 0) * (ccds.ccdzpt[I] < 30)]
if len(Igood) > 0:
exps.galdepth[j] = np.mean(ccds.galdepth[Igood])
else:
exps.galdepth[j] = 0.
# CCDs-table-based mapping from tileid to depth.
I = np.flatnonzero((exps.tilepass > 0) * (exps.galdepth > 0) *
(exps.tileid > 0))
tileid_to_depth = dict(zip(exps.tileid[I], exps.galdepth[I]))
T.cut(T.in_desi == 1)
T.cut(T.get('pass') <= 3)
# The tiles we'll examine
P3 = T[T.get('pass') == passnum]
print(len(P3), 'pass', passnum, 'and in DESI')
todo = P3[P3.z_done == 0]
print(len(todo), 'pass', passnum, 'tiles to do (Z_DONE=0)')
# Tiles with measured depths
T.cut((T.z_depth > 15) * (T.z_depth < 30))
print(len(T), 'tiles with measured depths')
# Passes other than 3... they ~ only barely overlap, so don't
# contribute significant depth.
#T.cut(T.get('pass') < 3)
udecs = np.unique(P3.dec)
print(len(udecs), 'unique Dec values in pass', passnum)
# Grab an arbitrary weight-map image and use that as a proxy!
wtfn = 'k4m_170501_112501_oow_zd_v1.fits.fz'
F = fitsio.FITS(wtfn)
tilewcs = []
# Read the WCS headers for each chip.
# They make the CRVAL be the boresight for all 4 chips... perfect!
# (because this means we can just set CRVAL = RA,Dec to shift the WCSes)
for i in range(1, len(F)):
hdr = F[i].read_header()
wcs = wcs_pv2sip_hdr(hdr)
tilewcs.append(wcs)
mp = multiproc(threads)
#args = [(i,t,udecs,P3,T,tilewcs,exps,bad_expids,tileid_to_depth)
args = [(i,t)
for i,t in enumerate(todo)]
thedepths = mp.map(one_tile, args)
alldepths = []
retirable = []
for arg,depths in zip(args, thedepths):
if depths is None:
continue
itile,tile = arg[:2]
target = 22.5
req_pcts = [0, 2, 2, 5, 5, 10, 10, 100]
req_depths = [0, 0, target-0.6, target-0.6,
target-0.3, target-0.3, target, target]
print(' Depths at 2, 5, and 10th percentile vs target:',
'%.2f' % (depths[2] - (target - 0.6)),
'%.2f' % (depths[5] - (target - 0.3)),
'%.2f' % (depths[10] - target))
alldepths.append((tile.tileid, depths))
if not ((depths[2] > target - 0.6) and
(depths[5] > target - 0.3) and
(depths[10] > target)):
continue
retirable.append((tile.tileid, depths))
#if len(retirable) == 10:
# break
# if ps.ploti >= 100:
# continue
#
# maglo, maghi = 21,23
#
# plt.clf()
# #plt.subplot(1,2,1)
# plt.subplot2grid((2,2), (0,0))
# plt.imshow(depth, interpolation='nearest', origin='lower',
# vmin=maglo, vmax=maghi)
# plt.colorbar(ticks=[np.arange(maglo, maghi+0.01, 0.5)])
# plt.xticks([]); plt.yticks([])
# #plt.subplot(1,2,2)
# plt.subplot2grid((2,2), (1,0))
# plt.imshow(nexp, interpolation='nearest', origin='lower',
# vmin=0, vmax=4)
# plt.colorbar(ticks=[0,1,2,3,4])
# plt.xticks([]); plt.yticks([])
#
# ax = plt.subplot2grid((2,2), (0,1), rowspan=2)
# plt.plot(req_pcts, np.clip(req_depths, maglo, maghi), 'k-',
# lw=2, alpha=0.5)
# plt.plot(pcts, np.clip(depths, maglo, maghi), 'b-')
# ax.yaxis.tick_right()
# plt.ylim(maglo, maghi)
# plt.xlim(0, 100)
# plt.xlabel('Coverage fraction')
# plt.ylabel('Existing depth')
# plt.suptitle('Tile %i' % tile.tileid)
# ps.savefig()
#if ps.ploti == 100:
# break
# print('Tiles that could be retired:')
# print('# Tileid 0th-percentile-extcorr-depth 1st-pctile 2nd-pctile ...')
# for tileid, depths in retirable:
# print(tileid, ' '.join(['%.3f' % d for d in depths]))
R = fits_table()
R.tileid = np.array ([t for t,d in alldepths])
R.depths = np.vstack([d for t,d in alldepths])
R.writeto(depthsfn)
if len(retirable):
R = fits_table()
R.tileid = np.array([t for t,d in retirable])
R.depths = np.vstack([d for t,d in retirable])
R.writeto(retirablefn)
else:
print('No tiles in pass', passnum, 'are retirable')
I first ran this code for each brick:
legacyanalysis/depth-cut.py
--> generates depthcut/*/ccds-*.fits tables of CCDs that pass the depth cut for that brick
legacyanalysis/check-depth-cut.py
--> to read the per-brick ccds-* tables and cut to the union of all CCDs that pass depth cut in some brick -> depth-cut-kept-ccds.fits
legacyanalysis/dr5-cut-ccds.py
--> to read depth-cut-kept-ccds.fits and cut the (already-created) annotated-ccds table and create the .kd.fits version of the CCDs table
'''
survey = LegacySurveyData()
# Read *old* annotated-CCDs tables.
ann = survey.get_annotated_ccds()
print('Got', len(ann), 'annotated CCDs')
ann.about()
# build mapping from expnum,ccdname to index in ann table.
annmap = dict([((e, c.strip()), i)
for i, (e, c) in enumerate(zip(ann.expnum, ann.ccdname))])
# Read the *new* zeropoints file.
ccds = fits_table(
'/global/cscratch1/sd/kaylanb/dr5_zpts/survey-ccds-legacypipe-hdufix-45455-nocuts.fits.gz'
)
print('Read', len(ccds), 'CCDs')
print('Cameras:', np.unique(ccds.camera))
def setup():
survey = LegacySurveyData()
ccds = survey.get_annotated_ccds()
return ccds
