def ccd_cuts_inplace(ccds, use_blacklist=True):
# coadd/ccd.fits file has extra white space
ccds.camera = np.char.strip(ccds.camera)
bands = ['g', 'r', 'z']
# Continue like runbrick
from legacypipe.runs import get_survey
survey = get_survey('dr4v2',
survey_dir=os.getenv('LEGACY_SURVEY_DIR'),
output_dir='./')
if use_blacklist:
I = survey.apply_blacklist(ccds)
ccds.cut(I)
# Sort images by band -- this also eliminates images whose
# *filter* string is not in *bands*.
ccds.cut(np.hstack([np.flatnonzero(ccds.filter == band)
for band in bands]))
I = survey.photometric_ccds(ccds)
ccds.cut(I)
I = survey.bad_exposures(ccds)
ccds.cut(I)
I = survey.other_bad_things(ccds)
ccds.cut(I)
I = survey.ccds_for_fitting(brick, ccds)
if I is not None:
ccds.cut(I)
from astrometry.util.fits import fits_table
from astrometry.util.plotutils import PlotSequence
from astrometry.util.multiproc import multiproc
from legacypipe.survey import *
from legacypipe.runs import get_survey
import photutils
import sys
import os
if __name__ == '__main__':
ps = PlotSequence('psfnorm')
survey = get_survey('dr4v2')
ccds = survey.get_ccds_readonly()
print(len(ccds), 'CCDs')
ccds = ccds[ccds.camera == '90prime']
print(len(ccds), 'Bok CCDs')
## Skip a few missing ones...
ccds = ccds[24:]
for i, ccd in enumerate(ccds):
print('Reading CCD', i)
try:
im = survey.get_image_object(ccd, makeNewWeightMap=False)
print('Reading', im)
psf = im.read_psf_model(0,
0,
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'-b',
'--brick',
help='Brick name to run; required unless --radec is given')
parser.add_argument(
'--survey-dir',
type=str,
default=None,
help='Override the $LEGACY_SURVEY_DIR environment variable')
parser.add_argument('-d',
'--outdir',
dest='output_dir',
help='Set output base directory, default "."')
parser.add_argument(
'--out',
help='Output filename -- if not set, defaults to path within --outdir.'
)
parser.add_argument('-r',
'--run',
default=None,
help='Set the run type to execute (for images)')
parser.add_argument(
'--catalog',
help=
'Use the given FITS catalog file, rather than reading from a data release directory'
)
parser.add_argument('--catalog-dir',
help='Set LEGACY_SURVEY_DIR to use to read catalogs')
parser.add_argument(
'--catalog-dir-north',
help='Set LEGACY_SURVEY_DIR to use to read Northern catalogs')
parser.add_argument(
'--catalog-dir-south',
help='Set LEGACY_SURVEY_DIR to use to read Southern catalogs')
parser.add_argument(
'--catalog-resolve-dec-ngc',
type=float,
help=
'Dec at which to switch from Northern to Southern catalogs (NGC only)',
default=32.375)
parser.add_argument('-v',
'--verbose',
dest='verbose',
action='count',
default=0,
help='Make more verbose')
opt = parser.parse_args()
if opt.brick is None:
parser.print_help()
return -1
verbose = opt.verbose
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
from legacypipe.runs import get_survey
survey = get_survey(opt.run,
survey_dir=opt.survey_dir,
output_dir=opt.output_dir)
columns = [
'release',
'brickid',
'objid',
]
cat = None
catsurvey = survey
if opt.catalog is not None:
cat = fits_table(opt.catalog, columns=columns)
print('Read', len(cat), 'sources from', opt.catalog)
else:
from astrometry.util.starutil_numpy import radectolb
# The "north" and "south" directories often don't have
# 'survey-bricks" files of their own -- use the 'survey' one
# instead.
brick = None
for s in [survey, catsurvey]:
try:
brick = s.get_brick_by_name(opt.brick)
break
except:
import traceback
traceback.print_exc()
pass
l, b = radectolb(brick.ra, brick.dec)
# NGC and above resolve line? -> north
if b > 0 and brick.dec >= opt.catalog_resolve_dec_ngc:
if opt.catalog_dir_north:
catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir_north)
else:
if opt.catalog_dir_south:
catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir_south)
fn = catsurvey.find_file('tractor', brick=opt.brick)
cat = fits_table(fn, columns=columns)
print('Read', len(cat), 'sources from', fn)
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 9999
version_hdr = get_version_header(program_name, opt.survey_dir, release)
from legacypipe.utils import add_bits
from legacypipe.bits import DQ_BITS
add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
for i, ap in enumerate(apertures_arcsec):
version_hdr.add_record(
dict(name='APRAD%i' % i,
value=ap,
comment='(optical) Aperture radius, in arcsec'))
cat, forced = merge_forced(survey, opt.brick, cat)
units = []
for i, col in enumerate(forced.get_columns()):
units.append(forced._header.get('TUNIT%i' % (i + 1), ''))
cols = forced.get_columns()
if opt.out:
cat.writeto(opt.out, primheader=version_hdr)
forced.writeto(opt.out, append=True, units=units, columns=cols)
else:
with survey.write_output('forced-brick', brick=opt.brick) as out:
cat.writeto(None, fits_object=out.fits, primheader=version_hdr)
forced.writeto(None,
fits_object=out.fits,
append=True,
units=units,
columns=cols)
def main():
global survey
init()
ps = PlotSequence('sky')
# export LEGACY_SURVEY_DIR=/scratch1/scratchdirs/desiproc/DRs/dr4-bootes/legacypipe-dir/
#survey = LegacySurveyData()
survey = get_survey('dr4v2')
ccds = survey.get_ccds_readonly()
print(len(ccds), 'CCDs')
ccds = ccds[ccds.camera == 'mosaic']
print(len(ccds), 'Mosaic CCDs')
# plt.clf()
# plt.hist(ccds.mjd_obs % 1.0, bins=50)
# plt.xlabel('MJD mod 1')
# ps.savefig()
ccds.imjd = np.floor(ccds.mjd_obs).astype(int)
mjds = np.unique(ccds.imjd)
print(len(mjds), 'unique MJDs')
mp = multiproc(nthreads=8, init=init)
allvals = []
medmjds = []
args = []
for kk, mjd in enumerate(mjds):
I = np.flatnonzero(ccds.imjd == mjd)
print('MJD', mjd, ' (%i of %i):' % (kk + 1, len(mjds)), len(I), 'CCDs')
if len(I) == 0:
continue
# pick one near the middle
#i = I[len(I)/2]
for i in [I[len(I) / 4], I[len(I) / 2], I[3 * len(I) / 4]]:
ccd = ccds[i]
key = dict(expnum=ccd.expnum, ccdname=ccd.ccdname)
oldvals = key
vals = cache.find(key)
print('Got', vals.count(), 'cache hits for', key)
gotone = False
for val in vals:
#if 'median_adu' in val:
if 'mode_adu' in val:
print('cache hit:', val)
allvals.append(val)
medmjds.append(mjd)
gotone = True
break
else:
print('partial cache hit:', val)
oldvals = val
###!
cache.delete_one(val)
if gotone:
continue
print('args: key', key, 'oldvals', oldvals)
args.append((mjd, key, oldvals, ccd))
# plt.clf()
# plt.hist(tim.getImage().ravel(), range=(-0.1, 0.1), bins=50,
# histtype='step', color='b')
# plt.axvline(med, color='b', alpha=0.3, lw=2)
# plt.axvline(0., color='k', alpha=0.3, lw=2)
# plt.xlabel('Sky-subtracted pixel values')
# plt.title('Date ' + ccd.date_obs + ': ' + str(im))
# ps.savefig()
if len(args):
meds = mp.map(read_sky_val, args)
print('Medians:', meds)
for (mjd, key, oldvals, ccd), val in zip(args, meds):
if val is None:
continue
allvals.append(val)
medmjds.append(mjd)
medians = []
median_adus = []
mode_adus = []
skyadus = []
keepmjds = []
for mjd, val in zip(medmjds, allvals):
madu = val['median_adu']
if madu is None:
continue
if 'median' in val:
medians.append(val['median'])
else:
from tractor.brightness import NanoMaggies
zpscale = NanoMaggies.zeropointToScale(val['ccdzpt'])
med = (val['median_adu'] - val['skyadu']) / zpscale
print('Computed median diff:', med, 'nmgy')
medians.append(med)
keepmjds.append(mjd)
median_adus.append(val['median_adu'])
mode_adus.append(val['mode_adu'])
skyadus.append(val['skyadu'])
medmjds = keepmjds
median_adus = np.array(median_adus)
mode_adus = np.array(mode_adus)
skyadus = np.array(skyadus)
medmjds = np.array(medmjds)
medians = np.array(medians)
plt.clf()
plt.plot(medmjds, median_adus - skyadus, 'b.')
plt.xlabel('MJD')
#plt.ylabel('Image median after sky subtraction (nmgy)')
plt.ylabel('Image median - SKYADU (ADU)')
#plt.ylim(-0.03, 0.03)
#plt.ylim(-10, 10)
plt.ylim(-1, 1)
plt.axhline(0, color='k', lw=2, alpha=0.5)
ps.savefig()
plt.clf()
plt.plot(medmjds, mode_adus - skyadus, 'b.')
plt.xlabel('MJD')
plt.ylabel('Image mode - SKYADU (ADU)')
plt.ylim(-1, 1)
plt.axhline(0, color='k', lw=2, alpha=0.5)
ps.savefig()
print('Median pcts:', np.percentile(medians, [0, 2, 50, 98, 100]))
mlo, mhi = np.percentile(medians, [2, 98])
I = np.flatnonzero((medians > mlo) * (medians < mhi))
d0 = np.mean(medmjds)
dmjd = medmjds[I] - d0
A = np.zeros((len(dmjd), 2))
A[:, 0] = 1.
A[:, 1] = dmjd
b = np.linalg.lstsq(A, medians[I])[0]
print('Lstsq:', b)
offset = b[0]
slope = b[1]
xx = np.array([dmjd.min(), dmjd.max()])
print('Offset', offset)
print('Slope', slope)
plt.clf()
plt.plot(medmjds, medians, 'b.')
ax = plt.axis()
plt.plot(xx + d0, offset + slope * xx, 'b-')
plt.axis(ax)
plt.xlabel('MJD')
plt.ylabel('Image median after sky subtraction (nmgy)')
plt.ylim(-0.03, 0.03)
plt.axhline(0, color='k', lw=2, alpha=0.5)
ps.savefig()
plt.clf()
plt.plot(median_adus, skyadus, 'b.')
ax = plt.axis()
lo = min(ax[0], ax[2])
hi = max(ax[1], ax[3])
plt.plot([lo, hi], [lo, hi], 'k-', alpha=0.5)
plt.xlabel('Median image ADU')
plt.ylabel('SKYADU')
plt.axis(ax)
ps.savefig()
plt.clf()
plt.plot(medmjds, skyadus / median_adus, 'b.')
plt.xlabel('MJD')
plt.ylim(0.98, 1.02)
plt.axhline(1.0, color='k', alpha=0.5)
plt.ylabel('SKYADU / median image ADU')
ps.savefig()
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
region = opt.region
run = opt.run
# We cache the table of good CCDs in the region of interest in this file...
cfn = '%s-all-ccds.fits' % region
if not os.path.exists(cfn):
from legacypipe.runs import get_survey
import legacypipe.runs
print(legacypipe.runs.__file__)
#survey = LegacySurveyData()
survey = get_survey(run)
C = survey.get_annotated_ccds()
print(len(C), 'annotated CCDs')
if region == 'cosmos':
#C.cut(np.hypot(C.ra_bore - 150.1, C.dec_bore - 2.2) < 1.)
C.cut(np.hypot(C.ra_bore - 150.1, C.dec_bore - 2.2) < 0.3)
elif region == 's82qso':
#C.cut((C.ra > 35.75) * (C.ra < 42.25) * (C.dec > -1.5) * (C.dec < 1.5))
C.cut((C.ra > 36) * (C.ra < 36.5) * (C.dec > 0) * (C.dec < 0.5))
else:
assert (False)
print(len(C), 'CCDs in', region)