def rbmain():
travis = 'travis' in sys.argv
extra_args = [
'--old-calibs-ok',
#'--verbose',
]
if travis:
extra_args.extend(
['--no-wise-ceres', '--no-gaia', '--no-large-galaxies'])
if 'ceres' in sys.argv:
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--ceres',
'--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres',
'--no-depth-cut'
])
sys.exit(0)
# demo RexGalaxy, with plots
if False:
from legacypipe.survey import RexGalaxy
from tractor import NanoMaggies, PixPos
from tractor import Image, GaussianMixturePSF, LinearPhotoCal
from legacypipe.survey import LogRadius
rex = RexGalaxy(PixPos(1., 2.), NanoMaggies(r=3.), LogRadius(0.))
print('Rex:', rex)
print('Rex params:', rex.getParams())
print('Rex nparams:', rex.numberOfParams())
H, W = 100, 100
tim = Image(data=np.zeros((H, W), np.float32),
inverr=np.ones((H, W), np.float32),
psf=GaussianMixturePSF(1., 0., 0., 4., 4., 0.),
photocal=LinearPhotoCal(1., band='r'))
derivs = rex.getParamDerivatives(tim)
print('Derivs:', len(derivs))
print('Rex params:', rex.getParamNames())
import pylab as plt
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('rex')
for d, nm in zip(derivs, rex.getParamNames()):
plt.clf()
plt.imshow(d.patch, interpolation='nearest', origin='lower')
plt.title('Derivative %s' % nm)
ps.savefig()
sys.exit(0)
# Test RexGalaxy
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6-rex'
main(args=[
'--brick',
'1102p240',
'--zoom',
'500',
'600',
'650',
'750',
'--force-all',
'--no-write',
'--no-wise',
#'--rex', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + extra_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0] == 'PSF ')
cmd = (
'(cd %s && sha256sum -c %s)' %
(outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
print(cmd)
rtn = os.system(cmd)
assert (rtn == 0)
# Test with a Tycho-2 star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir,
'--outdir', outdir
] + extra_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0] == 'PSF ')
# Test that we can run splinesky calib if required...
from legacypipe.decam import DecamImage
DecamImage.splinesky_boxsize = 128
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4'
fn = os.path.join(surveydir, 'calib', 'decam', 'splinesky', '00431',
'00431608', 'decam-00431608-N3.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image',
'--no-hybrid-psf', '--survey-dir', surveydir, '--outdir', outdir
] + extra_args + ['-v'])
print('Checking for calib file', fn)
assert (os.path.exists(fn))
# Wrap-around, hybrid PSF
surveydir = os.path.join(os.path.dirname(__file__), 'testcase8')
outdir = 'out-testcase8'
main(args=[
'--brick',
'1209p050',
'--zoom',
'720',
'1095',
'3220',
'3500',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + extra_args)
# Test with a Tycho-2 star + another saturated star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase7')
outdir = 'out-testcase7'
# remove --no-gaia
my_extra_args = [a for a in extra_args if a != '--no-gaia']
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia-dr2')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick',
'1102p240',
'--zoom',
'250',
'350',
'1550',
'1650',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + my_extra_args)
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 4)
# Check skipping blobs outside the brick's unique area.
# (this now doesn't detect any sources at all, reasonably)
# surveydir = os.path.join(os.path.dirname(__file__), 'testcase5')
# outdir = 'out-testcase5'
#
# fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits')
# if os.path.exists(fn):
# os.unlink(fn)
#
# main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150',
# '--force-all', '--no-write', '--coadd-bw',
# '--survey-dir', surveydir,
# '--early-coadds',
# '--outdir', outdir] + extra_args)
#
# assert(os.path.exists(fn))
# T = fits_table(fn)
# assert(len(T) == 1)
# Custom RA,Dec; blob ra,dec.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4b'
# Catalog written with one entry (--blobradec)
fn = os.path.join(outdir, 'tractor', 'cus',
'tractor-custom-186743p25461.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--radec', '186.743965', '25.461788', '--width', '250', '--height',
'250', '--force-all', '--no-write', '--no-wise', '--blobradec',
'186.740369', '25.453855', '--survey-dir', surveydir, '--outdir',
outdir
] + extra_args)
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 1)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
outdir = 'out-testcase3'
checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle')
if os.path.exists(checkpoint_fn):
os.unlink(checkpoint_fn)
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
] + extra_args)
# Read catalog into Tractor sources to test read_fits_catalog
from legacypipe.catalog import read_fits_catalog
from legacypipe.survey import LegacySurveyData, GaiaSource
from tractor.galaxy import DevGalaxy
from tractor import PointSource
survey = LegacySurveyData(survey_dir=outdir)
fn = survey.find_file('tractor', brick='2447p120')
print('Checking', fn)
T = fits_table(fn)
cat = read_fits_catalog(T, fluxPrefix='')
print('Read catalog:', cat)
assert (len(cat) == 2)
src = cat[0]
assert (type(src) == DevGalaxy)
assert (np.abs(src.pos.ra - 244.77973) < 0.00001)
assert (np.abs(src.pos.dec - 12.07233) < 0.00001)
src = cat[1]
print('Source', src)
assert (type(src) in [PointSource, GaiaSource])
assert (np.abs(src.pos.ra - 244.77830) < 0.00001)
assert (np.abs(src.pos.dec - 12.07250) < 0.00001)
# DevGalaxy(pos=RaDecPos[244.77975494973529, 12.072348111713127], brightness=NanoMaggies: g=19.2, r=17.9, z=17.1, shape=re=2.09234, e1=-0.198453, e2=0.023652,
# PointSource(RaDecPos[244.77833280764278, 12.072521274981987], NanoMaggies: g=25, r=23, z=21.7)
# Check that we can run again, using that checkpoint file.
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
] + extra_args)
# Assert...... something?
# Test --checkpoint without --threads
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1'
] + extra_args)
# MzLS + BASS data
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass')
# main(args=['--brick', '3521p002', '--zoom', '2400', '2450', '1200', '1250',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass'])
# From Kaylan's Bootes pre-DR4 run
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3')
# main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass3'] + extra_args)
# With plots!
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', 'out-testcase3', '--plots', '--nblobs', '1'
] + extra_args)
# Decals Image Simulations
# Uncomment WHEN galsim build for Travis
#os.environ["DECALS_SIM_DIR"]= os.path.join(os.path.dirname(__file__),'image_sims')
#brick= '2447p120'
#sim_main(args=['--brick', brick, '-n', '2', '-o', 'STAR', \
# '-ic', '1', '--rmag-range', '18', '26', '--threads', '1',\
# '--zoom', '1020', '1070', '2775', '2815'])
# Check if correct files written out
#rt_dir= os.path.join(os.getenv('DECALS_SIM_DIR'),brick,'star','001')
#assert( os.path.exists(os.path.join(rt_dir,'../','metacat-'+brick+'-star.fits')) )
#for fn in ['tractor-%s-star-01.fits' % brick,'simcat-%s-star-01.fits' % brick]:
# assert( os.path.exists(os.path.join(rt_dir,fn)) )
#for fn in ['image','model','resid','simscoadd']:
# assert( os.path.exists(os.path.join(rt_dir,'qa-'+brick+'-star-'+fn+'-01.jpg')) )
if not travis:
# With ceres
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--ceres',
'--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres'
] + extra_args)
def forced2():
from bigboss_test import radecroi
ps = PlotSequence('forced')
basedir = os.environ.get('BIGBOSS_DATA', '/project/projectdirs/bigboss')
wisedatadir = os.path.join(basedir, 'data', 'wise')
l1bdir = os.path.join(wisedatadir, 'level1b')
wisecat = fits_table(os.path.join(
wisedatadir, 'catalogs', 'wisecat2.fits'))
# CAS PhotoObjAll.resolveStatus bits
sprim = 0x100
#sbad = 0x800
sedge = 0x1000
sbest = 0x200
(ra0, ra1, dec0, dec1) = radecroi
ra = (ra0 + ra1) / 2.
dec = (dec0 + dec1) / 2.
cas = fits_table('sdss-cas-testarea-3.fits')
print('Read', len(cas), 'CAS sources')
cas.cut((cas.resolvestatus & sedge) == 0)
print('Cut to ', len(cas), 'without SURVEY_EDGE set')
# Drop "sbest" sources that have an "sprim" nearby.
Ibest = (cas.resolvestatus & (sprim | sbest)) == sbest
Iprim = (cas.resolvestatus & (sprim | sbest)) == sprim
I, J, d = match_radec(
cas.ra[Ibest], cas.dec[Ibest], cas.ra[Iprim], cas.dec[Iprim], 2. / 3600.)
Ibest[np.flatnonzero(Ibest)[I]] = False
#Ikeep = np.ones(len(Ibest), bool)
#Ikeep[I] = False
cas.cut(np.logical_or(Ibest, Iprim))
print('Cut to', len(cas), 'PRIMARY + BEST-not-near-PRIMARY')
I, J, d = match_radec(cas.ra, cas.dec, cas.ra,
cas.dec, 2. / 3600., notself=True)
plt.clf()
loghist((cas.ra[I] - cas.ra[J]) * 3600.,
(cas.dec[I] - cas.dec[J]) * 3600., 200)
plt.title('CAS self-matches')
ps.savefig()
psf = pyfits.open('wise-psf-w1-500-500.fits')[0].data
S = psf.shape[0]
# number of Gaussian components
K = 3
w, mu, sig = em_init_params(K, None, None, None)
II = psf.copy()
II = np.maximum(II, 0)
II /= II.sum()
xm, ym = -(S / 2), -(S / 2)
res = em_fit_2d(II, xm, ym, w, mu, sig)
if res != 0:
raise RuntimeError('Failed to fit PSF')
print('W1 PSF:')
print(' w', w)
print(' mu', mu)
print(' sigma', sig)
w1psf = GaussianMixturePSF(w, mu, sig)
w1psf.computeRadius()
print('PSF radius:', w1psf.getRadius(), 'pixels')
T = fits_table('wise-roi.fits')
for i in range(len(T)):
basefn = os.path.join(l1bdir, '%s%i-w1' %
(T.scan_id[i], T.frame_num[i]))
fn = basefn + '-int-1b.fits'
print('Looking for', fn)
if not os.path.exists(fn):
continue
print(' -> Found it!')
tim = read_wise_image(basefn, nanomaggies=True)
tim.psf = w1psf
wcs = tim.wcs.wcs
r0, r1, d0, d1 = wcs.radec_bounds()
print('RA,Dec bounds:', r0, r1, d0, d1)
w, h = wcs.imagew, wcs.imageh
rd = np.array([wcs.pixelxy2radec(x, y) for x, y in
[(1, 1), (w, 1), (w, h), (1, h), (1, 1)]])
I = np.flatnonzero((cas.ra > r0) * (cas.ra < r1) *
(cas.dec > d0) * (cas.dec < d1))
J = point_in_poly(cas.ra[I], cas.dec[I], rd)
I = I[J]
cashere = cas[I]
# 10-20k sources...
wbands = ['w1']
sdssband = 'i'
tsrcs = get_tractor_sources_cas_dr9(cashere, nanomaggies=True,
bandname=sdssband, bands=[
sdssband],
extrabands=wbands)
#keepsrcs = []
for src in tsrcs:
for br in src.getBrightnesses():
f = br.getBand(sdssband)
# if f < 0:
# continue
for wb in wbands:
br.setBand(wb, f)
# keepsrcs.append(src)
#tsrcs = keepsrcs
print('Created', len(tsrcs), 'tractor sources in this image')
I = np.flatnonzero((wisecat.ra > r0) * (wisecat.ra < r1) *
(wisecat.dec > d0) * (wisecat.dec < d1))
J = point_in_poly(wisecat.ra[I], wisecat.dec[I], rd)
I = I[J]
print('Found', len(I), 'WISE catalog sources in this image')
wc = wisecat[I]
tra = np.array([src.getPosition().ra for src in tsrcs])
tdec = np.array([src.getPosition().dec for src in tsrcs])
R = 4.
I, J, d = match_radec(wc.ra, wc.dec, tra, tdec,
R / 3600., nearest=True)
# cashere.ra, cashere.dec,
print('Found', len(I), 'SDSS-WISE matches within', R, 'arcsec')
for i, j in zip(I, J):
w1 = wc.w1mpro[i]
w1 = NanoMaggies.magToNanomaggies(w1)
bb = tsrcs[j].getBrightnesses()
for b in bb:
b.setBand('w1', w1 / float(len(bb)))
keepsrcs = []
for src in tsrcs:
# for b in src.getBrightness():
b = src.getBrightness()
if b.getBand(sdssband) > 0 or b.getBand(wbands[0]) > 0:
keepsrcs.append(src)
tsrcs = keepsrcs
print('Keeping', len(tsrcs), 'tractor sources from SDSS')
unmatched = np.ones(len(wc), bool)
unmatched[I] = False
wun = wc[unmatched]
print(len(wun), 'unmatched WISE sources')
for i in range(len(wun)):
pos = RaDecPos(wun.ra[i], wun.dec[i])
nm = NanoMaggies.magToNanomaggies(wun.w1mpro[i])
br = NanoMaggies(i=25., w1=nm)
tsrcs.append(PointSource(pos, br))
plt.clf()
plt.plot(rd[:, 0], rd[:, 1], 'k-')
plt.plot(cashere.ra, cashere.dec, 'r.', alpha=0.1)
plt.plot(wc.ra, wc.dec, 'bx', alpha=0.1)
setRadecAxes(r0, r1, d0, d1)
ps.savefig()
zlo, zhi = tim.zr
ima = dict(interpolation='nearest', origin='lower', vmin=zlo, vmax=zhi)
imchi = dict(interpolation='nearest', origin='lower',
vmin=-5, vmax=5)
plt.clf()
plt.imshow(tim.getImage(), **ima)
plt.hot()
plt.title(tim.name + ': data')
ps.savefig()
wsrcs = []
for i in range(len(wc)):
pos = RaDecPos(wc.ra[i], wc.dec[i])
nm = NanoMaggies.magToNanomaggies(wc.w1mpro[i])
br = NanoMaggies(i=25., w1=nm)
wsrcs.append(PointSource(pos, br))
tr = Tractor([tim], wsrcs)
tr.freezeParam('images')
for jj in range(2):
print('Rendering WISE model image...')
wmod = tr.getModelImage(0)
plt.clf()
plt.imshow(wmod, **ima)
plt.hot()
plt.title(tim.name + ': WISE sources only')
ps.savefig()
assert(np.all(np.isfinite(wmod)))
assert(np.all(np.isfinite(tim.getInvError())))
assert(np.all(np.isfinite(tim.getImage())))
wchi = tr.getChiImage(0)
plt.clf()
plt.imshow(wchi, **imchi)
plt.title(tim.name + ': chi, WISE sources only')
plt.gray()
ps.savefig()
if jj == 1:
break
tr.optimize()
tr = Tractor([tim], tsrcs)
print('Rendering model image...')
mod = tr.getModelImage(0)
plt.clf()
plt.imshow(mod, **ima)
plt.title(tim.name + ': SDSS + WISE sources')
ps.savefig()
print('tim', tim)
print('tim.photocal:', tim.photocal)
wsrcs = []
for i in range(len(wc)):
pos = RaDecPos(wc.ra[i], wc.dec[i])
nm = NanoMaggies.magToNanomaggies(wc.w1mpro[i])
br = NanoMaggies(i=25., w1=nm)
wsrcs.append(PointSource(pos, br))
tr = Tractor([tim], wsrcs)
print('Rendering WISE model image...')
wmod = tr.getModelImage(0)
plt.clf()
plt.imshow(wmod, **ima)
plt.title(tim.name + ': WISE sources only')
ps.savefig()
def main(decals=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for decals.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if decals is None:
decals = Decals()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from decals-ccds.fits table
T = decals.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(np.array([f.strip() == opt.filename for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert(len(T) == 1)
im = decals.get_image_object(T[0])
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True)
print('Got tim:', tim)
if opt.catfn in ['DR1', 'DR2']:
if opt.catalog_path is None:
opt.catalog_path = opt.catfn.lower()
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, decals=decals)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3],
'tractor-%s.fits' % b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
TT.append(T)
T = merge_tables(TT)
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T
T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T
cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE)
# print('Got cat:', cat)
print('Forced photom...')
opti = None
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T))
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap
ap = 1./(np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap
if opt.forced:
kwa = {}
if opt.plots is None:
kwa.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True, fitstats=True,
shared_params=False, **kwa)
if opt.plots:
(data,mod,ie,chi,roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([src.getBrightness().getFlux(tim.band)
for src in cat]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2 .astype(np.float32)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, decals.decals_dir)
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
fname = os.path.join(d2, d1, fname)
print('Trimmed filename to', fname)
#version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file'))
version_hdr.add_record(dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext'))
version_hdr.add_record(dict(name='CAMERA', value='DECam', comment='Dark Energy Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {'mjd':'sec', 'exptime':'sec', 'flux':'nanomaggy',
'flux_ivar':'1/nanomaggy^2'}
columns = F.get_columns()
for i,col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
print('Finished forced phot:', Time()-t0)
return 0
def showSipSolutions(srcs, wcs0, andDir, x0, y0, W, H, filterName, plotPrefix):
'''
srcs: afw Catalog of sources
wcs0: original WCS
andDir: astrometry_net_data directory
'''
imargs = dict(imageSize=(W, H), filterName=filterName, x0=x0, y0=y0)
# Set up astrometry_net_data
os.environ['ASTROMETRY_NET_DATA_DIR'] = andDir
andConfig = measAstrom.AstrometryNetDataConfig()
fn = os.path.join(andDir, 'andConfig.py')
andConfig.load(fn)
# Set up meas_astrom
conf = measAstrom.ANetBasicAstrometryConfig(sipOrder=4)
ast = measAstrom.ANetBasicAstrometryTask(conf,
andConfig,
logLevel=Log.DEBUG)
# What reference sources are in the original WCS
refs = ast.getReferenceSourcesForWcs(wcs0, **imargs)
print('Got', len(refs), 'reference objects for initial WCS')
# How does a straight TAN solution look?
conf2 = measAstrom.ANetBasicAstrometryConfig(sipOrder=4,
calculateSip=False)
ast2 = measAstrom.ANetBasicAstrometryTask(conf2,
andConfig,
logLevel=Log.DEBUG)
solve = ast2.determineWcs2(srcs, **imargs)
tanwcs = solve.tanWcs
# How about if we fit a SIP WCS using the *original* WCS?
wcs2 = ast.getSipWcsFromWcs(wcs0, (W, H), x0=x0, y0=y0)
# (We determineWcs() for a SIP solution below...)
# Make some plots in pixel space by pushing ref sources through WCSes
rx0, ry0 = [], []
rx2, ry2 = [], []
rx3, ry3 = [], []
for src in refs:
xy = wcs0.skyToPixel(src.getCoord())
rx0.append(xy[0])
ry0.append(xy[1])
xy = tanwcs.skyToPixel(src.getCoord())
rx2.append(xy[0])
ry2.append(xy[1])
xy = wcs2.skyToPixel(src.getCoord())
rx3.append(xy[0])
ry3.append(xy[1])
rx0 = np.array(rx0)
ry0 = np.array(ry0)
rx2 = np.array(rx2)
ry2 = np.array(ry2)
rx3 = np.array(rx3)
ry3 = np.array(ry3)
x = np.array([src.getX() for src in srcs])
y = np.array([src.getY() for src in srcs])
from astrometry.libkd.spherematch import match
from astrometry.util.plotutils import plothist, PlotSequence
ps = PlotSequence(plotPrefix)
# Match up various sources...
R = 2.
II, d = match(np.vstack((x, y)).T, np.vstack((rx0, ry0)).T, R)
I = II[:, 0]
J = II[:, 1]
pa = dict(range=((-R, R), (-R, R)))
plt.clf()
plothist(x[I] - rx0[J], y[I] - ry0[J], 200, **pa)
plt.title('Source positions - Reference positions (initial WCS)')
plt.xlabel('delta-X (pixels)')
plt.ylabel('delta-Y (pixels)')
ps.savefig()
II, d = match(np.vstack((x, y)).T, np.vstack((rx2, ry2)).T, R)
I = II[:, 0]
J = II[:, 1]
plt.clf()
plothist(x[I] - rx2[J], y[I] - ry2[J], 200, **pa)
plt.title('Source positions - Reference positions (TAN WCS)')
plt.xlabel('delta-X (pixels)')
plt.ylabel('delta-Y (pixels)')
ps.savefig()
II, d = match(np.vstack((x, y)).T, np.vstack((rx3, ry3)).T, R)
I = II[:, 0]
J = II[:, 1]
plt.clf()
plothist(x[I] - rx3[J], y[I] - ry3[J], 200, **pa)
plt.title('Source positions - Reference positions (SIP WCS #2)')
plt.xlabel('delta-X (pixels)')
plt.ylabel('delta-Y (pixels)')
ps.savefig()
II, d = match(np.vstack((rx0, ry0)).T, np.vstack((rx3, ry3)).T, R)
I = II[:, 0]
J = II[:, 1]
plt.clf()
plothist(rx0[I] - rx3[J], ry0[I] - ry3[J], 200, **pa)
plt.title(
'Reference positions (Original WCS) - Reference positions (SIP WCS #2)'
)
plt.xlabel('delta-X (pixels)')
plt.ylabel('delta-Y (pixels)')
ps.savefig()
matches = solve.tanMatches
msx, msy = [], []
mrx, mry = [], []
for m in matches:
ref, src = m.first, m.second
xy = tanwcs.skyToPixel(ref.getCoord())
mrx.append(xy[0])
mry.append(xy[1])
msx.append(src.getX())
msy.append(src.getY())
plt.clf()
plt.plot(x, y, 'r.')
plt.plot(msx, msy, 'o', mec='r')
plt.plot(rx0, ry0, 'g.')
plt.plot(mrx, mry, 'gx')
plt.title('TAN matches')
ps.savefig()
# Get SIP solution (4th order)
solve = ast.determineWcs2(srcs, **imargs)
wcs1 = solve.sipWcs
matches = solve.sipMatches
msx, msy = [], []
mrx, mry = [], []
for m in matches:
ref, src = m.first, m.second
xy = tanwcs.skyToPixel(ref.getCoord())
mrx.append(xy[0])
mry.append(xy[1])
msx.append(src.getX())
msy.append(src.getY())
plt.clf()
plt.plot(x, y, 'r.')
plt.plot(msx, msy, 'o', mec='r')
plt.plot(rx0, ry0, 'g.')
plt.plot(mrx, mry, 'gx')
plt.title('SIP matches')
ps.savefig()
rx1, ry1 = [], []
for src in refs:
xy = wcs1.skyToPixel(src.getCoord())
rx1.append(xy[0])
ry1.append(xy[1])
rx1 = np.array(rx1)
ry1 = np.array(ry1)
plt.clf()
plt.plot(x, y, 'o', mec='r', mfc='none')
plt.plot(rx0, ry0, 'bx')
plt.plot(rx1, ry1, 'g+')
plt.plot(rx2, ry2, 'mx')
plt.plot(rx3, ry3, 'r+')
ps.savefig()
plt.axis([x0, x0 + 500, y0, y0 + 500])
ps.savefig()
II, d = match(np.vstack((x, y)).T, np.vstack((rx1, ry1)).T, R)
I = II[:, 0]
J = II[:, 1]
plt.clf()
plothist(x[I] - rx1[J], y[I] - ry1[J], 200, **pa)
plt.title('Source positions - Reference positions (SIP WCS)')
plt.xlabel('delta-X (pixels)')
plt.ylabel('delta-Y (pixels)')
ps.savefig()
[-0.21878855, -0.0432496 ],
[-0.83365747, -0.13039277]])
sigma = np.array([[[ 7.72925584e-01, 5.23305564e-02],
[ 5.23305564e-02, 8.89078473e-01]],
[[ 9.84585869e+00, 7.79378820e-01],
[ 7.79378820e-01, 8.84764455e+00]],
[[ 2.02664489e+02, -8.16667434e-01],
[ -8.16667434e-01, 1.87881670e+02]]])
psf = GaussianMixturePSF(w, mu, sigma)
data = np.zeros((200, 200))
invvar = np.zeros_like(data)
tim = Image(data=data, invvar=invvar, psf=psf)
tractor = Tractor([tim], [s])
nn = np.linspace(0.5, 5.5, 12)
cols = int(np.ceil(np.sqrt(len(nn))))
rows = int(np.ceil(len(nn) / float(cols)))
plt.clf()
for i,n in enumerate(nn):
s.sersicindex.setValue(n)
mod = tractor.getModelImage(0)
plt.subplot(rows, cols, i+1)
plt.imshow(np.log10(np.maximum(1e-16, mod)), interpolation='nearest',
origin='lower')
plt.title('index %.2f' % n)
ps.savefig()
def main():
ps = PlotSequence('cov')
survey = LegacySurveyData()
ra,dec = 242.0, 10.2
fn = 'coverage-ccds.fits'
if not os.path.exists(fn):
ccds = survey.get_ccds()
ccds.cut(ccds.filter == 'r')
ccds.cut(ccds.propid == '2014B-0404')
ccds.cut(np.hypot(ccds.ra_bore - ra, ccds.dec_bore - dec) < 2.5)
print(np.unique(ccds.expnum), 'unique exposures')
print('propids', np.unique(ccds.propid))
ccds.writeto(fn)
else:
ccds = fits_table(fn)
plt.clf()
for e in np.unique(ccds.expnum):
I = np.flatnonzero(ccds.expnum == e)
plt.plot(ccds.ra[I], ccds.dec[I], '.')
ps.savefig()
degw = 3.0
pixscale = 10.
W = degw * 3600 / 10.
H = W
hi = 6
cmap = cmap_discretize('jet', hi+1)
wcs = Tan(ra, dec, W/2.+0.5, H/2.+0.5,
-pixscale/3600., 0., 0., pixscale/3600., float(W), float(H))
r0,d0 = wcs.pixelxy2radec(1,1)
r1,d1 = wcs.pixelxy2radec(W,H)
extent = [min(r0,r1),max(r0,r1), min(d0,d1),max(d0,d1)]
for expnums in [ [348666], [348666,348710, 348686],
[348659, 348667, 348658, 348666, 348665, 348669, 348668],
None,
[348683, 348687, 347333, 348686, 348685, 348692, 348694,
348659, 348667, 348658, 348666, 348665, 348669, 348668,
348707, 348709, 348708, 348710, 348711, 348716, 348717],
]:
nexp = np.zeros((H,W), np.uint8)
for ccd in ccds:
if expnums is not None and not ccd.expnum in expnums:
continue
ccdwcs = survey.get_approx_wcs(ccd)
r,d = ccdwcs.pixelxy2radec(1, 1)
ok,x0,y0 = wcs.radec2pixelxy(r, d)
r,d = ccdwcs.pixelxy2radec(ccd.width, ccd.height)
ok,x1,y1 = wcs.radec2pixelxy(r, d)
xlo = np.clip(int(np.round(min(x0,x1))) - 1, 0, W-1)
xhi = np.clip(int(np.round(max(x0,x1))) - 1, 0, W-1)
ylo = np.clip(int(np.round(min(y0,y1))) - 1, 0, H-1)
yhi = np.clip(int(np.round(max(y0,y1))) - 1, 0, H-1)
nexp[ylo:yhi+1, xlo:xhi+1] += 1
plt.clf()
plt.imshow(nexp, interpolation='nearest', origin='lower',
vmin=-0.5, vmax=hi+0.5, cmap=cmap, extent=extent)
plt.colorbar(ticks=np.arange(hi+1))
ps.savefig()
O = fits_table('obstatus/decam-tiles_obstatus.fits')
O.cut(np.hypot(O.ra - ra, O.dec - dec) < 2.5)
for p in [1,2,3]:
print('Pass', p, 'exposures:', O.r_expnum[O.get('pass') == p])
O.cut(O.get('pass') == 2)
print(len(O), 'pass 2 nearby')
d = np.hypot(O.ra - ra, O.dec - dec)
print('Dists:', d)
I = np.flatnonzero(d < 0.5)
assert(len(I) == 1)
ocenter = O[I[0]]
print('Center expnum', ocenter.r_expnum)
I = np.flatnonzero(d >= 0.5)
O.cut(I)
#center = ccds[ccds.expnum == ocenter.r_expnum]
#p2 = ccds[ccds.
ok,xc,yc = wcs.radec2pixelxy(ocenter.ra, ocenter.dec)
xx,yy = np.meshgrid(np.arange(W)+1, np.arange(H)+1)
c_d2 = (xc - xx)**2 + (yc - yy)**2
best = np.ones((H,W), bool)
for o in O:
ok,x,y = wcs.radec2pixelxy(o.ra, o.dec)
d2 = (x - xx)**2 + (y - yy)**2
best[d2 < c_d2] = False
del d2
del c_d2,xx,yy
# plt.clf()
# plt.imshow(best, interpolation='nearest', origin='lower', cmap='gray',
# vmin=0, vmax=1)
# ps.savefig()
plt.clf()
plt.imshow(nexp * best, interpolation='nearest', origin='lower',
vmin=-0.5, vmax=hi+0.5, cmap=cmap, extent=extent)
plt.colorbar(ticks=np.arange(hi+1))
ps.savefig()
plt.clf()
n,b,p = plt.hist(np.clip(nexp[best], 0, hi), range=(-0.5,hi+0.5), bins=hi+1)
plt.xlim(-0.5, hi+0.5)
ps.savefig()
print('b', b)
print('n', n)
print('fracs', np.array(n) / np.sum(n))
print('pcts', ', '.join(['%.1f' % f for f in 100. * np.array(n)/np.sum(n)]))
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--name1', help='Name for first data set')
parser.add_argument('--name2', help='Name for second data set')
parser.add_argument('--plot-prefix',
default='compare',
help='Prefix for plot filenames; default "%default"')
parser.add_argument('--match',
default=1.0,
help='Astrometric cross-match distance in arcsec')
parser.add_argument('dir1', help='First directory to compare')
parser.add_argument('dir2', help='Second directory to compare')
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = 'Comparing %s to %s' % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile('tractor-.*.fits')
cat1, cat2 = [], []
for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print('Skipping', fn, 'due to filename')
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), 'from', fn)
cat.append(t)
cat1 = merge_tables(cat1, columns='fillzero')
cat2 = merge_tables(cat2, columns='fillzero')
print('Total of', len(cat1), 'from', name1)
print('Total of', len(cat2), 'from', name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)
cat1.cut((cat1.decam_anymask[:, 1] == 0) *
(cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0))
cat2.cut((cat2.decam_anymask[:, 1] == 0) *
(cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0))
print('Total of', len(cat1), 'unmasked from', name1)
print('Total of', len(cat2), 'unmasked from', name2)
I, J, d = match_radec(cat1.ra,
cat1.dec,
cat2.ra,
cat2.dec,
opt.match / 3600.,
nearest=True)
print(len(I), 'matched')
plt.clf()
plt.hist(d * 3600., 100)
plt.xlabel('Match distance (arcsec)')
plt.title(tt)
ps.savefig()
matched1 = cat1[I]
matched2 = cat2[J]
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
K = np.flatnonzero((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
print('Median mw_trans', band, 'is',
np.median(matched1.decam_mw_transmission[:, iband]))
plt.clf()
plt.errorbar(
matched1.decam_flux[K, iband],
matched2.decam_flux[K, iband],
fmt='.',
color=cc,
xerr=1. / np.sqrt(matched1.decam_flux_ivar[K, iband]),
yerr=1. / np.sqrt(matched2.decam_flux_ivar[K, iband]),
alpha=0.1,
)
plt.xlabel('%s flux: %s' % (name1, band))
plt.ylabel('%s flux: %s' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
plt.axis([-100, 1000, -100, 1000])
plt.title(tt)
ps.savefig()
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
K = np.flatnonzero(good)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
P = np.flatnonzero(good * psf1 * psf2)
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1. / iv1 + 1. / iv2)
plt.clf()
plt.plot(
mag1[K],
(matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) /
std[K],
'.',
alpha=0.1,
color=cc)
plt.plot(
mag1[P],
(matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) /
std[P],
'.',
alpha=0.1,
color='k')
plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' %
(name2, name1, band))
plt.xlabel('%s mag: %s' % (name1, band))
plt.axhline(0, color='k', alpha=0.5)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
plt.clf()
lp, lt = [], []
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
#good = True
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1. / iv1 + 1. / iv2)
#std = np.hypot(std, 0.01)
G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) *
(mag1 >= 20) *
(mag1 < dict(g=24, r=23.5, z=22.5)[band]))
n, b, p = plt.hist(
(matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) /
std[G],
range=(-4, 4),
bins=50,
histtype='step',
color=cc,
normed=True)
sig = (matched2.decam_flux[G, iband] -
matched1.decam_flux[G, iband]) / std[G]
print('Raw mean and std of points:', np.mean(sig), np.std(sig))
med = np.median(sig)
rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
print('Median and percentile-based sigma:', med, rsigma)
lp.append(p[0])
lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))
bins = []
gaussint = []
for blo, bhi in zip(b, b[1:]):
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
#bins.extend([blo,bhi])
#gaussint.extend([c,c])
bins.append((blo + bhi) / 2.)
gaussint.append(c)
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.title(tt)
plt.xlabel('Flux difference / error (sigma)')
plt.axvline(0, color='k', alpha=0.1)
plt.ylim(0, 0.45)
plt.legend(lp, lt, loc='upper right')
ps.savefig()
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
plt.clf()
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband])
meanmag = NanoMaggies.nanomaggiesToMag(
(matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) /
2.)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0) * np.isfinite(mag1) *
np.isfinite(mag2))
K = np.flatnonzero(good)
P = np.flatnonzero(good * psf1 * psf2)
plt.errorbar(mag1[K],
mag2[K],
fmt='.',
color=cc,
xerr=magerr1[K],
yerr=magerr2[K],
alpha=0.1)
plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s (mag)' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
plt.axis([24, 16, 24, 16])
plt.title(tt)
ps.savefig()
plt.clf()
plt.errorbar(mag1[K],
mag2[K] - mag1[K],
fmt='.',
color=cc,
xerr=magerr1[K],
yerr=magerr2[K],
alpha=0.1)
plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -1, 1])
plt.title(tt)
ps.savefig()
magbins = np.arange(16, 24.001, 0.5)
plt.clf()
plt.plot(mag1[K],
(mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
'.',
color=cc,
alpha=0.1)
plt.plot(mag1[P],
(mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
'k.',
alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
plt.clf()
plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)
midmag = []
vals = np.zeros((len(magbins) - 1, 5))
median_err1 = []
iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)
# FIXME -- should we do some stats after taking off the mean difference?
for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
midmag.append((mlo + mhi) / 2.)
median_err1.append(np.median(magerr1[I]))
if len(I) == 0:
continue
# median and +- 1 sigma quantiles
ybin = y[I]
vals[bini, 0] = np.percentile(ybin, 16)
vals[bini, 1] = np.median(ybin)
vals[bini, 2] = np.percentile(ybin, 84)
# +- 2 sigma quantiles
vals[bini, 3] = np.percentile(ybin, 2.3)
vals[bini, 4] = np.percentile(ybin, 97.7)
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
'vs expected for Gaussian;', len(ybin), 'points')
# if iqd > iqd_gauss:
# # What error adding in quadrature would you need to make the IQD match?
# err = median_err1[-1]
# target_err = err * (iqd / iqd_gauss)
# sys_err = np.sqrt(target_err**2 - err**2)
# print('--> add systematic error', sys_err)
# ~ Johan's cuts
mlo = 21.
mhi = dict(g=24., r=23.5, z=22.5)[band]
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
ybin = y[I]
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', mlo, mhi, 'band', band,
': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;',
len(ybin), 'points')
if iqd > iqd_gauss:
# What error adding in quadrature would you need to make
# the IQD match?
err = np.median(np.hypot(magerr1[I], magerr2[I]))
print('Median error (hypot):', err)
target_err = err * (iqd / iqd_gauss)
print('Target:', target_err)
sys_err = np.sqrt((target_err**2 - err**2) / 2.)
print('--> add systematic error', sys_err)
# check...
err_sys = np.hypot(np.hypot(magerr1, sys_err),
np.hypot(magerr2, sys_err))
ysys = (mag2 - mag1) / err_sys
ysys = ysys[I]
print('Resulting median error:', np.median(err_sys[I]))
iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
# Hmmm, this doesn't work... totally overshoots.
plt.errorbar(midmag,
vals[:, 1],
fmt='o',
color='b',
yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]),
capthick=3,
zorder=20)
plt.errorbar(midmag,
vals[:, 1],
fmt='o',
color='b',
yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]),
capthick=2,
zorder=20)
plt.axhline(1., color='b', alpha=0.2)
plt.axhline(-1., color='b', alpha=0.2)
plt.axhline(2., color='b', alpha=0.2)
plt.axhline(-2., color='b', alpha=0.2)
for mag, err, y in zip(midmag, median_err1, vals[:, 3]):
if not np.isfinite(err):
continue
if y < -6:
continue
plt.text(mag,
y - 0.1,
'%.3f' % err,
va='top',
ha='center',
color='k',
fontsize=10)
plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axvline(21, color='k', alpha=0.3)
plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)
plt.axis([24.1, 16, -6, 6])
plt.title(tt)
ps.savefig()
#magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
if band == 'g':
magbins = [20, 24]
elif band == 'r':
magbins = [20, 23.5]
elif band == 'z':
magbins = [20, 22.5]
slo, shi = -5, 5
plt.clf()
ha = dict(bins=25, range=(slo, shi), histtype='step', normed=True)
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
midmag = []
nn = []
rgbs = []
lt, lp = [], []
for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
if len(I) == 0:
continue
ybin = y[I]
rgb = [0., 0., 0.]
rgb[0] = float(bini) / (len(magbins) - 1)
rgb[2] = 1. - rgb[0]
n, b, p = plt.hist(ybin, color=rgb, **ha)
lt.append('mag %g to %g' % (mlo, mhi))
lp.append(p[0])
midmag.append((mlo + mhi) / 2.)
nn.append(n)
rgbs.append(rgb)
bins = []
gaussint = []
for blo, bhi in zip(b, b[1:]):
#midbin.append((blo+bhi)/2.)
#gaussint.append(scipy.stats.norm.cdf(bhi) -
# scipy.stats.norm.cdf(blo))
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
bins.extend([blo, bhi])
gaussint.extend([c, c])
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo, shi)
ps.savefig()
bincenters = b[:-1] + (b[1] - b[0]) / 2.
plt.clf()
lp = []
for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]):
p = plt.plot(bincenters, n, '-', color=rgb)
lp.append(p[0])
plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo, shi)
ps.savefig()
def galex_forcedphot(galex_dir, cat, tiles, band, roiradecbox,
pixelized_psf=False, ps=None):
'''
Given a list of tractor sources *cat*
and a list of GALEX tiles *tiles* (a fits_table with RA,Dec,tilename)
runs forced photometry, returning a FITS table the same length as *cat*.
'''
from tractor import Tractor
from astrometry.util.ttime import Time
if False:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('wise-forced-w%i' % band)
plots = (ps is not None)
if plots:
import pylab as plt
use_ceres = True
wantims = True
get_models = True
gband = 'galex'
phot = fits_table()
tims = []
for tile in tiles:
info('Reading GALEX tile', tile.visitname.strip(), 'band', band)
tim = galex_tractor_image(tile, band, galex_dir, roiradecbox, gband)
if tim is None:
debug('Actually, no overlap with tile', tile.tilename)
continue
# if plots:
# sig1 = tim.sig1
# plt.clf()
# plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
# cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
# plt.colorbar()
# tag = '%s W%i' % (tile.tilename, band)
# plt.title('%s: tim data' % tag)
# ps.savefig()
if pixelized_psf:
psfimg = galex_psf(band, galex_dir)
tim.psf = PixelizedPSF(psfimg)
# if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
# mjd[I] = (tim.mjdmin + tim.mjdmax) / 2.
# # PSF norm for depth
# psf = tim.getPsf()
# h,w = tim.shape
# patch = psf.getPointSourcePatch(h//2, w//2).patch
# psfnorm = np.sqrt(np.sum(patch**2))
# # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags.
# psfdepth = 1. / (tim.sig1 / psfnorm)**2
# phot.get(wband + '_psfdepth')[I] = psfdepth
tim.tile = tile
tims.append(tim)
tractor = Tractor(tims, cat)
if use_ceres:
from tractor.ceres_optimizer import CeresOptimizer
ceres_block = 8
tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block)
tractor.freezeParamsRecursive('*')
tractor.thawPathsTo(gband)
t0 = Time()
R = tractor.optimize_forced_photometry(
fitstats=True, variance=True, shared_params=False,
wantims=wantims)
info('GALEX forced photometry took', Time() - t0)
#info('Result:', R)
if use_ceres:
term = R.ceres_status['termination']
# Running out of memory can cause failure to converge and term
# status = 2. Fail completely in this case.
if term != 0:
info('Ceres termination status:', term)
raise RuntimeError('Ceres terminated with status %i' % term)
if wantims:
ims1 = R.ims1
flux_invvars = R.IV
# if plots:
# # Create models for just the brightest sources
# bright_cat = [src for src in cat
# if src.getBrightness().getBand(wanyband) > 1000]
# debug('Bright soures:', len(bright_cat))
# btr = Tractor(tims, bright_cat)
# for tim in tims:
# mod = btr.getModelImage(tim)
# tile = tim.tile
# tag = '%s W%i' % (tile.tilename, band)
# sig1 = tim.sig1
# plt.clf()
# plt.imshow(mod, interpolation='nearest', origin='lower',
# cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
# plt.colorbar()
# plt.title('%s: bright-star models' % tag)
# ps.savefig()
if get_models:
models = []
for i,tim in enumerate(tims):
tile = tim.tile
(dat, mod, ie, _, _) = ims1[i]
models.append((tile.visitname, band, tim.wcs.wcs, dat, mod, ie))
if plots:
for i,tim in enumerate(tims):
tile = tim.tile
tag = '%s %s' % (tile.tilename, band)
(dat, mod, _, chi, _) = ims1[i]
sig1 = tim.sig1
plt.clf()
plt.imshow(dat, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi, interpolation='nearest', origin='lower',
cmap='gray', vmin=-5, vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
nm = np.array([src.getBrightness().getBand(gband) for src in cat])
nm_ivar = flux_invvars
# Sources out of bounds, eg, never change from their default
# (1-sigma or whatever) initial fluxes. Zero them out instead.
nm[nm_ivar == 0] = 0.
niceband = band + 'uv'
phot.set('flux_' + niceband, nm.astype(np.float32))
phot.set('flux_ivar_' + niceband, nm_ivar.astype(np.float32))
#phot.set(band + '_mjd', mjd)
rtn = gphotduck()
rtn.phot = phot
rtn.models = None
if get_models:
rtn.models = models
return rtn
def stage_fit_on_coadds(survey=None,
targetwcs=None,
pixscale=None,
bands=None,
tims=None,
brickname=None,
version_header=None,
coadd_tiers=None,
apodize=True,
subsky=True,
ubercal_sky=False,
subsky_radii=None,
nsatur=None,
fitoncoadds_reweight_ivar=True,
plots=False,
plots2=False,
ps=None,
coadd_bw=False,
W=None,
H=None,
brick=None,
blobs=None,
lanczos=True,
ccds=None,
write_metrics=True,
mp=None,
record_event=None,
**kwargs):
from legacypipe.coadds import make_coadds
from legacypipe.bits import DQ_BITS
from legacypipe.survey import LegacySurveyWcs
from legacypipe.coadds import get_coadd_headers
from tractor.image import Image
from tractor.basics import LinearPhotoCal
from tractor.sky import ConstantSky
from tractor.psf import PixelizedPSF
from tractor.tractortime import TAITime
import astropy.time
import fitsio
if plots or plots2:
import pylab as plt
from legacypipe.survey import get_rgb
# Custom sky-subtraction for large galaxies.
skydict = {}
if not subsky:
if ubercal_sky:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('fitoncoadds-{}'.format(brickname))
tims, skydict = ubercal_skysub(tims,
targetwcs,
survey,
brickname,
bands,
mp,
subsky_radii=subsky_radii,
plots=True,
plots2=False,
ps=ps,
verbose=True)
else:
print('Skipping sky-subtraction entirely.')
# Create coadds and then build custom tims from them.
for tim in tims:
ie = tim.inverr
if np.any(ie < 0):
print('Negative inverse error in image {}'.format(tim.name))
CC = []
if coadd_tiers:
# Sort by band and sort them into tiers.
tiers = [[] for i in range(coadd_tiers)]
for b in bands:
btims = []
seeing = []
for tim in tims:
if tim.band != b:
continue
btims.append(tim)
seeing.append(tim.psf_fwhm * tim.imobj.pixscale)
I = np.argsort(seeing)
btims = [btims[i] for i in I]
seeing = [seeing[i] for i in I]
N = min(coadd_tiers, len(btims))
splits = np.round(np.arange(N + 1) * float(len(btims)) /
N).astype(int)
print('Splitting', len(btims), 'images into', N, 'tiers: splits:',
splits)
print('Seeing limits:',
[seeing[min(s,
len(seeing) - 1)] for s in splits])
for s0, s1, tt in zip(splits, splits[1:], tiers):
tt.extend(btims[s0:s1])
for itier, tier in enumerate(tiers):
print('Producing coadds for tier', (itier + 1))
C = make_coadds(
tier,
bands,
targetwcs,
detmaps=True,
ngood=True,
lanczos=lanczos,
allmasks=True,
anymasks=True,
psf_images=True,
nsatur=2,
mp=mp,
plots=plots2,
ps=ps, # note plots2 here!
callback=None)
if plots:
plt.clf()
for iband, (band, psf) in enumerate(zip(bands, C.psf_imgs)):
plt.subplot(1, len(bands), iband + 1)
plt.imshow(psf, interpolation='nearest', origin='lower')
plt.title('Coadd PSF image: band %s' % band)
plt.suptitle('Tier %i' % (itier + 1))
ps.savefig()
# for band,img in zip(bands, C.coimgs):
# plt.clf()
# plt.imshow(img,
plt.clf()
plt.imshow(get_rgb(C.coimgs, bands), origin='lower')
plt.title('Tier %i' % (itier + 1))
ps.savefig()
CC.append(C)
else:
C = make_coadds(
tims,
bands,
targetwcs,
detmaps=True,
ngood=True,
lanczos=lanczos,
allmasks=True,
anymasks=True,
psf_images=True,
mp=mp,
plots=plots2,
ps=ps, # note plots2 here!
callback=None)
CC.append(C)
cotims = []
for C in CC:
if plots2:
for band, iv in zip(bands, C.cowimgs):
pass
# plt.clf()
# plt.imshow(np.sqrt(iv), interpolation='nearest', origin='lower')
# plt.title('Coadd Inverr: band %s' % band)
# ps.savefig()
for band, psf in zip(bands, C.psf_imgs):
plt.clf()
plt.imshow(psf, interpolation='nearest', origin='lower')
plt.title('Coadd PSF image: band %s' % band)
ps.savefig()
for band, img, iv in zip(bands, C.coimgs, C.cowimgs):
from scipy.ndimage.filters import gaussian_filter
# plt.clf()
# plt.hist((img * np.sqrt(iv))[iv>0], bins=50, range=(-5,8), log=True)
# plt.title('Coadd pixel values (sigmas): band %s' % band)
# ps.savefig()
psf_sigma = np.mean([
(tim.psf_sigma * tim.imobj.pixscale / pixscale)
for tim in tims if tim.band == band
])
gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
psfnorm = gnorm #np.sqrt(np.sum(psfimg**2))
detim = gaussian_filter(img, psf_sigma) / psfnorm**2
cosig1 = 1. / np.sqrt(np.median(iv[iv > 0]))
detsig1 = cosig1 / psfnorm
# plt.clf()
# plt.subplot(2,1,1)
# plt.hist(detim.ravel() / detsig1, bins=50, range=(-5,8), log=True)
# plt.title('Coadd detection map values / sig1 (sigmas): band %s' % band)
# plt.subplot(2,1,2)
# plt.hist(detim.ravel() / detsig1, bins=50, range=(-5,8))
# ps.savefig()
# # as in detection.py
# detiv = np.zeros_like(detim) + (1. / detsig1**2)
# detiv[iv == 0] = 0.
# detiv = gaussian_filter(detiv, psf_sigma)
#
# plt.clf()
# plt.hist((detim * np.sqrt(detiv)).ravel(), bins=50, range=(-5,8), log=True)
# plt.title('Coadd detection map values / detie (sigmas): band %s' % band)
# ps.savefig()
for iband, (band, img, iv, allmask, anymask, psfimg) in enumerate(
zip(bands, C.coimgs, C.cowimgs, C.allmasks, C.anymasks,
C.psf_imgs)):
mjd = np.mean(
[tim.imobj.mjdobs for tim in tims if tim.band == band])
mjd_tai = astropy.time.Time(mjd, format='mjd', scale='utc').tai.mjd
tai = TAITime(None, mjd=mjd_tai)
twcs = LegacySurveyWcs(targetwcs, tai)
#print('PSF sigmas (in pixels) for band', band, ':',
# ['%.2f' % tim.psf_sigma for tim in tims if tim.band == band])
print(
'PSF sigmas in coadd pixels:', ', '.join([
'%.2f' % (tim.psf_sigma * tim.imobj.pixscale / pixscale)
for tim in tims if tim.band == band
]))
psf_sigma = np.mean([
(tim.psf_sigma * tim.imobj.pixscale / pixscale) for tim in tims
if tim.band == band
])
print('Using average PSF sigma', psf_sigma)
psf = PixelizedPSF(psfimg)
gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
psfnorm = np.sqrt(np.sum(psfimg**2))
print('Gaussian PSF norm', gnorm, 'vs pixelized', psfnorm)
# if plots:
# from collections import Counter
# plt.clf()
# plt.imshow(mask, interpolation='nearest', origin='lower')
# plt.colorbar()
# plt.title('allmask')
# ps.savefig()
# print('allmask for band', band, ': values:', Counter(mask.ravel()))
# Scale invvar to take into account that we have resampled (~double-counted) pixels
tim_pixscale = np.mean(
[tim.imobj.pixscale for tim in tims if tim.band == band])
cscale = tim_pixscale / pixscale
print('average tim pixel scale / coadd scale:', cscale)
iv /= cscale**2
if fitoncoadds_reweight_ivar:
# We first tried setting the invvars constant per tim -- this
# makes things worse, since we *remove* the lowered invvars at
# the cores of galaxies.
#
# Here we're hacking the relative weights -- squaring the
# weights but then making the median the same, ie, squaring
# the dynamic range or relative weights -- ie, downweighting
# the cores even more than they already are from source
# Poisson terms.
median_iv = np.median(iv[iv > 0])
assert (median_iv > 0)
iv = iv * np.sqrt(iv) / np.sqrt(median_iv)
assert (np.all(np.isfinite(iv)))
assert (np.all(iv >= 0))
cotim = Image(img,
invvar=iv,
wcs=twcs,
psf=psf,
photocal=LinearPhotoCal(1., band=band),
sky=ConstantSky(0.),
name='coadd-' + band)
cotim.band = band
cotim.subwcs = targetwcs
cotim.psf_sigma = psf_sigma
cotim.sig1 = 1. / np.sqrt(np.median(iv[iv > 0]))
# Often, SATUR masks on galaxies / stars are surrounded by BLEED pixels. Soak these into
# the SATUR mask.
from scipy.ndimage.morphology import binary_dilation
anymask |= np.logical_and(((anymask & DQ_BITS['bleed']) > 0),
binary_dilation(
((anymask & DQ_BITS['satur']) > 0),
iterations=10)) * DQ_BITS['satur']
# Saturated in any image -> treat as saturated in coadd
# (otherwise you get weird systematics in the weighted coadds, and weird source detection!)
mask = allmask
mask[(anymask & DQ_BITS['satur'] > 0)] |= DQ_BITS['satur']
if coadd_tiers:
# nsatur -- reset SATUR bit
mask &= ~DQ_BITS['satur']
mask |= DQ_BITS['satur'] * C.satmaps[iband]
cotim.dq = mask
cotim.dq_saturation_bits = DQ_BITS['satur']
cotim.psfnorm = gnorm
cotim.galnorm = 1.0 # bogus!
cotim.imobj = Duck()
cotim.imobj.fwhm = 2.35 * psf_sigma
cotim.imobj.pixscale = pixscale
cotim.time = tai
cotim.primhdr = fitsio.FITSHDR()
get_coadd_headers(cotim.primhdr, tims, band, coadd_headers=skydict)
cotims.append(cotim)
if plots:
plt.clf()
bitmap = dict([(v, k) for k, v in DQ_BITS.items()])
k = 1
for i in range(12):
bitval = 1 << i
if not bitval in bitmap:
continue
# only 9 bits are actually used
plt.subplot(3, 3, k)
k += 1
plt.imshow((cotim.dq & bitval) > 0,
vmin=0,
vmax=1.5,
cmap='hot',
origin='lower')
plt.title(bitmap[bitval])
plt.suptitle('Coadd mask planes %s band' % band)
ps.savefig()
plt.clf()
h, w = cotim.shape
rgb = np.zeros((h, w, 3), np.uint8)
rgb[:, :, 0] = (cotim.dq & DQ_BITS['satur'] > 0) * 255
rgb[:, :, 1] = (cotim.dq & DQ_BITS['bleed'] > 0) * 255
plt.imshow(rgb, origin='lower')
plt.suptitle('Coadd DQ band %s: red = SATUR, green = BLEED' %
band)
ps.savefig()
# Save an image of the coadd PSF
# copy version_header before modifying it.
hdr = fitsio.FITSHDR()
for r in version_header.records():
hdr.add_record(r)
hdr.add_record(
dict(name='IMTYPE',
value='coaddpsf',
comment='LegacySurveys image type'))
hdr.add_record(
dict(name='BAND', value=band,
comment='Band of this coadd/PSF'))
hdr.add_record(
dict(name='PSF_SIG',
value=psf_sigma,
comment='Average PSF sigma (coadd pixels)'))
hdr.add_record(
dict(name='PIXSCAL',
value=pixscale,
comment='Pixel scale of this PSF (arcsec)'))
hdr.add_record(
dict(name='INPIXSC',
value=tim_pixscale,
comment='Native image pixscale scale (average, arcsec)'))
hdr.add_record(
dict(name='MJD', value=mjd, comment='Average MJD for coadd'))
hdr.add_record(
dict(name='MJD_TAI',
value=mjd_tai,
comment='Average MJD (in TAI) for coadd'))
with survey.write_output('copsf', brick=brickname,
band=band) as out:
out.fits.write(psfimg, header=hdr)
# EVIL
return dict(tims=cotims, coadd_headers=skydict)
def main():
ps = PlotSequence('conv')
S = 51
center = S/2
print('Center', center)
#for psf_sigma in [2., 1.5, 1.]:
for psf_sigma in [2.]:
rms2 = []
x = np.arange(S)
y = np.arange(S)
xx,yy = np.meshgrid(x, y)
scale = 1.5 / psf_sigma
pixpsf = render_airy((scale, center), x, y)
psf = (scale,center)
eval_psf = render_airy
plt.clf()
plt.subplot(2,1,1)
plt.plot(x, pixpsf[center,:], 'b-')
plt.plot(x, pixpsf[:,center], 'r-')
plt.subplot(2,1,2)
plt.plot(x, np.maximum(1e-16, pixpsf[center,:]), 'b-')
plt.plot(x, np.maximum(1e-16, pixpsf[:,center]), 'r-')
plt.yscale('log')
ps.savefig()
plt.clf()
plt.imshow(pixpsf, interpolation='nearest', origin='lower')
ps.savefig()
plt.clf()
plt.imshow(np.log10(np.maximum(1e-16, pixpsf)),
interpolation='nearest', origin='lower')
plt.colorbar()
plt.title('log PSF')
ps.savefig()
# psf
#psf = scipy.stats.norm(loc=center + 0.5, scale=psf_sigma)
# plt.clf()
# plt.imshow(Pcdf, interpolation='nearest', origin='lower')
# ps.savefig()
# #Pcdf = psf.cdf(xx) * psf.cdf(yy)
# #pixpsf = integrate_gaussian(psf, xx, yy)
#
# padpsf = np.zeros((S*2-1, S*2-1))
# ph,pw = pixpsf.shape
# padpsf[S/2:S/2+ph, S/2:S/2+pw] = pixpsf
# Fpsf = np.fft.rfft2(padpsf)
#
# padh,padw = padpsf.shape
# v = np.fft.rfftfreq(padw)
# w = np.fft.fftfreq(padh)
# fmax = max(max(np.abs(v)), max(np.abs(w)))
# cut = fmax / 2. * 1.000001
# #print('Frequence cut:', cut)
# Ffiltpsf = Fpsf.copy()
# #print('Ffiltpsf', Ffiltpsf.shape)
# #print((np.abs(w) < cut).shape)
# #print((np.abs(v) < cut).shape)
# Ffiltpsf[np.abs(w) > cut, :] = 0.
# Ffiltpsf[:, np.abs(v) > cut] = 0.
# #print('pad v', v)
# #print('pad w', w)
#
# filtpsf = np.fft.irfft2(Ffiltpsf, s=(padh,padw))
#
# print('filtered PSF real', np.max(np.abs(filtpsf.real)))
# print('filtered PSF imag', np.max(np.abs(filtpsf.imag)))
#
# plt.clf()
# plt.subplot(2,3,1)
# dimshow(Fpsf.real)
# plt.colorbar()
# plt.title('Padded PSF real')
# plt.subplot(2,3,4)
# dimshow(Fpsf.imag)
# plt.colorbar()
# plt.title('Padded PSF imag')
#
# plt.subplot(2,3,2)
# dimshow(Ffiltpsf.real)
# plt.colorbar()
# plt.title('Filt PSF real')
# plt.subplot(2,3,5)
# dimshow(Ffiltpsf.imag)
# plt.colorbar()
# plt.title('Filt PSF imag')
#
# plt.subplot(2,3,3)
# dimshow(filtpsf.real)
# plt.title('PSF real')
# plt.colorbar()
#
# plt.subplot(2,3,6)
# dimshow(filtpsf.imag)
# plt.title('PSF imag')
# plt.colorbar()
#
# ps.savefig()
#
#
# pixpsf = filtpsf
gal_sigmas = [2, 1, 0.5, 0.25]
for gal_sigma in gal_sigmas:
# plt.clf()
# plt.imshow(Gcdf, interpolation='nearest', origin='lower')
# plt.savefig('dcdf.png')
# plt.clf()
# plt.imshow(np.exp(-0.5 * ((xx-center)**2 + (yy-center)**2)/2.**2),
# interpolation='nearest', origin='lower')
# plt.savefig('g.png')
# my convolution
pixscale = 1.
cd = pixscale * np.eye(2) / 3600.
P,FG,Gmine,v,w = galaxy_psf_convolution(
gal_sigma, 0., 0., GaussianGalaxy, cd,
0., 0., pixpsf, debug=True)
#print('v:', v)
#print('w:', w)
#print('P:', P.shape)
print()
print('PSF %g, Gal %g' % (psf_sigma, gal_sigma))
rmax = np.argmax(np.abs(w))
cmax = np.argmax(np.abs(v))
l2_rmax = np.sqrt(np.sum(P[rmax,:].real**2 + P[rmax,:].imag**2))
l2_cmax = np.sqrt(np.sum(P[:,cmax].real**2 + P[:,cmax].imag**2))
print('PSF L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax)
l2_rmax = np.sqrt(np.sum(FG[rmax,:].real**2 + FG[rmax,:].imag**2))
l2_cmax = np.sqrt(np.sum(FG[:,cmax].real**2 + FG[:,cmax].imag**2))
print('Gal L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax)
C = P * FG
l2_rmax = np.sqrt(np.sum(C[rmax,:].real**2 + C[rmax,:].imag**2))
l2_cmax = np.sqrt(np.sum(C[:,cmax].real**2 + C[:,cmax].imag**2))
print('PSF*Gal L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax)
print()
Fpsf, Fgal = compare_subsampled(
S, 1, ps, psf, pixpsf, Gmine,v,w,
gal_sigma, psf_sigma, cd, get_ffts=True, eval_psf=eval_psf)
plt.clf()
plt.subplot(2,4,1)
dimshow(P.real)
plt.colorbar()
plt.title('PSF real')
plt.subplot(2,4,5)
dimshow(P.imag)
plt.colorbar()
plt.title('PSF imag')
plt.subplot(2,4,2)
dimshow(FG.real)
plt.colorbar()
plt.title('Gal real')
plt.subplot(2,4,6)
dimshow(FG.imag)
plt.colorbar()
plt.title('Gal imag')
plt.subplot(2,4,3)
dimshow((P * FG).real)
plt.colorbar()
plt.title('P*Gal real')
plt.subplot(2,4,7)
dimshow((P * FG).imag)
plt.colorbar()
plt.title('P*Gal imag')
plt.subplot(2,4,4)
dimshow((Fgal).real)
plt.colorbar()
plt.title('pixGal real')
plt.subplot(2,4,8)
dimshow((Fgal).imag)
plt.colorbar()
plt.title('pixGal imag')
plt.suptitle('PSF %g, Gal %g' % (psf_sigma, gal_sigma))
ps.savefig()
subsample = [1,2,4]
rms1 = []
for s in subsample:
rms = compare_subsampled(S, s, ps, psf, pixpsf, Gmine,v,w, gal_sigma, psf_sigma, cd, eval_psf=eval_psf)
rms1.append(rms)
rms2.append(rms1)
print()
print('PSF sigma =', psf_sigma)
print('RMSes:')
for rms1,gal_sigma in zip(rms2, gal_sigmas):
print('Gal sigma', gal_sigma, 'rms:',
', '.join(['%.3g' % r for r in rms1]))
def main(decals=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for decals.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if decals is None:
decals = Decals()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from decals-ccds.fits table
T = decals.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(
np.array([f.strip() == opt.filename
for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert (len(T) == 1)
im = decals.get_image_object(T[0])
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True)
print('Got tim:', tim)
if opt.catfn in ['DR1', 'DR2']:
if opt.catalog_path is None:
opt.catalog_path = opt.catfn.lower()
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, decals=decals)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3],
'tractor-%s.fits' % b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) *
(yy >= -margin) * (yy <= (H + margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
TT.append(T)
T = merge_tables(TT)
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T
T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T
cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE)
# print('Got cat:', cat)
print('Forced photom...')
opti = None
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T))
ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack(
[tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap
ap = 1. / (np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap
if opt.forced:
kwa = {}
if opt.plots is None:
kwa.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True,
fitstats=True,
shared_params=False,
**kwa)
if opt.plots:
(data, mod, ie, chi, roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest',
origin='lower',
vmin=-5,
vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([
src.getBrightness().getFlux(tim.band) for src in cat
]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2.astype(np.float32)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, decals.decals_dir)
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
fname = os.path.join(d2, d1, fname)
print('Trimmed filename to', fname)
#version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file'))
version_hdr.add_record(
dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file'))
version_hdr.add_record(
dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext'))
version_hdr.add_record(
dict(name='CAMERA', value='DECam', comment='Dark Energy Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='EXPOSURE',
value='decam-%s-%s' % (im.expnum, im.ccdname),
comment='Name of this image'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {
'mjd': 'sec',
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2'
}
columns = F.get_columns()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
print('Finished forced phot:', Time() - t0)
return 0
def main():
W = H = 50
pixscale = 0.262 / 3600.
band = 'r'
truewcs = Tan(0., 0., W / 2., H / 2., -pixscale, 0., 0., pixscale,
float(W), float(H))
src = PointSource(RaDecPos(
0.,
0.,
), NanoMaggies(**{band: 1.}))
src.symmetric_derivs = True
forced_cat = [src]
sig1 = 0.25
flux = 100.
psf_sigma = 2.0
psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
nsigma = flux * psfnorm / sig1
print('S/N:', nsigma)
v = psf_sigma**2
# Create a pixelized PSF model by rendering the Gaussian on a stamp
xx, yy = np.meshgrid(np.arange(-12, 13), np.arange(-12, 13))
pp = np.exp(-0.5 * (xx**2 + yy**2) / psf_sigma**2)
pp /= np.sum(pp)
psf = PixelizedPSF(pp)
tim = Image(data=np.zeros((H, W), np.float32),
inverr=np.ones((H, W), np.float32) * 1. / sig1,
wcs=ConstantFitsWcs(truewcs),
photocal=LinearPhotoCal(1., band=band),
sky=ConstantSky(0.),
psf=psf)
tim.band = band
tim.sig1 = sig1
#dra_pix = np.linspace(-5, 5, 21)
dra_pix = np.linspace(-2, 2, 21)
ddec_pix = np.zeros_like(dra_pix)
dra2_pix = np.linspace(-5, 5, 21)
ddec2_pix = -0.5 * dra2_pix
ps = PlotSequence('pm')
for dras, ddecs, srcflux in [
(dra_pix * pixscale, ddec_pix * pixscale, flux),
(dra_pix * pixscale, ddec_pix * pixscale, flux / 2),
(dra2_pix * pixscale, ddec2_pix * pixscale, flux),
]:
FF = []
slicex = []
slicey = []
residx = []
residy = []
for dra, ddec in zip(dras, ddecs):
src = PointSource(RaDecPos(0. + dra, 0. + ddec),
NanoMaggies(**{band: srcflux}))
tr = Tractor([tim], [src])
truemod = tr.getModelImage(0)
noise = np.random.normal(size=truemod.shape) * sig1
tim.data = truemod + noise
F = run_forced_phot(forced_cat,
tim,
ceres=False,
derivs=True,
do_apphot=False,
fixed_also=True) #, ps=ps)
#print('Src:', forced_cat)
t = Tractor([tim], forced_cat)
m = t.getModelImage(0)
mh, mw = m.shape
slicex.append(m[mh // 2, :])
slicey.append(m[:, mw // 2])
residx.append((m - truemod)[mh // 2, :])
residy.append((m - truemod)[:, mw // 2])
#F.about()
F.true_dra = dra + np.zeros(len(F))
F.true_ddec = ddec + np.zeros(len(F))
FF.append(F)
F = merge_tables(FF)
plt.clf()
plt.plot(F.true_dra * 3600.,
F.flux_dra / F.flux * 3600.,
'b.',
label='RA')
plt.plot(F.true_ddec * 3600.,
F.flux_ddec / F.flux * 3600.,
'g.',
label='Dec')
mx = max(max(np.abs(F.true_dra)), max(np.abs(F.true_ddec)))
mx *= 3600.
plt.plot([-mx, mx], [-mx, mx], 'k-', alpha=0.1)
plt.xlabel('True offset (arcsec)')
plt.ylabel('Flux deriv / Flux * 3600 (arcsec)')
plt.legend()
ps.savefig()
plt.clf()
plt.plot(np.hypot(F.true_dra, F.true_ddec) * 3600.,
F.flux,
'b.',
label='Flux (dra/dec)')
plt.plot(np.hypot(F.true_dra, F.true_ddec) * 3600.,
F.flux_fixed,
'g.',
label='Flux (fixed)')
plt.xlabel('True offset (arcsec)')
plt.ylabel('Flux')
ps.savefig()
plt.clf()
N = len(slicex)
cc = float(N - 1)
for i, s in enumerate(slicex):
#plt.plot(s + i*10, 'b-')
rgb = (0, i / cc, 1. - i / cc)
#print('rgb', rgb)
plt.plot(s, '-', color=rgb, alpha=0.5)
ps.savefig()
plt.clf()
N = len(residx)
cc = float(N - 1)
for i, s in enumerate(residx):
rgb = (0, i / cc, 1. - i / cc)
plt.plot(s, '-', color=rgb, alpha=0.5)
ps.savefig()
def plot_unmatched():
from bigboss_test import radecroi
'''
select
run, rerun, camcol, field, nChild, probPSF,
psfFlux_u, psfFlux_g, psfFlux_r, psfFlux_i, psfFlux_z,
deVRad_u, deVRad_g, deVRad_r, deVRad_i, deVRad_z,
deVAB_u, deVAB_g, deVAB_r, deVAB_i, deVAB_z,
deVPhi_u, deVPhi_g, deVPhi_r, deVPhi_i, deVPhi_z,
deVFlux_u, deVFlux_g, deVFlux_r, deVFlux_i, deVFlux_z,
expRad_u, expRad_g, expRad_r, expRad_i, expRad_z,
expAB_u, expAB_g, expAB_r, expAB_i, expAB_z,
expPhi_u, expPhi_g, expPhi_r, expPhi_i, expPhi_z,
expFlux_u, expFlux_g, expFlux_r, expFlux_i, expFlux_z,
fracDeV_u, fracDeV_g, fracDeV_r, fracDeV_i, fracDeV_z,
flags_u, flags_g, flags_r, flags_i, flags_z,
probPSF_u, probPSF_g, probPSF_r, probPSF_i, probPSF_z,
ra, dec
from PhotoPrimary
where ra between 333.5 and 335.5 and dec between -0.5 and 1.5
'''
''' -> 124k rows. (sdss-cas-testarea.fits) Distinct runs:
2585, 2728, 7712, 4203, 2583, 4192, 4207, 4184, 2662, 7717
Run #sources
2583 663
2585 675
2662 4
2728 156
4184 762
4192 36135
4203 5
4207 44078
7712 12047
7717 29911
'''
'''
select
run, rerun, camcol, field, nChild, probPSF,
psfFlux_u, psfFlux_g, psfFlux_r, psfFlux_i, psfFlux_z,
deVRad_u, deVRad_g, deVRad_r, deVRad_i, deVRad_z,
deVAB_u, deVAB_g, deVAB_r, deVAB_i, deVAB_z,
deVPhi_u, deVPhi_g, deVPhi_r, deVPhi_i, deVPhi_z,
deVFlux_u, deVFlux_g, deVFlux_r, deVFlux_i, deVFlux_z,
expRad_u, expRad_g, expRad_r, expRad_i, expRad_z,
expAB_u, expAB_g, expAB_r, expAB_i, expAB_z,
expPhi_u, expPhi_g, expPhi_r, expPhi_i, expPhi_z,
expFlux_u, expFlux_g, expFlux_r, expFlux_i, expFlux_z,
fracDeV_u, fracDeV_g, fracDeV_r, fracDeV_i, fracDeV_z,
flags_u, flags_g, flags_r, flags_i, flags_z,
probPSF_u, probPSF_g, probPSF_r, probPSF_i, probPSF_z,
ra, dec, resolveStatus, score into mydb.wisetest from PhotoObjAll
where ra between 333.5 and 335.5 and dec between -0.5 and 1.5
and (resolveStatus & (
dbo.fResolveStatus('SURVEY_PRIMARY') |
dbo.fResolveStatus('SURVEY_BADFIELD') |
dbo.fResolveStatus('SURVEY_EDGE'))) != 0
--> sdss-cas-testarea-2.fits
select
run, rerun, camcol, field, nChild, probPSF,
psfFlux_u, psfFlux_g, psfFlux_r, psfFlux_i, psfFlux_z,
deVRad_u, deVRad_g, deVRad_r, deVRad_i, deVRad_z,
deVAB_u, deVAB_g, deVAB_r, deVAB_i, deVAB_z,
deVPhi_u, deVPhi_g, deVPhi_r, deVPhi_i, deVPhi_z,
deVFlux_u, deVFlux_g, deVFlux_r, deVFlux_i, deVFlux_z,
expRad_u, expRad_g, expRad_r, expRad_i, expRad_z,
expAB_u, expAB_g, expAB_r, expAB_i, expAB_z,
expPhi_u, expPhi_g, expPhi_r, expPhi_i, expPhi_z,
expFlux_u, expFlux_g, expFlux_r, expFlux_i, expFlux_z,
fracDeV_u, fracDeV_g, fracDeV_r, fracDeV_i, fracDeV_z,
flags_u, flags_g, flags_r, flags_i, flags_z,
probPSF_u, probPSF_g, probPSF_r, probPSF_i, probPSF_z,
ra, dec, resolveStatus, score into mydb.wisetest from PhotoObjAll
where ra between 333.5 and 335.5 and dec between -0.5 and 1.5
and (resolveStatus & (
dbo.fResolveStatus('SURVEY_PRIMARY') |
dbo.fResolveStatus('SURVEY_BADFIELD') |
dbo.fResolveStatus('SURVEY_EDGE') |
dbo.fResolveStatus('SURVEY_BEST')
)) != 0
--> sdss-cas-testarea-3.fits
'''
''' Check it out: spatial source density looks fine. No overlap between runs.
'''
rng = ((333.5, 335.5), (-0.5, 1.5))
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('sdss')
if False:
r, d = np.mean(rng[0]), np.mean(rng[1])
RCF = radec_to_sdss_rcf(r, d, radius=2. * 60.,
tablefn='window_flist-DR9.fits')
print('Found', len(RCF), 'fields in range')
for run, c, f, ra, dec in RCF:
print(' ', run, c, f, 'at', ra, dec)
from astrometry.blind.plotstuff import PlotSequence
plot = Plotstuff(rdw=(r, d, 10), size=(1000, 1000), outformat='png')
plot.color = 'white'
plot.alpha = 0.5
plot.apply_settings()
T = fits_table('window_flist-DR9.fits')
I, J, d = match_radec(T.ra, T.dec, r, d, 10)
T.cut(I)
print('Plotting', len(T))
for i, (m0, m1, n0, n1, node, incl) in enumerate(zip(T.mu_start, T.mu_end, T.nu_start, T.nu_end, T.node, T.incl)):
#rr,dd = [],[]
for j, (m, n) in enumerate([(m0, n0), (m0, n1), (m1, n1), (m1, n0), (m0, n0)]):
ri, di = munu_to_radec_deg(m, n, node, incl)
# rr.append(ri)
# dd.append(di)
if j == 0:
plot.move_to_radec(ri, di)
else:
plot.line_to_radec(ri, di)
plot.stroke()
plot.plot_grid(2, 2, 5, 5)
plot.write('fields.png')
# CAS PhotoObjAll.resolveStatus bits
sprim = 0x100
sbad = 0x800
sedge = 0x1000
sbest = 0x200
if False:
T = fits_table('sdss-cas-testarea.fits')
plt.clf()
plothist(T.ra, T.dec, 200, range=rng)
plt.title('PhotoPrimary')
ps.savefig()
T = fits_table('sdss-cas-testarea-2.fits')
plt.clf()
plothist(T.ra, T.dec, 200, range=rng)
plt.title('PhotoObjAll: SURVEY_PRIMARY | SURVEY_BADFIELD | SURVEY_EDGE')
ps.savefig()
T = fits_table('sdss-cas-testarea-3.fits')
plt.clf()
plothist(T.ra, T.dec, 200, range=rng)
plt.title(
'PhotoObjAll: SURVEY_PRIMARY | SURVEY_BADFIELD | SURVEY_EDGE | SURVEY_BEST')
ps.savefig()
for j, (flags, txt) in enumerate([(sprim, 'PRIM'), (sbad, 'BAD'), (sedge, 'EDGE'),
(sbest, 'BEST')]):
I = np.flatnonzero(
(T.resolvestatus & (sprim | sbad | sedge | sbest)) == flags)
print(len(I), 'with', txt)
if len(I) == 0:
continue
plt.clf()
plothist(T.ra[I], T.dec[I], 200, range=rng)
plt.title('%i with %s' % (len(I), txt))
ps.savefig()
for j, (flags, txt) in enumerate([(sprim | sbad, 'PRIM + BAD'),
(sprim | sedge, 'PRIM + EDGE'),
(sprim | sbest, 'PRIM + BEST')]):
I = np.flatnonzero((T.resolvestatus & flags) > 0)
print(len(I), 'with', txt)
if len(I) == 0:
continue
plt.clf()
plothist(T.ra[I], T.dec[I], 200, range=rng)
plt.title('%i with %s' % (len(I), txt))
ps.savefig()
# for run in np.unique(T.run):
# I = (T.run == run)
# plt.clf()
# plothist(T.ra[I], T.dec[I], 200, range=rng)
# plt.title('Run %i' % run)
# ps.savefig()
R = 1. / 3600.
I, J, d = match_radec(T.ra, T.dec, T.ra, T.dec, R, notself=True)
print(len(I), 'matches')
plt.clf()
loghist((T.ra[I] - T.ra[J]) * 3600., (T.dec[I] - T.dec[J])
* 3600., 200, range=((-1, 1), (-1, 1)))
ps.savefig()
ps = PlotSequence('forced')
basedir = os.environ.get('BIGBOSS_DATA', '/project/projectdirs/bigboss')
wisedatadir = os.path.join(basedir, 'data', 'wise')
wisecat = fits_table(os.path.join(
wisedatadir, 'catalogs', 'wisecat2.fits'))
# plt.clf()
#plothist(wisecat.ra, wisecat.dec, 200, range=rng)
# plt.savefig('wisecat.png')
(ra0, ra1, dec0, dec1) = radecroi
ra = (ra0 + ra1) / 2.
dec = (dec0 + dec1) / 2.
#cas = fits_table('sdss-cas-testarea.fits')
#cas = fits_table('sdss-cas-testarea-2.fits')
cas = fits_table('sdss-cas-testarea-3.fits')
print('Read', len(cas), 'CAS sources')
cas.cut((cas.resolvestatus & sedge) == 0)
print('Cut to ', len(cas), 'without SURVEY_EDGE set')
# Check out WISE / SDSS matches.
wise = wisecat
sdss = cas
print(len(sdss), 'SDSS sources')
print(len(wise), 'WISE sources')
R = 10.
I, J, d = match_radec(wise.ra, wise.dec, sdss.ra, sdss.dec,
R / 3600., nearest=True)
print(len(I), 'matches')
print('max dist:', d.max())
plt.clf()
plt.hist(d * 3600., 100, range=(0, R), log=True)
plt.xlabel('Match distance (arcsec)')
plt.ylabel('Number of matches')
plt.title('SDSS-WISE astrometric matches')
ps.savefig()
plt.clf()
loghist((wise.ra[I] - sdss.ra[J]) * 3600., (wise.dec[I] - sdss.dec[J]) * 3600.,
200, range=((-R, R), (-R, R)))
plt.title('SDSS-WISE astrometric matches')
plt.xlabel('dRA (arcsec)')
plt.ylabel('dDec (arcsec)')
ps.savefig()
R = 4.
I, J, d = match_radec(wise.ra, wise.dec, sdss.ra, sdss.dec,
R / 3600., nearest=True)
print(len(I), 'matches')
unmatched = np.ones(len(wise), bool)
unmatched[I] = False
wun = wise[unmatched]
plt.clf()
plothist(sdss.ra, sdss.dec, 200, range=rng)
plt.title('SDSS source density')
ps.savefig()
plt.clf()
plothist(wise.ra, wise.dec, 200, range=rng)
plt.title('WISE source density')
ps.savefig()
plt.clf()
#plt.plot(wun.ra, wun.dec, 'r.')
#plt.axis(rng[0] + rng[1])
plothist(wun.ra, wun.dec, 200, range=rng)
plt.title('Unmatched WISE sources')
ps.savefig()
for band in 'ugriz':
sdss.set('psfmag_' + band,
NanoMaggies.nanomaggiesToMag(sdss.get('psfflux_' + band)))
# plt.clf()
# loghist(wise.w1mpro[I], sdss.psfmag_r[J], 200)
# plt.xlabel('WISE w1mpro')
# plt.ylabel('SDSS psfflux_r')
# ps.savefig()
for band in 'riz':
ax = [0, 10, 25, 5]
plt.clf()
mag = sdss.get('psfmag_' + band)[J]
loghist(mag - wise.w1mpro[I], mag, 200,
range=((ax[0], ax[1]), (ax[3], ax[2])))
plt.xlabel('SDSS %s - WISE w1' % band)
plt.ylabel('SDSS ' + band)
plt.axis(ax)
ps.savefig()
for w, t in [(wise[I], 'Matched'), (wun, 'Unmatched')]:
plt.clf()
w1 = w.get('w1mpro')
w2 = w.get('w2mpro')
ax = [-1, 3, 18, 6]
loghist(w1 - w2, w1, 200,
range=((ax[0], ax[1]), (ax[3], ax[2])))
plt.xlabel('W1 - W2')
plt.ylabel('W1')
plt.title('WISE CMD for %s sources' % t)
plt.axis(ax)
ps.savefig()
sdssobj = DR9()
band = 'r'
RCF = np.unique(zip(sdss.run, sdss.camcol, sdss.field))
wcses = []
fns = []
pfn = 'wcses.pickle'
if os.path.exists(pfn):
print('Reading', pfn)
wcses, fns = unpickle_from_file(pfn)
else:
for r, c, f in RCF:
fn = sdssobj.retrieve('frame', r, c, f, band)
print('got', fn)
fns.append(fn)
wcs = Tan(fn, 0)
print('got wcs', wcs)
wcses.append(wcs)
pickle_to_file((wcses, fns), pfn)
print('Wrote to', pfn)
wisefns = glob(os.path.join(wisedatadir, 'level3', '*w1-int-3.fits'))
wisewcs = []
for fn in wisefns:
print('Reading', fn)
wcs = anwcs(fn, 0)
print('Got', wcs)
wisewcs.append(wcs)
I = np.argsort(wun.w1mpro)
wun.cut(I)
for i in range(len(wun)):
ra, dec = wun.ra[i], wun.dec[i]
insdss = -1
for j, wcs in enumerate(wcses):
if wcs.is_inside(ra, dec):
insdss = j
break
inwise = -1
for j, wcs in enumerate(wisewcs):
if wcs.is_inside(ra, dec):
inwise = j
break
N = 0
if insdss != -1:
N += 1
if inwise != -1:
N += 1
if N == 0:
continue
if N != 2:
continue
plt.clf()
ss = 1
plt.subplot(2, N, ss)
ss += 1
M = 0.02
I = np.flatnonzero((sdss.ra > (ra - M)) * (sdss.ra < (ra + M)) *
(sdss.dec > (dec - M)) * (sdss.dec < (dec + M)))
sdssnear = sdss[I]
plt.plot(sdss.ra[I], sdss.dec[I], 'b.', alpha=0.7)
I = np.flatnonzero((wise.ra > (ra - M)) * (wise.ra < (ra + M)) *
(wise.dec > (dec - M)) * (wise.dec < (dec + M)))
wisenear = wise[I]
plt.plot(wise.ra[I], wise.dec[I], 'rx', alpha=0.7)
if insdss:
wcs = wcses[j]
w, h = wcs.imagew, wcs.imageh
rd = np.array([wcs.pixelxy2radec(x, y) for x, y in
[(1, 1), (w, 1), (w, h), (1, h), (1, 1)]])
plt.plot(rd[:, 0], rd[:, 1], 'b-', alpha=0.5)
if inwise:
wcs = wisewcs[j]
w, h = wcs.imagew, wcs.imageh
rd = np.array([wcs.pixelxy2radec(x, y) for x, y in
[(1, 1), (w, 1), (w, h), (1, h), (1, 1)]])
plt.plot(rd[:, 0], rd[:, 1], 'r-', alpha=0.5)
plt.plot([ra], [dec], 'o', mec='k',
mfc='none', mew=3, ms=20, alpha=0.5)
#plt.axis([ra+M, ra-M, dec-M, dec+M])
plt.axis([ra - M, ra + M, dec - M, dec + M])
plt.xticks([ra], ['RA = %0.3f' % ra])
plt.yticks([dec], ['Dec = %0.3f' % dec])
SW = 20
ss = N + 1
plt.subplot(2, N, ss)
if insdss != -1:
ss += 1
j = insdss
wcs = wcses[j]
xc, yc = wcs.radec2pixelxy(ra, dec)
r, c, f = RCF[j]
frame = sdssobj.readFrame(r, c, f, band)
#S = 50
S = SW * 3.472
im = frame.image
H, W = im.shape
y0, x0 = max(0, yc - S), max(0, xc - S)
y1, x1 = min(H, yc + S), min(W, xc + S)
subim = im[y0:y1, x0:x1]
# plt.imshow(subim, interpolation='nearest', origin='lower',
# vmax=0.3, extent=[x0,x1,y0,y1])
plt.imshow(subim.T, interpolation='nearest', origin='lower',
vmax=0.3, extent=[y0, y1, x0, x1])
# vmax=subim.max()*1.01)
ax = plt.axis()
sx, sy = wcs.radec2pixelxy(sdssnear.ra, sdssnear.dec)
#plt.plot(x, y, 'o', mec='b', mfc='none', ms=15)
plt.plot(sy, sx, 'o', mec='b', mfc='none', ms=15)
x, y = wcs.radec2pixelxy(wisenear.ra, wisenear.dec)
#plt.plot(x, y, 'rx', ms=10)
plt.plot(y, x, 'rx', ms=10)
# Which way up?
x, y = wcs.radec2pixelxy(np.array([ra, ra]), np.array(
[0.5, 2.0]) * S * 0.396 / 3600. + dec)
#plt.plot(x, y, 'b-', alpha=0.5, lw=2)
plt.plot(y, x, 'b-', alpha=0.5, lw=2)
plt.axis(ax)
plt.gray()
plt.title('SDSS %s (%i/%i/%i)' % (band, r, c, f))
# Try to guess the PRIMARY run from the nearest object.
for I in np.argsort((sx - xc)**2 + (sy - yc)**2):
r, c, f = sdssnear.run[I], sdssnear.camcol[I], sdssnear.field[I]
jj = None
for j, (ri, ci, fi) in enumerate(RCF):
if ri == r and ci == c and fi == f:
jj = j
break
assert(jj is not None)
wcs = wcses[jj]
xc, yc = wcs.radec2pixelxy(ra, dec)
frame = sdssobj.readFrame(r, c, f, band)
S = SW * 3.472
im = frame.image
H, W = im.shape
y0, x0 = max(0, yc - S), max(0, xc - S)
y1, x1 = min(H, yc + S), min(W, xc + S)
subim = im[y0:y1, x0:x1]
if np.prod(subim.shape) == 0:
continue
print('subim shape', subim.shape)
plt.subplot(2, N, 2)
plt.imshow(subim.T, interpolation='nearest', origin='lower',
vmax=0.3, extent=[y0, y1, x0, x1])
plt.gray()
plt.title('SDSS %s (%i/%i/%i)' % (band, r, c, f))
break
if inwise != -1:
plt.subplot(2, N, ss)
ss += 1
j = inwise
wcs = wisewcs[j]
ok, x, y = wcs.radec2pixelxy(ra, dec)
im = pyfits.open(wisefns[j])[0].data
S = SW
H, W = im.shape
y0, x0 = max(0, y - S), max(0, x - S)
subim = im[y0: min(H, y + S), x0: min(W, x + S)]
plt.imshow(subim, interpolation='nearest', origin='lower',
vmax=subim.max() * 1.01)
ax = plt.axis()
x, y = [], []
for r, d in zip(wisenear.ra, wisenear.dec):
ok, xi, yi = wcs.radec2pixelxy(r, d)
x.append(xi)
y.append(yi)
x = np.array(x)
y = np.array(y)
plt.plot(x - x0, y - y0, 'rx', ms=15)
x, y = [], []
for r, d in zip(sdssnear.ra, sdssnear.dec):
ok, xi, yi = wcs.radec2pixelxy(r, d)
x.append(xi)
y.append(yi)
x = np.array(x)
y = np.array(y)
plt.plot(x - x0, y - y0, 'o', mec='b', mfc='none', ms=10)
# Which way up?
pixscale = 1.375 / 3600.
ok, x1, y1 = wcs.radec2pixelxy(ra, dec + 0.5 * S * pixscale)
ok, x2, y2 = wcs.radec2pixelxy(ra, dec + 2.0 * S * pixscale)
plt.plot([x1 - x0, x2 - x0], [y1 - y0, y2 - y0],
'r-', alpha=0.5, lw=2)
plt.axis([ax[1], ax[0], ax[2], ax[3]])
# plt.axis(ax)
plt.gray()
plt.title('WISE W1 (coadd)')
plt.suptitle('WISE unmatched source: w1=%.1f, RA,Dec = (%.3f, %.3f)' %
(wun.w1mpro[i], ra, dec))
ps.savefig()
rcfs = zip(sdssnear.run, sdssnear.camcol, sdssnear.field)
print('Nearby SDSS sources are from:', np.unique(rcfs))
return
def forcedphot():
T1 = fits_table('cs82data/cas-primary-DR8.fits')
print(len(T1), 'primary')
T1.cut(T1.nchild == 0)
print(len(T1), 'children')
rl, rh = T1.ra.min(), T1.ra.max()
dl, dh = T1.dec.min(), T1.dec.max()
tims = []
# Coadd
basedir = os.path.join('cs82data', 'wise', 'level3')
basefn = os.path.join(basedir, '3342p000_ab41-w1')
tim = read_wise_coadd(basefn, radecroi=[rl, rh, dl, dh], nanomaggies=True)
tims.append(tim)
# Individuals
basedir = os.path.join('cs82data', 'wise', 'level1b')
for fn in ['04933b137-w1', '04937b137-w1', '04941b137-w1', '04945b137-w1', '04948a112-w1',
'04949b137-w1', '04952a112-w1', '04953b137-w1', '04956a112-w1', '04960a112-w1',
'04964a112-w1', '04968a112-w1', '05204a106-w1']:
basefn = os.path.join(basedir, fn)
tim = read_wise_image(
basefn, radecroi=[rl, rh, dl, dh], nanomaggies=True)
tims.append(tim)
# tractor.Image's setMask() does a binary dilation on bad pixels!
for tim in tims:
# tim.mask = np.zeros(tim.shape, dtype=bool)
tim.invvar = tim.origInvvar
tim.mask = np.zeros(tim.shape, dtype=bool)
print('tim:', tim)
ps = PlotSequence('forced')
plt.clf()
plt.plot(T1.ra, T1.dec, 'r.')
for tim in tims:
wcs = tim.getWcs()
H, W = tim.shape
rr, dd = [], []
for x, y in zip([1, 1, W, W, 1], [1, H, H, 1, 1]):
rd = wcs.pixelToPosition(x, y)
rr.append(rd.ra)
dd.append(rd.dec)
plt.plot(rr, dd, 'k-', alpha=0.5)
# setRadecAxes(rl,rh,dl,dh)
ps.savefig()
T2 = fits_table('wise-cut.fits')
T2.w1 = T2.w1mpro
R = 1. / 3600.
I, J, d = match_radec(T1.ra, T1.dec, T2.ra, T2.dec, R)
print(len(I), 'matches')
refband = 'r'
#bandnum = band_index('r')
Lstar = (T1.probpsf == 1) * 1.0
Lgal = (T1.probpsf == 0)
fracdev = T1.get('fracdev_%s' % refband)
Ldev = Lgal * fracdev
Lexp = Lgal * (1. - fracdev)
ndev, nexp, ncomp, nstar = 0, 0, 0, 0
cat = Catalog()
# for i,t in enumerate(T1):
jmatch = np.zeros(len(T1))
jmatch[:] = -1
jmatch[I] = J
for i in range(len(T1)):
j = jmatch[i]
if j >= 0:
# match source: grab WISE catalog mag
w1 = T2.w1[j]
else:
# unmatched: set it faint
w1 = 18.
bright = NanoMaggies(w1=NanoMaggies.magToNanomaggies(w1))
pos = RaDecPos(T1.ra[i], T1.dec[i])
if Lstar[i] > 0:
# Star
star = PointSource(pos, bright)
cat.append(star)
nstar += 1
continue
hasdev = (Ldev[i] > 0)
hasexp = (Lexp[i] > 0)
iscomp = (hasdev and hasexp)
if iscomp:
dbright = bright * Ldev[i]
ebright = bright * Lexp[i]
elif hasdev:
dbright = bright
elif hasexp:
ebright = bright
else:
assert(False)
if hasdev:
re = T1.get('devrad_%s' % refband)[i]
ab = T1.get('devab_%s' % refband)[i]
phi = T1.get('devphi_%s' % refband)[i]
dshape = GalaxyShape(re, ab, phi)
if hasexp:
re = T1.get('exprad_%s' % refband)[i]
ab = T1.get('expab_%s' % refband)[i]
phi = T1.get('expphi_%s' % refband)[i]
eshape = GalaxyShape(re, ab, phi)
if iscomp:
gal = CompositeGalaxy(pos, ebright, eshape, dbright, dshape)
ncomp += 1
elif hasdev:
gal = DevGalaxy(pos, dbright, dshape)
ndev += 1
elif hasexp:
gal = ExpGalaxy(pos, ebright, eshape)
nexp += 1
cat.append(gal)
print('Created', ndev, 'pure deV', nexp, 'pure exp and', end=' ')
print(ncomp, 'composite galaxies', end=' ')
print('and', nstar, 'stars')
tractor = Tractor(tims, cat)
for i, tim in enumerate(tims):
ima = dict(interpolation='nearest', origin='lower',
vmin=tim.zr[0], vmax=tim.zr[1])
mod = tractor.getModelImage(i)
plt.clf()
plt.imshow(mod, **ima)
plt.gray()
plt.title('model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(tim.getImage(), **ima)
plt.gray()
plt.title('data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(tim.getInvvar(), interpolation='nearest', origin='lower')
plt.gray()
plt.title('invvar')
ps.savefig()
# for tim in tims:
wcs = tim.getWcs()
H, W = tim.shape
poly = []
for r, d in zip([rl, rl, rh, rh, rl], [dl, dh, dh, dl, dl]):
x, y = wcs.positionToPixel(RaDecPos(r, d))
poly.append((x, y))
xx, yy = np.meshgrid(np.arange(W), np.arange(H))
xy = np.vstack((xx.flat, yy.flat)).T
grid = points_inside_poly(xy, poly)
grid = grid.reshape((H, W))
tim.setInvvar(tim.getInvvar() * grid)
# plt.clf()
# plt.imshow(grid, interpolation='nearest', origin='lower')
# plt.gray()
# ps.savefig()
plt.clf()
plt.imshow(tim.getInvvar(), interpolation='nearest', origin='lower')
plt.gray()
plt.title('invvar')
ps.savefig()
if i == 1:
plt.clf()
plt.imshow(tim.goodmask, interpolation='nearest', origin='lower')
plt.gray()
plt.title('goodmask')
ps.savefig()
miv = (1. / (tim.uncplane)**2)
for bit in range(-1, 32):
if bit >= 0:
miv[(tim.maskplane & (1 << bit)) != 0] = 0.
if bit == 31:
plt.clf()
plt.imshow(miv, interpolation='nearest', origin='lower')
plt.gray()
plt.title(
'invvar with mask bits up to %i blanked out' % bit)
ps.savefig()
def main():
# ps = PlotSequence('pro')
# star_profiles(ps)
# sys.exit(0)
#survey_dir = '/project/projectdirs/desiproc/dr3'
#survey = LegacySurveyData(survey_dir=survey_dir)
survey = LegacySurveyData()
ralo,rahi = 240,245
declo,dechi = 5, 12
ps = PlotSequence('comp')
bands = 'grz'
ccdfn = 'ccds-forced.fits'
if not os.path.exists(ccdfn):
ccds = survey.get_annotated_ccds()
ccds.cut((ccds.ra > ralo) * (ccds.ra < rahi) *
(ccds.dec > declo) * (ccds.dec < dechi))
print(len(ccds), 'CCDs')
ccds.path = np.array([os.path.join(#'dr3',
'forced', ('%08i' % e)[:5], '%08i' % e, 'decam-%08i-%s-forced.fits' % (e, n.strip()))
for e,n in zip(ccds.expnum, ccds.ccdname)])
I, = np.nonzero([os.path.exists(fn) for fn in ccds.path])
print(len(I), 'CCDs with forced photometry')
ccds.cut(I)
#ccds = ccds[:500]
#e,I = np.unique(ccds.expnum, return_index=True)
#print(len(I), 'unique exposures')
#ccds.cut(I)
FF = read_forcedphot_ccds(ccds, survey)
FF.writeto('forced-all-matches.fits')
# - z band -- no trend w/ PS1 mag (brighter-fatter)
ccds.writeto(ccdfn)
ccdfn2 = 'ccds-forced-2.fits'
if not os.path.exists(ccdfn2):
ccds = fits_table(ccdfn)
# Split into brighter/fainter halves
FF = fits_table('forced-all-matches.fits')
print(len(FF), 'forced measurements')
FF.cut(FF.masked == False)
print(len(FF), 'forced measurements not masked')
ccds.brightest_mdiff = np.zeros(len(ccds))
ccds.brightest_mscatter = np.zeros(len(ccds))
ccds.bright_mdiff = np.zeros(len(ccds))
ccds.bright_mscatter = np.zeros(len(ccds))
ccds.faint_mdiff = np.zeros(len(ccds))
ccds.faint_mscatter = np.zeros(len(ccds))
for iccd in range(len(ccds)):
I = np.flatnonzero(FF.iforced == iccd)
if len(I) == 0:
continue
if len(I) < 10:
continue
F = FF[I]
b = np.percentile(F.psmag, 10)
m = np.median(F.psmag)
print(len(F), 'matches for CCD', iccd, 'median mag', m, '10th pct', b)
J = np.flatnonzero(F.psmag < b)
diff = F.mag[J] - F.psmag[J]
ccds.brightest_mdiff[iccd] = np.median(diff)
ccds.brightest_mscatter[iccd] = (np.percentile(diff, 84) -
np.percentile(diff, 16))/2.
J = np.flatnonzero(F.psmag < m)
diff = F.mag[J] - F.psmag[J]
ccds.bright_mdiff[iccd] = np.median(diff)
ccds.bright_mscatter[iccd] = (np.percentile(diff, 84) -
np.percentile(diff, 16))/2.
J = np.flatnonzero(F.psmag > m)
diff = F.mag[J] - F.psmag[J]
ccds.faint_mdiff[iccd] = np.median(diff)
ccds.faint_mscatter[iccd] = (np.percentile(diff, 84) -
np.percentile(diff, 16))/2.
ccds.writeto(ccdfn2)
ccds = fits_table(ccdfn2)
plt.clf()
plt.hist(ccds.nforced, bins=100)
plt.title('nforced')
ps.savefig()
plt.clf()
plt.hist(ccds.nmatched, bins=100)
plt.title('nmatched')
ps.savefig()
#ccds.cut(ccds.nmatched >= 150)
ccds.cut(ccds.nmatched >= 50)
print('Cut to', len(ccds), 'with >50 matched')
ccds.cut(ccds.photometric)
print('Cut to', len(ccds), 'photometric')
neff = 1. / ccds.psfnorm_mean**2
# Narcsec is in arcsec**2
narcsec = neff * ccds.pixscale_mean**2
# to arcsec
narcsec = np.sqrt(narcsec)
# Correction factor to get back to equivalent of Gaussian sigma
narcsec /= (2. * np.sqrt(np.pi))
# Conversion factor to FWHM (2.35)
narcsec *= 2. * np.sqrt(2. * np.log(2.))
ccds.psfsize = narcsec
for band in bands:
I = np.flatnonzero((ccds.filter == band)
* (ccds.photometric) * (ccds.blacklist_ok))
mlo,mhi = -0.01, 0.05
plt.clf()
plt.plot(ccds.ccdzpt[I],
ccds.exptime[I], 'k.', alpha=0.1)
J = np.flatnonzero((ccds.filter == band) * (ccds.photometric == False))
plt.plot(ccds.ccdzpt[J],
ccds.exptime[J], 'r.', alpha=0.1)
plt.xlabel('Zeropoint (mag)')
plt.ylabel('exptime')
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
plt.clf()
plt.plot(ccds.ccdzpt[I],
np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
plt.xlabel('Zeropoint (mag)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
#plt.axis([0, mxsee, mlo,mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
plt.clf()
plt.plot(ccds.ccdzpt[I], ccds.psfsize[I], 'k.', alpha=0.1)
plt.xlabel('Zeropoint (mag)')
plt.ylabel('PSF size (arcsec)')
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
# plt.clf()
# plt.plot(ccds.avsky[I],
# np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
# plt.xlabel('avsky')
# plt.ylabel('DECaLS PSF - PS1 (mag)')
# plt.axhline(0, color='k', alpha=0.2)
# plt.title('DR3: EDR region, Forced phot: %s band' % band)
# ps.savefig()
#
# plt.clf()
# plt.plot(ccds.meansky[I],
# np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
# plt.xlabel('meansky')
# plt.ylabel('DECaLS PSF - PS1 (mag)')
# plt.axhline(0, color='k', alpha=0.2)
# plt.title('DR3: EDR region, Forced phot: %s band' % band)
# ps.savefig()
plt.clf()
lo,hi = (-0.02, 0.05)
ha = dict(bins=50, histtype='step', range=(lo,hi))
n,b,p1 = plt.hist(ccds.brightest_mdiff[I], color='r', **ha)
n,b,p2 = plt.hist(ccds.bright_mdiff[I], color='g', **ha)
n,b,p3 = plt.hist(ccds.faint_mdiff[I], color='b', **ha)
plt.legend((p1[0],p2[0],p3[0]), ('Brightest 10%', 'Brightest 50%',
'Faintest 50%'))
plt.xlabel('DECaLS PSF - PS1 (mag)')
plt.ylabel('Number of CCDs')
plt.title('DR3: EDR region, Forced phot: %s band' % band)
plt.xlim(lo,hi)
ps.savefig()
for band in bands:
I = np.flatnonzero(ccds.filter == band)
mxsee = 4.
mlo,mhi = -0.01, 0.05
plt.clf()
plt.plot(np.clip(ccds.psfsize[I], 0, mxsee),
np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
# for p in [1,2,3]:
# J = np.flatnonzero(ccds.tilepass[I] == p)
# if len(J):
# plt.plot(np.clip(ccds.psfsize[I[J]], 0, mxsee),
# np.clip(ccds.mdiff[I[J]], mlo,mhi), '.', color='rgb'[p-1], alpha=0.2)
#plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.')
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
plt.axis([0, mxsee, mlo,mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
# Group by exposure
for band in bands:
I = np.flatnonzero((ccds.filter == band)
* (ccds.photometric) * (ccds.blacklist_ok))
E,J = np.unique(ccds.expnum[I], return_index=True)
print(len(E), 'unique exposures in', band)
exps = ccds[I[J]]
print(len(exps), 'unique exposures in', band)
assert(len(np.unique(exps.expnum)) == len(exps))
exps.ddiff = np.zeros(len(exps))
exps.dsize = np.zeros(len(exps))
exps.nccds = np.zeros(len(exps), int)
exps.brightest_ddiff = np.zeros(len(exps))
exps.bright_ddiff = np.zeros(len(exps))
exps.faint_ddiff = np.zeros(len(exps))
for iexp,exp in enumerate(exps):
J = np.flatnonzero(ccds.expnum[I] == exp.expnum)
J = I[J]
print(len(J), 'CCDs in exposure', exp.expnum)
exps.brightest_mdiff[iexp] = np.median(ccds.brightest_mdiff[J])
exps.bright_mdiff[iexp] = np.median(ccds.bright_mdiff[J])
exps.faint_mdiff[iexp] = np.median(ccds.faint_mdiff[J])
exps.brightest_ddiff[iexp] = (
np.percentile(ccds.brightest_mdiff[J], 84) -
np.percentile(ccds.brightest_mdiff[J], 16))/2.
exps.bright_ddiff[iexp] = (
np.percentile(ccds.bright_mdiff[J], 84) -
np.percentile(ccds.bright_mdiff[J], 16))/2.
exps.faint_ddiff[iexp] = (
np.percentile(ccds.faint_mdiff[J], 84) -
np.percentile(ccds.faint_mdiff[J], 16))/2.
exps.mdiff[iexp] = np.median(ccds.mdiff[J])
exps.ddiff[iexp] = (np.percentile(ccds.mdiff[J], 84) - np.percentile(ccds.mdiff[J], 16))/2.
exps.psfsize[iexp] = np.median(ccds.psfsize[J])
exps.dsize[iexp] = (np.percentile(ccds.psfsize[J], 84) - np.percentile(ccds.psfsize[J], 16))/2.
exps.nccds[iexp] = len(J)
mxsee = 4.
mlo,mhi = -0.01, 0.05
exps.cut(exps.nccds >= 10)
plt.clf()
plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.mdiff, mlo,mhi), yerr=exps.ddiff,
#xerr=exps.dsize,
fmt='.', color='k')
# plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.brightest_mdiff, mlo,mhi),
# yerr=exps.brightest_ddiff, fmt='r.')
# plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.bright_mdiff, mlo,mhi),
# yerr=exps.bright_ddiff, fmt='g.')
# plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.faint_mdiff, mlo,mhi),
# yerr=exps.faint_ddiff, fmt='b.')
# plt.plot(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.brightest_mdiff, mlo,mhi), 'r.')
# plt.plot(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.bright_mdiff, mlo,mhi), 'g.')
# plt.plot(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.faint_mdiff, mlo,mhi), 'b.')
#plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.')
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
plt.axis([0, mxsee, mlo,mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
plt.clf()
p1 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.brightest_mdiff, mlo,mhi), 'r.', alpha=0.5)
p2 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.bright_mdiff, mlo,mhi), 'g.', alpha=0.5)
p3 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.faint_mdiff, mlo,mhi), 'b.', alpha=0.5)
plt.legend((p1[0],p2[0],p3[0]), ('Brightest 10%', 'Brightest 50%',
'Faintest 50%'))
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
plt.axis([0, mxsee, mlo,mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
J = np.argsort(-exps.mdiff)
for j in J:
print(' Photometric diff', exps.mdiff[j], 'PSF size', exps.psfsize[j], 'expnum', exps.expnum[j])
sys.exit(0)
def unwise_forcedphot(cat,
tiles,
band=1,
roiradecbox=None,
use_ceres=True,
ceres_block=8,
save_fits=False,
get_models=False,
ps=None,
psf_broadening=None,
pixelized_psf=False,
get_masks=None,
move_crpix=False,
modelsky_dir=None,
tag=None):
'''
Given a list of tractor sources *cat*
and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
runs forced photometry, returning a FITS table the same length as *cat*.
*get_masks*: the WCS to resample mask bits into.
'''
from tractor import PointSource, Tractor, ExpGalaxy, DevGalaxy
from tractor.sersic import SersicGalaxy
if tag is None:
tag = ''
else:
tag = tag + ': '
if not pixelized_psf and psf_broadening is None:
# PSF broadening in post-reactivation data, by band.
# Newer version from Aaron's email to decam-chatter, 2018-06-14.
broadening = {1: 1.0405, 2: 1.0346, 3: None, 4: None}
psf_broadening = broadening[band]
if False:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('wise-forced-w%i' % band)
plots = (ps is not None)
if plots:
import pylab as plt
wantims = (plots or save_fits or get_models)
wanyband = 'w'
if get_models:
models = []
wband = 'w%i' % band
Nsrcs = len(cat)
phot = fits_table()
# Filled in based on unique tile overlap
phot.wise_coadd_id = np.array([' '] * Nsrcs, dtype='U8')
phot.wise_x = np.zeros(Nsrcs, np.float32)
phot.wise_y = np.zeros(Nsrcs, np.float32)
phot.set('psfdepth_%s' % wband, np.zeros(Nsrcs, np.float32))
nexp = np.zeros(Nsrcs, np.int16)
mjd = np.zeros(Nsrcs, np.float64)
central_flux = np.zeros(Nsrcs, np.float32)
ra = np.array([src.getPosition().ra for src in cat])
dec = np.array([src.getPosition().dec for src in cat])
fskeys = ['prochi2', 'profracflux']
fitstats = {}
if get_masks:
mh, mw = get_masks.shape
maskmap = np.zeros((mh, mw), np.uint32)
tims = []
for tile in tiles:
info(tag + 'Reading WISE tile', tile.coadd_id, 'band', band)
tim = get_unwise_tractor_image(tile.unwise_dir,
tile.coadd_id,
band,
bandname=wanyband,
roiradecbox=roiradecbox)
if tim is None:
debug('Actually, no overlap with WISE coadd tile', tile.coadd_id)
continue
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage(),
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data' % tag)
ps.savefig()
plt.clf()
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
range=(-5, 10),
bins=100)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.title(tag)
ps.savefig()
if move_crpix and band in [1, 2]:
realwcs = tim.wcs.wcs
x, y = realwcs.crpix
tile_crpix = tile.get('crpix_w%i' % band)
dx = tile_crpix[0] - 1024.5
dy = tile_crpix[1] - 1024.5
realwcs.set_crpix(x + dx, y + dy)
debug('unWISE', tile.coadd_id, 'band', band, 'CRPIX', x, y,
'shift by', dx, dy, 'to', realwcs.crpix)
if modelsky_dir and band in [1, 2]:
fn = os.path.join(modelsky_dir,
'%s.%i.mod.fits' % (tile.coadd_id, band))
if not os.path.exists(fn):
raise RuntimeError('WARNING: does not exist:', fn)
x0, x1, y0, y1 = tim.roi
bg = fitsio.FITS(fn)[2][y0:y1, x0:x1]
assert (bg.shape == tim.shape)
if plots:
plt.clf()
plt.subplot(1, 2, 1)
plt.imshow(tim.getImage(),
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=5 * sig1)
plt.subplot(1, 2, 2)
plt.imshow(bg,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=5 * sig1)
tag = '%s W%i' % (tile.coadd_id, band)
plt.suptitle(tag)
ps.savefig()
plt.clf()
ha = dict(range=(-5, 10), bins=100, histtype='step')
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
color='b',
label='Original',
**ha)
plt.hist(((tim.getImage() - bg) *
tim.inverr)[tim.inverr > 0].ravel(),
color='g',
label='Minus Background',
**ha)
plt.axvline(0, color='k', alpha=0.5)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.legend()
plt.title(tag + ': background')
ps.savefig()
# Actually subtract the background!
tim.data -= bg
# Floor the per-pixel variances,
# and add Poisson contribution from sources
if band in [1, 2]:
# in Vega nanomaggies per pixel
floor_sigma = {1: 0.5, 2: 2.0}
poissons = {1: 0.15, 2: 0.3}
with np.errstate(divide='ignore'):
new_ie = 1. / np.sqrt(
(1. / tim.inverr)**2 + floor_sigma[band] +
poissons[band]**2 * np.maximum(0., tim.data))
new_ie[tim.inverr == 0] = 0.
if plots:
plt.clf()
plt.plot((1. / tim.inverr[tim.inverr > 0]).ravel(),
(1. / new_ie[tim.inverr > 0]).ravel(), 'b.')
plt.title('unWISE per-pixel error: %s band %i' %
(tile.coadd_id, band))
plt.xlabel('original')
plt.ylabel('floored')
ps.savefig()
assert (np.all(np.isfinite(new_ie)))
assert (np.all(new_ie >= 0.))
tim.inverr = new_ie
# Expand a 3-pixel radius around weight=0 (saturated) pixels
# from Eddie via crowdsource
# https://github.com/schlafly/crowdsource/blob/7069da3e7d9d3124be1cbbe1d21ffeb63fc36dcc/python/wise_proc.py#L74
## FIXME -- W3/W4 ??
satlimit = 85000
msat = ((tim.data > satlimit) | ((tim.nims == 0) &
(tim.nuims > 1)))
from scipy.ndimage.morphology import binary_dilation
xx, yy = np.mgrid[-3:3 + 1, -3:3 + 1]
dilate = xx**2 + yy**2 <= 3**2
msat = binary_dilation(msat, dilate)
nbefore = np.sum(tim.inverr == 0)
tim.inverr[msat] = 0
nafter = np.sum(tim.inverr == 0)
debug('Masking an additional', (nafter - nbefore),
'near-saturated pixels in unWISE', tile.coadd_id, 'band',
band)
# Read mask file?
if get_masks:
from astrometry.util.resample import resample_with_wcs, OverlapError
# unwise_dir can be a colon-separated list of paths
tilemask = None
for d in tile.unwise_dir.split(':'):
fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id,
'unwise-%s-msk.fits.gz' % tile.coadd_id)
if os.path.exists(fn):
debug('Reading unWISE mask file', fn)
x0, x1, y0, y1 = tim.roi
tilemask = fitsio.FITS(fn)[0][y0:y1, x0:x1]
break
if tilemask is None:
info('unWISE mask file for tile', tile.coadd_id,
'does not exist')
else:
try:
tanwcs = tim.wcs.wcs
assert (tanwcs.shape == tilemask.shape)
Yo, Xo, Yi, Xi, _ = resample_with_wcs(get_masks,
tanwcs,
intType=np.int16)
# Only deal with mask pixels that are set.
I, = np.nonzero(tilemask[Yi, Xi] > 0)
# Trim to unique area for this tile
rr, dd = get_masks.pixelxy2radec(Xo[I] + 1, Yo[I] + 1)
good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2,
tile.dec1, tile.dec2)
I = I[good]
maskmap[Yo[I], Xo[I]] = tilemask[Yi[I], Xi[I]]
except OverlapError:
# Shouldn't happen by this point
print('Warning: no overlap between WISE tile',
tile.coadd_id, 'and brick')
if plots:
plt.clf()
plt.imshow(tilemask, interpolation='nearest', origin='lower')
plt.title('Tile %s: mask' % tile.coadd_id)
ps.savefig()
plt.clf()
plt.imshow(maskmap, interpolation='nearest', origin='lower')
plt.title('Tile %s: accumulated maskmap' % tile.coadd_id)
ps.savefig()
# The tiles have some overlap, so zero out pixels outside the
# tile's unique area.
th, tw = tim.shape
xx, yy = np.meshgrid(np.arange(tw), np.arange(th))
rr, dd = tim.wcs.wcs.pixelxy2radec(xx + 1, yy + 1)
unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1,
tile.dec2)
debug('Tile', tile.coadd_id, '- total of', np.sum(unique),
'unique pixels out of', len(unique.flat), 'total pixels')
if get_models:
# Save the inverr before blanking out non-unique pixels, for making coadds with no gaps!
# (actually, slightly more subtly, expand unique area by 1 pixel)
from scipy.ndimage.morphology import binary_dilation
du = binary_dilation(unique)
tim.coadd_inverr = tim.inverr * du
tim.inverr[unique == False] = 0.
del xx, yy, rr, dd, unique
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage() * (tim.inverr > 0),
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data (unique)' % tag)
ps.savefig()
if pixelized_psf:
from unwise_psf import unwise_psf
if (band == 1) or (band == 2):
# we only have updated PSFs for W1 and W2
psfimg = unwise_psf.get_unwise_psf(band,
tile.coadd_id,
modelname='neo6_unwisecat')
else:
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id)
if band == 4:
# oversample (the unwise_psf models are at native W4 5.5"/pix,
# while the unWISE coadds are made at 2.75"/pix.
ph, pw = psfimg.shape
subpsf = np.zeros((ph * 2 - 1, pw * 2 - 1), np.float32)
from astrometry.util.util import lanczos3_interpolate
xx, yy = np.meshgrid(
np.arange(0., pw - 0.51, 0.5, dtype=np.float32),
np.arange(0., ph - 0.51, 0.5, dtype=np.float32))
xx = xx.ravel()
yy = yy.ravel()
ix = xx.astype(np.int32)
iy = yy.astype(np.int32)
dx = (xx - ix).astype(np.float32)
dy = (yy - iy).astype(np.float32)
psfimg = psfimg.astype(np.float32)
rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat],
[psfimg])
if plots:
plt.clf()
plt.imshow(psfimg, interpolation='nearest', origin='lower')
plt.title('Original PSF model')
ps.savefig()
plt.clf()
plt.imshow(subpsf, interpolation='nearest', origin='lower')
plt.title('Subsampled PSF model')
ps.savefig()
psfimg = subpsf
del xx, yy, ix, iy, dx, dy
from tractor.psf import PixelizedPSF
psfimg /= psfimg.sum()
fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0}
psfimg *= fluxrescales[band]
tim.psf = PixelizedPSF(psfimg)
if psf_broadening is not None and not pixelized_psf:
# psf_broadening is a factor by which the PSF FWHMs
# should be scaled; the PSF is a little wider
# post-reactivation.
psf = tim.getPsf()
from tractor import GaussianMixturePSF
if isinstance(psf, GaussianMixturePSF):
debug('Broadening PSF: from', psf)
p0 = psf.getParams()
pnames = psf.getParamNames()
p1 = [
p * psf_broadening**2 if 'var' in name else p
for (p, name) in zip(p0, pnames)
]
psf.setParams(p1)
debug('Broadened PSF:', psf)
else:
print(
'WARNING: cannot apply psf_broadening to WISE PSF of type',
type(psf))
wcs = tim.wcs.wcs
_, fx, fy = wcs.radec2pixelxy(ra, dec)
x = np.round(fx - 1.).astype(int)
y = np.round(fy - 1.).astype(int)
good = (x >= 0) * (x < tw) * (y >= 0) * (y < th)
# Which sources are in this brick's unique area?
usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1,
tile.dec2)
I, = np.nonzero(good * usrc)
nexp[I] = tim.nuims[y[I], x[I]]
if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
mjd[I] = (tim.mjdmin + tim.mjdmax) / 2.
phot.wise_coadd_id[I] = tile.coadd_id
phot.wise_x[I] = fx[I] - 1.
phot.wise_y[I] = fy[I] - 1.
central_flux[I] = tim.getImage()[y[I], x[I]]
del x, y, good, usrc
# PSF norm for depth
psf = tim.getPsf()
h, w = tim.shape
patch = psf.getPointSourcePatch(h // 2, w // 2).patch
psfnorm = np.sqrt(np.sum(patch**2))
# To handle zero-depth, we return 1/nanomaggies^2 units rather than mags.
# In the small empty patches of the sky (eg W4 in 0922p702), we get sig1 = NaN
if np.isfinite(tim.sig1):
phot.get('psfdepth_%s' % wband)[I] = 1. / (tim.sig1 / psfnorm)**2
tim.tile = tile
tims.append(tim)
if plots:
plt.clf()
mn, mx = 0.1, 20000
plt.hist(np.log10(np.clip(central_flux, mn, mx)),
bins=100,
range=(np.log10(mn), np.log10(mx)))
logt = np.arange(0, 5)
plt.xticks(logt, ['%i' % i for i in 10.**logt])
plt.title('Central fluxes (W%i)' % band)
plt.axvline(np.log10(20000), color='k')
plt.axvline(np.log10(1000), color='k')
ps.savefig()
# Eddie's non-secret recipe:
#- central pixel <= 1000: 19x19 pix box size
#- central pixel in 1000 - 20000: 59x59 box size
#- central pixel > 20000 or saturated: 149x149 box size
#- object near "bright star": 299x299 box size
nbig = nmedium = nsmall = 0
for src, cflux in zip(cat, central_flux):
if cflux > 20000:
R = 100
nbig += 1
elif cflux > 1000:
R = 30
nmedium += 1
else:
R = 15
nsmall += 1
if isinstance(src, PointSource):
src.fixedRadius = R
else:
### FIXME -- sizes for galaxies..... can we set PSF size separately?
galrad = 0
# RexGalaxy is a subclass of ExpGalaxy
if isinstance(src, (ExpGalaxy, DevGalaxy, SersicGalaxy)):
galrad = src.shape.re
pixscale = 2.75
src.halfsize = int(np.hypot(R, galrad * 5 / pixscale))
debug('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall,
'small')
tractor = Tractor(tims, cat)
if use_ceres:
from tractor.ceres_optimizer import CeresOptimizer
tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block)
tractor.freezeParamsRecursive('*')
tractor.thawPathsTo(wanyband)
t0 = Time()
R = tractor.optimize_forced_photometry(fitstats=True,
variance=True,
shared_params=False,
wantims=wantims)
info(tag + 'unWISE forced photometry took', Time() - t0)
if use_ceres:
term = R.ceres_status['termination']
# Running out of memory can cause failure to converge and term
# status = 2. Fail completely in this case.
if term != 0:
info(tag + 'Ceres termination status:', term)
raise RuntimeError('Ceres terminated with status %i' % term)
if wantims:
ims1 = R.ims1
# can happen if empty source list (we still want to generate coadds)
if ims1 is None:
ims1 = R.ims0
flux_invvars = R.IV
if R.fitstats is not None:
for k in fskeys:
x = getattr(R.fitstats, k)
fitstats[k] = np.array(x).astype(np.float32)
if save_fits:
for i, tim in enumerate(tims):
tile = tim.tile
(dat, mod, _, chi, _) = ims1[i]
wcshdr = fitsio.FITSHDR()
tim.wcs.wcs.add_to_header(wcshdr)
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' % tag,
dat,
clobber=True,
header=wcshdr)
fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr)
fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr)
if plots:
# Create models for just the brightest sources
bright_cat = [
src for src in cat if src.getBrightness().getBand(wanyband) > 1000
]
debug('Bright soures:', len(bright_cat))
btr = Tractor(tims, bright_cat)
for tim in tims:
mod = btr.getModelImage(tim)
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
sig1 = tim.sig1
plt.clf()
plt.imshow(mod,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=25 * sig1)
plt.colorbar()
plt.title('%s: bright-star models' % tag)
ps.savefig()
if get_models:
for i, tim in enumerate(tims):
tile = tim.tile
(dat, mod, _, _, _) = ims1[i]
models.append(
(tile.coadd_id, band, tim.wcs.wcs, dat, mod, tim.coadd_inverr))
if plots:
for i, tim in enumerate(tims):
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
(dat, mod, _, chi, _) = ims1[i]
sig1 = tim.sig1
plt.clf()
plt.imshow(dat,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=25 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
plt.clf()
plt.imshow(mod,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=25 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-5,
vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = flux_invvars
# Sources out of bounds, eg, never change from their initial
# fluxes. Zero them out instead.
nm[nm_ivar == 0] = 0.
phot.set('flux_%s' % wband, nm.astype(np.float32))
phot.set('flux_ivar_%s' % wband, nm_ivar.astype(np.float32))
for k in fskeys:
phot.set(k + '_' + wband,
fitstats.get(k, np.zeros(len(phot), np.float32)))
phot.set('nobs_%s' % wband, nexp)
phot.set('mjd_%s' % wband, mjd)
rtn = wphotduck()
rtn.phot = phot
rtn.models = None
rtn.maskmap = None
if get_models:
rtn.models = models
if get_masks:
rtn.maskmap = maskmap
return rtn
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--plots', action='store_true')
parser.add_argument('--brick', help='Brick name to run')
parser.add_argument(
'--input-dir',
default='/global/projecta/projectdirs/cosmo/work/legacysurvey/dr7')
#/global/cscratch1/sd/desiproc/dr7out')
parser.add_argument('--survey-dir',
default='/global/cscratch1/sd/dstn/dr7-depthcut')
parser.add_argument('--output-dir',
default='/global/cscratch1/sd/dstn/bright')
opt = parser.parse_args()
plots = opt.plots
ps = PlotSequence('bright')
brickname = opt.brick
insurvey = LegacySurveyData(opt.input_dir, cache_dir=opt.survey_dir)
outsurvey = LegacySurveyData(opt.output_dir, output_dir=opt.output_dir)
bfn = insurvey.find_file('blobmap', brick=brickname)
print('Found blob map', bfn)
blobs = fitsio.read(bfn)
h, w = blobs.shape
brick = insurvey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brick)
radius = np.sqrt(2.) * 0.25 * 1.01
neighbors = insurvey.get_bricks_near(brick.ra, brick.dec, radius)
print(len(neighbors), 'bricks nearby')
def showbool(X):
d = downsample_max(X, 8)
h, w = X.shape
plt.imshow(d,
interpolation='nearest',
origin='lower',
vmin=0,
vmax=1,
extent=[0, w, 0, h],
cmap='gray')
brightblobs = set()
for nb in neighbors:
if nb.brickname == brickname:
# ignore myself!
continue
print('Neighbor:', nb.brickname)
mfn = insurvey.find_file('maskbits', brick=nb.brickname)
if not os.path.exists(mfn):
print('No maskbits file:', mfn)
continue
maskbits = fitsio.read(mfn)
bright = ((maskbits & MASKBITS['BRIGHT']) > 0)
print(np.sum(bright > 0), 'BRIGHT pixels set')
primary = (maskbits & MASKBITS['NPRIMARY'] == 0)
print(np.sum(primary), 'PRIMARY pixels set')
edge = binary_dilation(primary, structure=np.ones((3, 3), bool))
edge = edge * np.logical_not(primary)
brightedge = edge & bright
if plots:
plt.clf()
showbool(bright)
plt.title('bright: brick %s' % nb.brickname)
ps.savefig()
# plt.clf()
# showbool(primary)
# plt.title('PRIMARY, brick %s' % nb.brickname)
# ps.savefig()
#
# plt.clf()
# showbool(edge)
# plt.title('boundary, brick %s' % nb.brickname)
# ps.savefig()
plt.clf()
showbool(brightedge)
plt.title('bright at edge, brick %s' % nb.brickname)
ps.savefig()
nwcs = wcs_for_brick(nb)
yy, xx = np.nonzero(brightedge)
print(len(yy), 'bright edge pixels')
if len(yy) == 0:
continue
rr, dd = nwcs.pixelxy2radec(xx + 1, yy + 1)
print('RA range', rr.min(), rr.max(), 'vs brick', brick.ra1, brick.ra2)
print('Dec range', dd.min(), dd.max(), 'vs brick', brick.dec1,
brick.dec2)
# Find pixels that are within this brick's unique area
I, = np.nonzero((rr >= brick.ra1) * (rr <= brick.ra2) *
(dd >= brick.dec1) * (dd <= brick.dec2))
if plots:
plt.clf()
plt.plot(
[brick.ra1, brick.ra1, brick.ra2, brick.ra2, brick.ra1],
[brick.dec1, brick.dec2, brick.dec2, brick.dec1, brick.dec1],
'b-')
plt.plot(rr, dd, 'k.')
plt.plot(rr[I], dd[I], 'r.')
plt.title('Bright pixels from %s' % nb.brickname)
ps.savefig()
if len(I) == 0:
print('No edge pixels touch')
#plt.plot(br,bd, 'b-')
continue
#print('Edge pixels touch!')
#plt.plot(br,bd, 'r-', zorder=20)
ok, x, y = brickwcs.radec2pixelxy(rr[I], dd[I])
x = np.round(x).astype(int) - 1
y = np.round(y).astype(int) - 1
print('Pixel ranges X', x.min(), x.max(), 'Y', y.min(), y.max())
assert (np.all((x >= 0) * (x < w) * (y >= 0) * (y < h)))
print('Adding blobs:', np.unique(blobs[y, x]))
brightblobs.update(blobs[y, x])
print('Blobs touching bright pixels:', brightblobs)
print()
brightblobs.discard(-1)
if len(brightblobs) == 0:
print('No neighboring bright blobs to update!')
return
print('Updating', len(brightblobs), 'blobs:', brightblobs)
tmap = np.zeros(blobs.max() + 2, bool)
for b in brightblobs:
tmap[b + 1] = True
touching = tmap[blobs + 1]
if plots:
plt.clf()
showbool(touching)
plt.title('Blobs touching bright, brick %s' % brickname)
ps.savefig()
mfn = insurvey.find_file('maskbits', brick=brickname)
maskbits, hdr = fitsio.read(mfn, header=True)
updated = maskbits | (MASKBITS['BRIGHT'] * touching)
if np.all(maskbits == updated):
print('No bits updated! (Bright stars were already masked)')
return
maskbits = updated
if plots:
plt.clf()
showbool((maskbits & MASKBITS['BRIGHT']) > 0)
plt.title('New maskbits map for BRIGHT, brick %s' % brickname)
ps.savefig()
with outsurvey.write_output('maskbits', brick=brickname) as out:
out.fits.write(maskbits, hdr=hdr)
tfn = insurvey.find_file('tractor', brick=brickname)
phdr = fitsio.read_header(tfn, ext=0)
hdr = fitsio.read_header(tfn, ext=1)
T = fits_table(tfn)
print('Read', len(T), 'sources')
print('Bright:', Counter(T.brightstarinblob))
iby = np.clip(np.round(T.by), 0, h - 1).astype(int)
ibx = np.clip(np.round(T.bx), 0, w - 1).astype(int)
if plots:
before = np.flatnonzero(T.brightstarinblob)
T.brightstarinblob |= touching[iby, ibx]
print('Bright:', Counter(T.brightstarinblob))
# yuck -- copy the TUNIT headers from input to output.
units = [
hdr.get('TUNIT%i' % (i + 1), '') for i in range(len(T.get_columns()))
]
if plots:
plt.clf()
showbool((maskbits & MASKBITS['BRIGHT']) > 0)
ax = plt.axis()
after = np.flatnonzero(T.brightstarinblob)
plt.plot(T.bx[before], T.by[before], 'gx')
plt.plot(T.bx[after], T.by[after], 'r.')
plt.axis(ax)
plt.title('sources with brightstarinblob, brick %s' % brickname)
ps.savefig()
with outsurvey.write_output('tractor', brick=brickname) as out:
T.writeto(None,
fits_object=out.fits,
primheader=phdr,
header=hdr,
units=units)
def main():
ps = PlotSequence('conv')
S = 51
center = S / 2
print('Center', center)
#for psf_sigma in [2., 1.5, 1.]:
for psf_sigma in [2.]:
rms2 = []
x = np.arange(S)
y = np.arange(S)
xx, yy = np.meshgrid(x, y)
scale = 1.5 / psf_sigma
pixpsf = render_airy((scale, center), x, y)
psf = (scale, center)
eval_psf = render_airy
plt.clf()
plt.subplot(2, 1, 1)
plt.plot(x, pixpsf[center, :], 'b-')
plt.plot(x, pixpsf[:, center], 'r-')
plt.subplot(2, 1, 2)
plt.plot(x, np.maximum(1e-16, pixpsf[center, :]), 'b-')
plt.plot(x, np.maximum(1e-16, pixpsf[:, center]), 'r-')
plt.yscale('log')
ps.savefig()
plt.clf()
plt.imshow(pixpsf, interpolation='nearest', origin='lower')
ps.savefig()
plt.clf()
plt.imshow(np.log10(np.maximum(1e-16, pixpsf)),
interpolation='nearest',
origin='lower')
plt.colorbar()
plt.title('log PSF')
ps.savefig()
# psf
#psf = scipy.stats.norm(loc=center + 0.5, scale=psf_sigma)
# plt.clf()
# plt.imshow(Pcdf, interpolation='nearest', origin='lower')
# ps.savefig()
# #Pcdf = psf.cdf(xx) * psf.cdf(yy)
# #pixpsf = integrate_gaussian(psf, xx, yy)
#
# padpsf = np.zeros((S*2-1, S*2-1))
# ph,pw = pixpsf.shape
# padpsf[S/2:S/2+ph, S/2:S/2+pw] = pixpsf
# Fpsf = np.fft.rfft2(padpsf)
#
# padh,padw = padpsf.shape
# v = np.fft.rfftfreq(padw)
# w = np.fft.fftfreq(padh)
# fmax = max(max(np.abs(v)), max(np.abs(w)))
# cut = fmax / 2. * 1.000001
# #print('Frequence cut:', cut)
# Ffiltpsf = Fpsf.copy()
# #print('Ffiltpsf', Ffiltpsf.shape)
# #print((np.abs(w) < cut).shape)
# #print((np.abs(v) < cut).shape)
# Ffiltpsf[np.abs(w) > cut, :] = 0.
# Ffiltpsf[:, np.abs(v) > cut] = 0.
# #print('pad v', v)
# #print('pad w', w)
#
# filtpsf = np.fft.irfft2(Ffiltpsf, s=(padh,padw))
#
# print('filtered PSF real', np.max(np.abs(filtpsf.real)))
# print('filtered PSF imag', np.max(np.abs(filtpsf.imag)))
#
# plt.clf()
# plt.subplot(2,3,1)
# dimshow(Fpsf.real)
# plt.colorbar()
# plt.title('Padded PSF real')
# plt.subplot(2,3,4)
# dimshow(Fpsf.imag)
# plt.colorbar()
# plt.title('Padded PSF imag')
#
# plt.subplot(2,3,2)
# dimshow(Ffiltpsf.real)
# plt.colorbar()
# plt.title('Filt PSF real')
# plt.subplot(2,3,5)
# dimshow(Ffiltpsf.imag)
# plt.colorbar()
# plt.title('Filt PSF imag')
#
# plt.subplot(2,3,3)
# dimshow(filtpsf.real)
# plt.title('PSF real')
# plt.colorbar()
#
# plt.subplot(2,3,6)
# dimshow(filtpsf.imag)
# plt.title('PSF imag')
# plt.colorbar()
#
# ps.savefig()
#
#
# pixpsf = filtpsf
gal_sigmas = [2, 1, 0.5, 0.25]
for gal_sigma in gal_sigmas:
# plt.clf()
# plt.imshow(Gcdf, interpolation='nearest', origin='lower')
# plt.savefig('dcdf.png')
# plt.clf()
# plt.imshow(np.exp(-0.5 * ((xx-center)**2 + (yy-center)**2)/2.**2),
# interpolation='nearest', origin='lower')
# plt.savefig('g.png')
# my convolution
pixscale = 1.
cd = pixscale * np.eye(2) / 3600.
P, FG, Gmine, v, w = galaxy_psf_convolution(gal_sigma,
0.,
0.,
GaussianGalaxy,
cd,
0.,
0.,
pixpsf,
debug=True)
#print('v:', v)
#print('w:', w)
#print('P:', P.shape)
print()
print('PSF %g, Gal %g' % (psf_sigma, gal_sigma))
rmax = np.argmax(np.abs(w))
cmax = np.argmax(np.abs(v))
l2_rmax = np.sqrt(np.sum(P[rmax, :].real**2 + P[rmax, :].imag**2))
l2_cmax = np.sqrt(np.sum(P[:, cmax].real**2 + P[:, cmax].imag**2))
print('PSF L_2 in highest-frequency rows & cols:', l2_rmax,
l2_cmax)
l2_rmax = np.sqrt(np.sum(FG[rmax, :].real**2 +
FG[rmax, :].imag**2))
l2_cmax = np.sqrt(np.sum(FG[:, cmax].real**2 +
FG[:, cmax].imag**2))
print('Gal L_2 in highest-frequency rows & cols:', l2_rmax,
l2_cmax)
C = P * FG
l2_rmax = np.sqrt(np.sum(C[rmax, :].real**2 + C[rmax, :].imag**2))
l2_cmax = np.sqrt(np.sum(C[:, cmax].real**2 + C[:, cmax].imag**2))
print('PSF*Gal L_2 in highest-frequency rows & cols:', l2_rmax,
l2_cmax)
print()
Fpsf, Fgal = compare_subsampled(S,
1,
ps,
psf,
pixpsf,
Gmine,
v,
w,
gal_sigma,
psf_sigma,
cd,
get_ffts=True,
eval_psf=eval_psf)
plt.clf()
plt.subplot(2, 4, 1)
dimshow(P.real)
plt.colorbar()
plt.title('PSF real')
plt.subplot(2, 4, 5)
dimshow(P.imag)
plt.colorbar()
plt.title('PSF imag')
plt.subplot(2, 4, 2)
dimshow(FG.real)
plt.colorbar()
plt.title('Gal real')
plt.subplot(2, 4, 6)
dimshow(FG.imag)
plt.colorbar()
plt.title('Gal imag')
plt.subplot(2, 4, 3)
dimshow((P * FG).real)
plt.colorbar()
plt.title('P*Gal real')
plt.subplot(2, 4, 7)
dimshow((P * FG).imag)
plt.colorbar()
plt.title('P*Gal imag')
plt.subplot(2, 4, 4)
dimshow((Fgal).real)
plt.colorbar()
plt.title('pixGal real')
plt.subplot(2, 4, 8)
dimshow((Fgal).imag)
plt.colorbar()
plt.title('pixGal imag')
plt.suptitle('PSF %g, Gal %g' % (psf_sigma, gal_sigma))
ps.savefig()
subsample = [1, 2, 4]
rms1 = []
for s in subsample:
rms = compare_subsampled(S,
s,
ps,
psf,
pixpsf,
Gmine,
v,
w,
gal_sigma,
psf_sigma,
cd,
eval_psf=eval_psf)
rms1.append(rms)
rms2.append(rms1)
print()
print('PSF sigma =', psf_sigma)
print('RMSes:')
for rms1, gal_sigma in zip(rms2, gal_sigmas):
print('Gal sigma', gal_sigma, 'rms:',
', '.join(['%.3g' % r for r in rms1]))
def main():
# ps = PlotSequence('pro')
# star_profiles(ps)
# sys.exit(0)
#survey_dir = '/project/projectdirs/desiproc/dr3'
#survey = LegacySurveyData(survey_dir=survey_dir)
survey = LegacySurveyData()
ralo, rahi = 240, 245
declo, dechi = 5, 12
ps = PlotSequence('comp')
bands = 'grz'
ccdfn = 'ccds-forced.fits'
if not os.path.exists(ccdfn):
ccds = survey.get_annotated_ccds()
ccds.cut((ccds.ra > ralo) * (ccds.ra < rahi) * (ccds.dec > declo) *
(ccds.dec < dechi))
print(len(ccds), 'CCDs')
ccds.path = np.array([
os.path.join( #'dr3',
'forced', ('%08i' % e)[:5], '%08i' % e,
'decam-%08i-%s-forced.fits' % (e, n.strip()))
for e, n in zip(ccds.expnum, ccds.ccdname)
])
I, = np.nonzero([os.path.exists(fn) for fn in ccds.path])
print(len(I), 'CCDs with forced photometry')
ccds.cut(I)
#ccds = ccds[:500]
#e,I = np.unique(ccds.expnum, return_index=True)
#print(len(I), 'unique exposures')
#ccds.cut(I)
FF = read_forcedphot_ccds(ccds, survey)
FF.writeto('forced-all-matches.fits')
# - z band -- no trend w/ PS1 mag (brighter-fatter)
ccds.writeto(ccdfn)
ccdfn2 = 'ccds-forced-2.fits'
if not os.path.exists(ccdfn2):
ccds = fits_table(ccdfn)
# Split into brighter/fainter halves
FF = fits_table('forced-all-matches.fits')
print(len(FF), 'forced measurements')
FF.cut(FF.masked == False)
print(len(FF), 'forced measurements not masked')
ccds.brightest_mdiff = np.zeros(len(ccds))
ccds.brightest_mscatter = np.zeros(len(ccds))
ccds.bright_mdiff = np.zeros(len(ccds))
ccds.bright_mscatter = np.zeros(len(ccds))
ccds.faint_mdiff = np.zeros(len(ccds))
ccds.faint_mscatter = np.zeros(len(ccds))
for iccd in range(len(ccds)):
I = np.flatnonzero(FF.iforced == iccd)
if len(I) == 0:
continue
if len(I) < 10:
continue
F = FF[I]
b = np.percentile(F.psmag, 10)
m = np.median(F.psmag)
print(len(F), 'matches for CCD', iccd, 'median mag', m, '10th pct',
b)
J = np.flatnonzero(F.psmag < b)
diff = F.mag[J] - F.psmag[J]
ccds.brightest_mdiff[iccd] = np.median(diff)
ccds.brightest_mscatter[iccd] = (np.percentile(diff, 84) -
np.percentile(diff, 16)) / 2.
J = np.flatnonzero(F.psmag < m)
diff = F.mag[J] - F.psmag[J]
ccds.bright_mdiff[iccd] = np.median(diff)
ccds.bright_mscatter[iccd] = (np.percentile(diff, 84) -
np.percentile(diff, 16)) / 2.
J = np.flatnonzero(F.psmag > m)
diff = F.mag[J] - F.psmag[J]
ccds.faint_mdiff[iccd] = np.median(diff)
ccds.faint_mscatter[iccd] = (np.percentile(diff, 84) -
np.percentile(diff, 16)) / 2.
ccds.writeto(ccdfn2)
ccds = fits_table(ccdfn2)
plt.clf()
plt.hist(ccds.nforced, bins=100)
plt.title('nforced')
ps.savefig()
plt.clf()
plt.hist(ccds.nmatched, bins=100)
plt.title('nmatched')
ps.savefig()
#ccds.cut(ccds.nmatched >= 150)
ccds.cut(ccds.nmatched >= 50)
print('Cut to', len(ccds), 'with >50 matched')
ccds.cut(ccds.photometric)
print('Cut to', len(ccds), 'photometric')
neff = 1. / ccds.psfnorm_mean**2
# Narcsec is in arcsec**2
narcsec = neff * ccds.pixscale_mean**2
# to arcsec
narcsec = np.sqrt(narcsec)
# Correction factor to get back to equivalent of Gaussian sigma
narcsec /= (2. * np.sqrt(np.pi))
# Conversion factor to FWHM (2.35)
narcsec *= 2. * np.sqrt(2. * np.log(2.))
ccds.psfsize = narcsec
for band in bands:
I = np.flatnonzero(
(ccds.filter == band) * (ccds.photometric) * (ccds.blacklist_ok))
mlo, mhi = -0.01, 0.05
plt.clf()
plt.plot(ccds.ccdzpt[I], ccds.exptime[I], 'k.', alpha=0.1)
J = np.flatnonzero((ccds.filter == band) * (ccds.photometric == False))
plt.plot(ccds.ccdzpt[J], ccds.exptime[J], 'r.', alpha=0.1)
plt.xlabel('Zeropoint (mag)')
plt.ylabel('exptime')
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
plt.clf()
plt.plot(ccds.ccdzpt[I],
np.clip(ccds.mdiff[I], mlo, mhi),
'k.',
alpha=0.1)
plt.xlabel('Zeropoint (mag)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
#plt.axis([0, mxsee, mlo,mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
plt.clf()
plt.plot(ccds.ccdzpt[I], ccds.psfsize[I], 'k.', alpha=0.1)
plt.xlabel('Zeropoint (mag)')
plt.ylabel('PSF size (arcsec)')
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
# plt.clf()
# plt.plot(ccds.avsky[I],
# np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
# plt.xlabel('avsky')
# plt.ylabel('DECaLS PSF - PS1 (mag)')
# plt.axhline(0, color='k', alpha=0.2)
# plt.title('DR3: EDR region, Forced phot: %s band' % band)
# ps.savefig()
#
# plt.clf()
# plt.plot(ccds.meansky[I],
# np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
# plt.xlabel('meansky')
# plt.ylabel('DECaLS PSF - PS1 (mag)')
# plt.axhline(0, color='k', alpha=0.2)
# plt.title('DR3: EDR region, Forced phot: %s band' % band)
# ps.savefig()
plt.clf()
lo, hi = (-0.02, 0.05)
ha = dict(bins=50, histtype='step', range=(lo, hi))
n, b, p1 = plt.hist(ccds.brightest_mdiff[I], color='r', **ha)
n, b, p2 = plt.hist(ccds.bright_mdiff[I], color='g', **ha)
n, b, p3 = plt.hist(ccds.faint_mdiff[I], color='b', **ha)
plt.legend((p1[0], p2[0], p3[0]),
('Brightest 10%', 'Brightest 50%', 'Faintest 50%'))
plt.xlabel('DECaLS PSF - PS1 (mag)')
plt.ylabel('Number of CCDs')
plt.title('DR3: EDR region, Forced phot: %s band' % band)
plt.xlim(lo, hi)
ps.savefig()
for band in bands:
I = np.flatnonzero(ccds.filter == band)
mxsee = 4.
mlo, mhi = -0.01, 0.05
plt.clf()
plt.plot(np.clip(ccds.psfsize[I], 0, mxsee),
np.clip(ccds.mdiff[I], mlo, mhi),
'k.',
alpha=0.1)
# for p in [1,2,3]:
# J = np.flatnonzero(ccds.tilepass[I] == p)
# if len(J):
# plt.plot(np.clip(ccds.psfsize[I[J]], 0, mxsee),
# np.clip(ccds.mdiff[I[J]], mlo,mhi), '.', color='rgb'[p-1], alpha=0.2)
#plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.')
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
plt.axis([0, mxsee, mlo, mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
# Group by exposure
for band in bands:
I = np.flatnonzero(
(ccds.filter == band) * (ccds.photometric) * (ccds.blacklist_ok))
E, J = np.unique(ccds.expnum[I], return_index=True)
print(len(E), 'unique exposures in', band)
exps = ccds[I[J]]
print(len(exps), 'unique exposures in', band)
assert (len(np.unique(exps.expnum)) == len(exps))
exps.ddiff = np.zeros(len(exps))
exps.dsize = np.zeros(len(exps))
exps.nccds = np.zeros(len(exps), int)
exps.brightest_ddiff = np.zeros(len(exps))
exps.bright_ddiff = np.zeros(len(exps))
exps.faint_ddiff = np.zeros(len(exps))
for iexp, exp in enumerate(exps):
J = np.flatnonzero(ccds.expnum[I] == exp.expnum)
J = I[J]
print(len(J), 'CCDs in exposure', exp.expnum)
exps.brightest_mdiff[iexp] = np.median(ccds.brightest_mdiff[J])
exps.bright_mdiff[iexp] = np.median(ccds.bright_mdiff[J])
exps.faint_mdiff[iexp] = np.median(ccds.faint_mdiff[J])
exps.brightest_ddiff[iexp] = (
np.percentile(ccds.brightest_mdiff[J], 84) -
np.percentile(ccds.brightest_mdiff[J], 16)) / 2.
exps.bright_ddiff[iexp] = (
np.percentile(ccds.bright_mdiff[J], 84) -
np.percentile(ccds.bright_mdiff[J], 16)) / 2.
exps.faint_ddiff[iexp] = (
np.percentile(ccds.faint_mdiff[J], 84) -
np.percentile(ccds.faint_mdiff[J], 16)) / 2.
exps.mdiff[iexp] = np.median(ccds.mdiff[J])
exps.ddiff[iexp] = (np.percentile(ccds.mdiff[J], 84) -
np.percentile(ccds.mdiff[J], 16)) / 2.
exps.psfsize[iexp] = np.median(ccds.psfsize[J])
exps.dsize[iexp] = (np.percentile(ccds.psfsize[J], 84) -
np.percentile(ccds.psfsize[J], 16)) / 2.
exps.nccds[iexp] = len(J)
mxsee = 4.
mlo, mhi = -0.01, 0.05
exps.cut(exps.nccds >= 10)
plt.clf()
plt.errorbar(
np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.mdiff, mlo, mhi),
yerr=exps.ddiff,
#xerr=exps.dsize,
fmt='.',
color='k')
# plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.brightest_mdiff, mlo,mhi),
# yerr=exps.brightest_ddiff, fmt='r.')
# plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.bright_mdiff, mlo,mhi),
# yerr=exps.bright_ddiff, fmt='g.')
# plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.faint_mdiff, mlo,mhi),
# yerr=exps.faint_ddiff, fmt='b.')
# plt.plot(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.brightest_mdiff, mlo,mhi), 'r.')
# plt.plot(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.bright_mdiff, mlo,mhi), 'g.')
# plt.plot(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.faint_mdiff, mlo,mhi), 'b.')
#plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.')
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
plt.axis([0, mxsee, mlo, mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
plt.clf()
p1 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.brightest_mdiff, mlo, mhi),
'r.',
alpha=0.5)
p2 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.bright_mdiff, mlo, mhi),
'g.',
alpha=0.5)
p3 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.faint_mdiff, mlo, mhi),
'b.',
alpha=0.5)
plt.legend((p1[0], p2[0], p3[0]),
('Brightest 10%', 'Brightest 50%', 'Faintest 50%'))
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
plt.axis([0, mxsee, mlo, mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
J = np.argsort(-exps.mdiff)
for j in J:
print(' Photometric diff', exps.mdiff[j], 'PSF size',
exps.psfsize[j], 'expnum', exps.expnum[j])
sys.exit(0)
def main(passnum, threads):
global udecs
global P3
global T
global tilewcs
global exps
global bad_expids
global tileid_to_depth
ps = PlotSequence('covfill-p%i' % passnum)
retirablefn = 'retirable-p%i.fits' % passnum
depthsfn = 'all-depths-p%i.fits' % passnum
if os.path.exists(retirablefn):
R = fits_table(retirablefn)
pcts = np.arange(0, 101)
target = 22.5
req_pcts = [0, 2, 2, 5, 5, 10, 10, 100]
req_depths = [0, 0, target-0.6, target-0.6,
target-0.3, target-0.3, target, target]
maglo, maghi = 21,23
plt.clf()
for depths in R.depths:
plt.plot(pcts, np.clip(depths, maglo, maghi), 'b-',
alpha=0.1)
plt.plot(req_pcts, np.clip(req_depths, maglo, maghi), 'k-',
lw=2, alpha=0.5)
plt.ylim(maglo, maghi)
plt.xlim(0, 100)
plt.xlabel('Coverage fraction')
plt.ylabel('Existing depth')
plt.suptitle('MzLS: retirable pass-%i tiles: %i' % (passnum,len(R)))
ps.savefig()
# Where are they on the sky?
T = fits_table('obstatus/mosaic-tiles_obstatus.fits')
T.cut(T.in_desi == 1)
T.cut(T.get('pass') <= 3)
tileid_to_index = np.zeros(T.tileid.max()+1, int)
tileid_to_index[T.tileid] = np.arange(len(T))
R.ra = T.ra [tileid_to_index[R.tileid]]
R.dec = T.dec[tileid_to_index[R.tileid]]
plt.clf()
plt.plot(T.ra, T.dec, 'k.', alpha=0.02)
I = (T.z_done == 1)
plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.1)
plt.plot(R.ra, R.dec, 'b.')
ax = [310,80,30,85]
#xl,xh = plt.xlim()
#plt.xlim(xh,xl)
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.title('MzLS: retirable pass-%i tiles' % passnum)
plt.axis(ax)
ps.savefig()
for p in [1,2,3]:
plt.clf()
plt.plot(T.ra, T.dec, 'k.', alpha=0.02)
#plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.1)
I = np.flatnonzero((T.get('pass') == p) * (T.z_done == 1)
* (T.z_depth > 1) * (T.z_depth < 30))
plt.scatter(T.ra[I], T.dec[I], c=T.z_depth[I],
vmin=20, vmax=23, s=4)
I = np.flatnonzero((T.get('pass') == p) * (T.z_done == 1)
* (T.z_depth == 30))
plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.5)
plt.colorbar()
plt.title('MzLS: Finished tiles in pass %i' % p)
plt.axis(ax)
ps.savefig()
sys.exit(0)
# NERSC: export LEGACY_SURVEY_DIR=/global/cscratch1/sd/dstn/dr4plus
# (dstn laptop: export LEGACY_SURVEY_DIR=~/legacypipe-dir-mzls/)
survey = LegacySurveyData()
ccds = survey.get_annotated_ccds()
print('Annotated CCDs:', len(ccds))
ccds.cut(ccds.camera == 'mosaic')
print(len(ccds), 'Mosaic')
print('Unique exposures:', len(np.unique(ccds.expnum)))
ccds.cut(ccds.exptime > 60)
print('Exptime > 60 sec:', len(ccds))
nccds = Counter(ccds.expnum)
for k,v in nccds.most_common():
if v <= 4:
break
print('Expnum', k, 'appears', v, 'times')
print('Tile pass numbers:', Counter(ccds.tilepass).most_common())
# Fix parsing of OBJECT field to tileid...
from obsbot import get_tile_id_from_name
tileids = []
for o in ccds.object:
tid = get_tile_id_from_name(o.strip())
if tid is None:
tid = 0
tileids.append(tid)
tileids = np.array(tileids)
print(len(np.unique(tileids)), 'unique tile ids in annotated file, from OBJECT')
print(len(np.unique(ccds.tileid)), 'unique tile ids in ann file from TILEID')
D = np.flatnonzero(tileids != ccds.tileid)
print(len(D), 'different tileids')
print('From OBJECT:', tileids[D])
print('From TILEID:', ccds.tileid[D])
ccds.tileid = tileids
T = fits_table('obstatus/mosaic-tiles_obstatus.fits')
f = open('obstatus/bad_expid.txt')
bad_expids = []
for line in f:
line = line.strip()
if len(line) == 0:
continue
if line[0] == '#':
continue
words = line.split()
try:
expnum = int(words[0])
except:
print('Skipping line:', line)
continue
bad_expids.append(expnum)
print('Read', len(bad_expids), 'bad exposure numbers')
# Update ccds.tilepass from ccds.tileid
tileidtopass = dict(zip(T.tileid, T.get('pass')))
tileidtoebv = dict(zip(T.tileid, T.ebv_med))
ccds.tilepass = np.array([tileidtopass.get(tileid, 0)
for tileid in ccds.tileid])
ccds.tileebv = np.array([tileidtoebv.get(tileid, 0)
for tileid in ccds.tileid])
print('Tile pass numbers after update:', Counter(ccds.tilepass).most_common())
e,I = np.unique(ccds.expnum, return_index=True)
exps = ccds[I]
#print('Object names,exptimes for tilepass==0:', zip(exps.object[exps.tilepass == 0], exps.exptime[exps.tilepass == 0]))
# Average the depth per exposure
for j,expnum in enumerate(exps.expnum):
I = np.flatnonzero(ccds.expnum == expnum)
if len(I) != 4:
print('Exposure', expnum, 'has', len(I), 'CCD entries')
continue
# Don't include zeros in computing average depths!
Igood = I[(ccds.galdepth[I] > 0) * (ccds.ccdzpt[I] < 30)]
if len(Igood) > 0:
exps.galdepth[j] = np.mean(ccds.galdepth[Igood])
else:
exps.galdepth[j] = 0.
# CCDs-table-based mapping from tileid to depth.
I = np.flatnonzero((exps.tilepass > 0) * (exps.galdepth > 0) *
(exps.tileid > 0))
tileid_to_depth = dict(zip(exps.tileid[I], exps.galdepth[I]))
T.cut(T.in_desi == 1)
T.cut(T.get('pass') <= 3)
# The tiles we'll examine
P3 = T[T.get('pass') == passnum]
print(len(P3), 'pass', passnum, 'and in DESI')
todo = P3[P3.z_done == 0]
print(len(todo), 'pass', passnum, 'tiles to do (Z_DONE=0)')
# Tiles with measured depths
T.cut((T.z_depth > 15) * (T.z_depth < 30))
print(len(T), 'tiles with measured depths')
# Passes other than 3... they ~ only barely overlap, so don't
# contribute significant depth.
#T.cut(T.get('pass') < 3)
udecs = np.unique(P3.dec)
print(len(udecs), 'unique Dec values in pass', passnum)
# Grab an arbitrary weight-map image and use that as a proxy!
wtfn = 'k4m_170501_112501_oow_zd_v1.fits.fz'
F = fitsio.FITS(wtfn)
tilewcs = []
# Read the WCS headers for each chip.
# They make the CRVAL be the boresight for all 4 chips... perfect!
# (because this means we can just set CRVAL = RA,Dec to shift the WCSes)
for i in range(1, len(F)):
hdr = F[i].read_header()
wcs = wcs_pv2sip_hdr(hdr)
tilewcs.append(wcs)
mp = multiproc(threads)
#args = [(i,t,udecs,P3,T,tilewcs,exps,bad_expids,tileid_to_depth)
args = [(i,t)
for i,t in enumerate(todo)]
thedepths = mp.map(one_tile, args)
alldepths = []
retirable = []
for arg,depths in zip(args, thedepths):
if depths is None:
continue
itile,tile = arg[:2]
target = 22.5
req_pcts = [0, 2, 2, 5, 5, 10, 10, 100]
req_depths = [0, 0, target-0.6, target-0.6,
target-0.3, target-0.3, target, target]
print(' Depths at 2, 5, and 10th percentile vs target:',
'%.2f' % (depths[2] - (target - 0.6)),
'%.2f' % (depths[5] - (target - 0.3)),
'%.2f' % (depths[10] - target))
alldepths.append((tile.tileid, depths))
if not ((depths[2] > target - 0.6) and
(depths[5] > target - 0.3) and
(depths[10] > target)):
continue
retirable.append((tile.tileid, depths))
#if len(retirable) == 10:
# break
# if ps.ploti >= 100:
# continue
#
# maglo, maghi = 21,23
#
# plt.clf()
# #plt.subplot(1,2,1)
# plt.subplot2grid((2,2), (0,0))
# plt.imshow(depth, interpolation='nearest', origin='lower',
# vmin=maglo, vmax=maghi)
# plt.colorbar(ticks=[np.arange(maglo, maghi+0.01, 0.5)])
# plt.xticks([]); plt.yticks([])
# #plt.subplot(1,2,2)
# plt.subplot2grid((2,2), (1,0))
# plt.imshow(nexp, interpolation='nearest', origin='lower',
# vmin=0, vmax=4)
# plt.colorbar(ticks=[0,1,2,3,4])
# plt.xticks([]); plt.yticks([])
#
# ax = plt.subplot2grid((2,2), (0,1), rowspan=2)
# plt.plot(req_pcts, np.clip(req_depths, maglo, maghi), 'k-',
# lw=2, alpha=0.5)
# plt.plot(pcts, np.clip(depths, maglo, maghi), 'b-')
# ax.yaxis.tick_right()
# plt.ylim(maglo, maghi)
# plt.xlim(0, 100)
# plt.xlabel('Coverage fraction')
# plt.ylabel('Existing depth')
# plt.suptitle('Tile %i' % tile.tileid)
# ps.savefig()
#if ps.ploti == 100:
# break
# print('Tiles that could be retired:')
# print('# Tileid 0th-percentile-extcorr-depth 1st-pctile 2nd-pctile ...')
# for tileid, depths in retirable:
# print(tileid, ' '.join(['%.3f' % d for d in depths]))
R = fits_table()
R.tileid = np.array ([t for t,d in alldepths])
R.depths = np.vstack([d for t,d in alldepths])
R.writeto(depthsfn)
if len(retirable):
R = fits_table()
R.tileid = np.array([t for t,d in retirable])
R.depths = np.vstack([d for t,d in retirable])
R.writeto(retirablefn)
else:
print('No tiles in pass', passnum, 'are retirable')
def stage0(**kwargs):
ps = PlotSequence('cfht')
decals = CfhtDecals()
B = decals.get_bricks()
print('Bricks:')
B.about()
ra, dec = 190.0, 11.0
#bands = 'ugri'
bands = 'gri'
B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec)))
print('Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5])
brick = B[0]
pixscale = 0.186
#W,H = 1024,1024
#W,H = 2048,2048
#W,H = 3600,3600
W, H = 4800, 4800
targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H)
ccdfn = 'cfht-ccds.fits'
if os.path.exists(ccdfn):
T = fits_table(ccdfn)
else:
T = get_ccd_list()
T.writeto(ccdfn)
print(len(T), 'CCDs')
T.cut(ccds_touching_wcs(targetwcs, T))
print(len(T), 'CCDs touching brick')
T.cut(np.array([b in bands for b in T.filter]))
print(len(T), 'in bands', bands)
ims = []
for t in T:
im = CfhtImage(t)
# magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME'])
# fwhm = t.seeing / (pixscale * 3600)
# print '-> FWHM', fwhm, 'pix'
im.seeing = t.seeing
im.pixscale = t.pixscale
print('seeing', t.seeing)
print('pixscale', im.pixscale * 3600, 'arcsec/pix')
im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height)
ims.append(im)
# Read images, clip to ROI
targetrd = np.array([
targetwcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
keepims = []
tims = []
for im in ims:
print()
print('Reading expnum', im.expnum, 'name', im.extname, 'band', im.band,
'exptime', im.exptime)
band = im.band
wcs = im.read_wcs()
imh, imw = wcs.imageh, wcs.imagew
imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)]
ok, tx, ty = wcs.radec2pixelxy(targetrd[:-1, 0], targetrd[:-1, 1])
tpoly = zip(tx, ty)
clip = clip_polygon(imgpoly, tpoly)
clip = np.array(clip)
#print 'Clip', clip
if len(clip) == 0:
continue
x0, y0 = np.floor(clip.min(axis=0)).astype(int)
x1, y1 = np.ceil(clip.max(axis=0)).astype(int)
slc = slice(y0, y1 + 1), slice(x0, x1 + 1)
## FIXME -- it seems I got lucky and the cross product is
## negative == clockwise, as required by clip_polygon. One
## could check this and reverse the polygon vertex order.
# dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0]
# dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1]
# cross = dx0*dy1 - dx1*dy0
# print 'Cross:', cross
print('Image slice: x [%i,%i], y [%i,%i]' % (x0, x1, y0, y1))
print('Reading image from', im.imgfn, 'HDU', im.hdu)
img, imghdr = im.read_image(header=True, slice=slc)
goodpix = (img != 0)
print('Number of pixels == 0:', np.sum(img == 0))
print('Number of pixels != 0:', np.sum(goodpix))
if np.sum(goodpix) == 0:
continue
# print 'Image shape', img.shape
print('Image range', img.min(), img.max())
print('Goodpix image range:', (img[goodpix]).min(),
(img[goodpix]).max())
if img[goodpix].min() == img[goodpix].max():
print('No dynamic range in image')
continue
# print 'Reading invvar from', im.wtfn, 'HDU', im.hdu
# invvar = im.read_invvar(slice=slc)
# # print 'Invvar shape', invvar.shape
# # print 'Invvar range:', invvar.min(), invvar.max()
# invvar[goodpix == 0] = 0.
# if np.all(invvar == 0.):
# print 'Skipping zero-invvar image'
# continue
# assert(np.all(np.isfinite(img)))
# assert(np.all(np.isfinite(invvar)))
# assert(not(np.all(invvar == 0.)))
# # Estimate per-pixel noise via Blanton's 5-pixel MAD
# slice1 = (slice(0,-5,10),slice(0,-5,10))
# slice2 = (slice(5,None,10),slice(5,None,10))
# # print 'sliced shapes:', img[slice1].shape, img[slice2].shape
# # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape
# # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2])
# # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape
# mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel())
# sig1 = 1.4826 * mad / np.sqrt(2.)
# print 'MAD sig1:', sig1
# # invvar was 1 or 0
# invvar *= (1./(sig1**2))
# medsky = np.median(img[goodpix])
# Read full image for sig1 and sky estimate
fullimg = im.read_image()
fullgood = (fullimg != 0)
# Estimate per-pixel noise via Blanton's 5-pixel MAD
slice1 = (slice(0, -5, 10), slice(0, -5, 10))
slice2 = (slice(5, None, 10), slice(5, None, 10))
mad = np.median(
np.abs(fullimg[slice1] -
fullimg[slice2])[fullgood[slice1] *
fullgood[slice2]].ravel())
sig1 = 1.4826 * mad / np.sqrt(2.)
print('MAD sig1:', sig1)
medsky = np.median(fullimg[fullgood])
invvar = np.zeros_like(img)
invvar[goodpix] = 1. / sig1**2
# Median-smooth sky subtraction
plt.clf()
dimshow(np.round((img - medsky) / sig1), vmin=-3, vmax=5)
plt.title('Scalar median: %s' % im.name)
ps.savefig()
# medsky = np.zeros_like(img)
# # astrometry.util.util
# median_smooth(img, np.logical_not(goodpix), 256, medsky)
fullmed = np.zeros_like(fullimg)
median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed)
fullmed += medsky
medimg = fullmed[slc]
plt.clf()
dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5)
plt.title('Median filtered: %s' % im.name)
ps.savefig()
#print 'Subtracting median:', medsky
#img -= medsky
img -= medimg
primhdr = im.read_image_primary_header()
magzp = decals.get_zeropoint_for(im)
print('magzp', magzp)
zpscale = NanoMaggies.zeropointToScale(magzp)
print('zpscale', zpscale)
# Scale images to Nanomaggies
img /= zpscale
sig1 /= zpscale
invvar *= zpscale**2
orig_zpscale = zpscale
zpscale = 1.
assert (np.sum(invvar > 0) > 0)
print('After scaling:')
print('sig1', sig1)
print('invvar range', invvar.min(), invvar.max())
print('image range', img.min(), img.max())
assert (np.all(np.isfinite(img)))
assert (np.all(np.isfinite(invvar)))
assert (np.isfinite(sig1))
plt.clf()
lo, hi = -5 * sig1, 10 * sig1
n, b, p = plt.hist(img[goodpix].ravel(),
100,
range=(lo, hi),
histtype='step',
color='k')
xx = np.linspace(lo, hi, 200)
plt.plot(xx, max(n) * np.exp(-xx**2 / (2. * sig1**2)), 'r-')
plt.xlim(lo, hi)
plt.title('Pixel histogram: %s' % im.name)
ps.savefig()
twcs = ConstantFitsWcs(wcs)
if x0 or y0:
twcs.setX0Y0(x0, y0)
info = im.get_image_info()
fullh, fullw = info['dims']
# read fit PsfEx model
psfex = PsfEx.fromFits(im.psffitfn)
print('Read', psfex)
# HACK -- highly approximate PSF here!
#psf_fwhm = imghdr['FWHM']
#psf_fwhm = im.seeing
psf_fwhm = im.seeing / (im.pixscale * 3600)
print('PSF FWHM', psf_fwhm, 'pixels')
psf_sigma = psf_fwhm / 2.35
psf = NCircularGaussianPSF([psf_sigma], [1.])
print('img type', img.dtype)
tim = Image(img,
invvar=invvar,
wcs=twcs,
psf=psf,
photocal=LinearPhotoCal(zpscale, band=band),
sky=ConstantSky(0.),
name=im.name + ' ' + band)
tim.zr = [-3. * sig1, 10. * sig1]
tim.sig1 = sig1
tim.band = band
tim.psf_fwhm = psf_fwhm
tim.psf_sigma = psf_sigma
tim.sip_wcs = wcs
tim.x0, tim.y0 = int(x0), int(y0)
tim.psfex = psfex
tim.imobj = im
mn, mx = tim.zr
tim.ima = dict(interpolation='nearest',
origin='lower',
cmap='gray',
vmin=mn,
vmax=mx)
tims.append(tim)
keepims.append(im)
ims = keepims
print('Computing resampling...')
# save resampling params
for tim in tims:
wcs = tim.sip_wcs
subh, subw = tim.shape
subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh)
tim.subwcs = subwcs
try:
Yo, Xo, Yi, Xi, rims = resample_with_wcs(targetwcs, subwcs, [], 2)
except OverlapError:
print('No overlap')
continue
if len(Yo) == 0:
continue
tim.resamp = (Yo, Xo, Yi, Xi)
print('Creating coadds...')
# Produce per-band coadds, for plots
coimgs = []
cons = []
for ib, band in enumerate(bands):
coimg = np.zeros((H, W), np.float32)
con = np.zeros((H, W), np.uint8)
for tim in tims:
if tim.band != band:
continue
(Yo, Xo, Yi, Xi) = tim.resamp
if len(Yo) == 0:
continue
nn = (tim.getInvvar()[Yi, Xi] > 0)
coimg[Yo, Xo] += tim.getImage()[Yi, Xi] * nn
con[Yo, Xo] += nn
# print
# print 'tim', tim.name
# print 'number of resampled pix:', len(Yo)
# reim = np.zeros_like(coimg)
# ren = np.zeros_like(coimg)
# reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn
# ren[Yo,Xo] = nn
# print 'number of resampled pix with positive invvar:', ren.sum()
# plt.clf()
# plt.subplot(2,2,1)
# mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]]
# print 'Percentiles:', mn,mx
# dimshow(reim, vmin=mn, vmax=mx)
# plt.colorbar()
# plt.subplot(2,2,2)
# dimshow(con)
# plt.colorbar()
# plt.subplot(2,2,3)
# dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1])
# plt.colorbar()
# plt.subplot(2,2,4)
# plt.hist(reim.ravel(), 100, histtype='step', color='b')
# plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r')
# plt.suptitle('%s: %s' % (band, tim.name))
# ps.savefig()
coimg /= np.maximum(con, 1)
coimgs.append(coimg)
cons.append(con)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ps.savefig()
plt.clf()
for i, b in enumerate(bands):
plt.subplot(2, 2, i + 1)
dimshow(cons[i], ticks=False)
plt.title('%s band' % b)
plt.colorbar()
plt.suptitle('Number of exposures')
ps.savefig()
print('Grabbing SDSS sources...')
bandlist = [b for b in bands]
cat, T = get_sdss_sources(bandlist, targetwcs)
# record coordinates in target brick image
ok, T.tx, T.ty = targetwcs.radec2pixelxy(T.ra, T.dec)
T.tx -= 1
T.ty -= 1
T.itx = np.clip(np.round(T.tx).astype(int), 0, W - 1)
T.ity = np.clip(np.round(T.ty).astype(int), 0, H - 1)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5)
plt.axis(ax)
plt.title('SDSS sources')
ps.savefig()
print('Detmaps...')
# Render the detection maps
detmaps = dict([(b, np.zeros((H, W), np.float32)) for b in bands])
detivs = dict([(b, np.zeros((H, W), np.float32)) for b in bands])
for tim in tims:
iv = tim.getInvvar()
psfnorm = 1. / (2. * np.sqrt(np.pi) * tim.psf_sigma)
detim = tim.getImage().copy()
detim[iv == 0] = 0.
detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2
detsig1 = tim.sig1 / psfnorm
subh, subw = tim.shape
detiv = np.zeros((subh, subw), np.float32) + (1. / detsig1**2)
detiv[iv == 0] = 0.
(Yo, Xo, Yi, Xi) = tim.resamp
detmaps[tim.band][Yo, Xo] += detiv[Yi, Xi] * detim[Yi, Xi]
detivs[tim.band][Yo, Xo] += detiv[Yi, Xi]
rtn = dict()
for k in [
'T', 'coimgs', 'cons', 'detmaps', 'detivs', 'targetrd', 'pixscale',
'targetwcs', 'W', 'H', 'bands', 'tims', 'ps', 'brick', 'cat'
]:
rtn[k] = locals()[k]
return rtn
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None,
use_ceres=True, ceres_block=8,
save_fits=False, get_models=False, ps=None,
psf_broadening=None,
pixelized_psf=False,
get_masks=None,
move_crpix=False,
modelsky_dir=None):
'''
Given a list of tractor sources *cat*
and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
runs forced photometry, returning a FITS table the same length as *cat*.
*get_masks*: the WCS to resample mask bits into.
'''
from tractor import NanoMaggies, PointSource, Tractor, ExpGalaxy, DevGalaxy, FixedCompositeGalaxy
if not pixelized_psf and psf_broadening is None:
# PSF broadening in post-reactivation data, by band.
# Newer version from Aaron's email to decam-chatter, 2018-06-14.
broadening = { 1: 1.0405, 2: 1.0346, 3: None, 4: None }
psf_broadening = broadening[band]
if False:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('wise-forced-w%i' % band)
plots = (ps is not None)
if plots:
import pylab as plt
wantims = (plots or save_fits or get_models)
wanyband = 'w'
if get_models:
models = {}
wband = 'w%i' % band
fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix',
'pronexp']
Nsrcs = len(cat)
phot = fits_table()
# Filled in based on unique tile overlap
phot.wise_coadd_id = np.array([' '] * Nsrcs)
phot.set(wband + '_psfdepth', np.zeros(len(phot), np.float32))
ra = np.array([src.getPosition().ra for src in cat])
dec = np.array([src.getPosition().dec for src in cat])
nexp = np.zeros(Nsrcs, np.int16)
mjd = np.zeros(Nsrcs, np.float64)
central_flux = np.zeros(Nsrcs, np.float32)
fitstats = {}
tims = []
if get_masks:
mh,mw = get_masks.shape
maskmap = np.zeros((mh,mw), np.uint32)
for tile in tiles:
print('Reading WISE tile', tile.coadd_id, 'band', band)
tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band,
bandname=wanyband, roiradecbox=roiradecbox)
if tim is None:
print('Actually, no overlap with tile', tile.coadd_id)
continue
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data' % tag)
ps.savefig()
plt.clf()
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
range=(-5,10), bins=100)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.title(tag)
ps.savefig()
if move_crpix and band in [1, 2]:
realwcs = tim.wcs.wcs
x,y = realwcs.crpix
tile_crpix = tile.get('crpix_w%i' % band)
dx = tile_crpix[0] - 1024.5
dy = tile_crpix[1] - 1024.5
realwcs.set_crpix(x+dx, y+dy)
#print('CRPIX', x,y, 'shift by', dx,dy, 'to', realwcs.crpix)
if modelsky_dir and band in [1, 2]:
fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band))
if not os.path.exists(fn):
raise RuntimeError('WARNING: does not exist:', fn)
x0,x1,y0,y1 = tim.roi
bg = fitsio.FITS(fn)[2][y0:y1, x0:x1]
#print('Read background map:', bg.shape, bg.dtype, 'vs image', tim.shape)
if plots:
plt.clf()
plt.subplot(1,2,1)
plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=5 * sig1)
plt.subplot(1,2,2)
plt.imshow(bg, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=5 * sig1)
tag = '%s W%i' % (tile.coadd_id, band)
plt.suptitle(tag)
ps.savefig()
plt.clf()
ha = dict(range=(-5,10), bins=100, histtype='step')
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
color='b', label='Original', **ha)
plt.hist(((tim.getImage()-bg) * tim.inverr)[tim.inverr > 0].ravel(),
color='g', label='Minus Background', **ha)
plt.axvline(0, color='k', alpha=0.5)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.legend()
plt.title(tag + ': background')
ps.savefig()
# Actually subtract the background!
tim.data -= bg
# Floor the per-pixel variances
if band in [1,2]:
# in Vega nanomaggies per pixel
floor_sigma = {1: 0.5, 2: 2.0}
with np.errstate(divide='ignore'):
new_ie = 1. / np.hypot(1./tim.inverr, floor_sigma[band])
new_ie[tim.inverr == 0] = 0.
if plots:
plt.clf()
plt.plot((1. / tim.inverr[tim.inverr>0]).ravel(), (1./new_ie[tim.inverr>0]).ravel(), 'b.')
plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band))
plt.xlabel('original')
plt.ylabel('floored')
ps.savefig()
tim.inverr = new_ie
# Read mask file?
if get_masks:
from astrometry.util.resample import resample_with_wcs, OverlapError
# unwise_dir can be a colon-separated list of paths
tilemask = None
for d in tile.unwise_dir.split(':'):
fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id,
'unwise-%s-msk.fits.gz' % tile.coadd_id)
if os.path.exists(fn):
print('Reading unWISE mask file', fn)
x0,x1,y0,y1 = tim.roi
tilemask = fitsio.FITS(fn)[0][y0:y1,x0:x1]
break
if tilemask is None:
print('unWISE mask file for tile', tile.coadd_id, 'does not exist')
else:
try:
tanwcs = tim.wcs.wcs
assert(tanwcs.shape == tilemask.shape)
Yo,Xo,Yi,Xi,_ = resample_with_wcs(get_masks, tanwcs, intType=np.int16)
# Only deal with mask pixels that are set.
I, = np.nonzero(tilemask[Yi,Xi] > 0)
# Trim to unique area for this tile
rr,dd = get_masks.pixelxy2radec(Yo[I]+1, Xo[I]+1)
good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
I = I[good]
maskmap[Yo[I],Xo[I]] = tilemask[Yi[I], Xi[I]]
except OverlapError:
# Shouldn't happen by this point
print('No overlap between WISE tile', tile.coadd_id, 'and brick')
# The tiles have some overlap, so zero out pixels outside the
# tile's unique area.
th,tw = tim.shape
xx,yy = np.meshgrid(np.arange(tw), np.arange(th))
rr,dd = tim.wcs.wcs.pixelxy2radec(xx+1, yy+1)
unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
#print(np.sum(unique), 'of', (th*tw), 'pixels in this tile are unique')
tim.inverr[unique == False] = 0.
del xx,yy,rr,dd,unique
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage() * (tim.inverr > 0),
interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data (unique)' % tag)
ps.savefig()
if pixelized_psf:
import unwise_psf
if (band == 1) or (band == 2):
# we only have updated PSFs for W1 and W2
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id,
modelname='neo4_unwisecat')
else:
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id)
if band == 4:
# oversample (the unwise_psf models are at native W4 5.5"/pix,
# while the unWISE coadds are made at 2.75"/pix.
ph,pw = psfimg.shape
subpsf = np.zeros((ph*2-1, pw*2-1), np.float32)
from astrometry.util.util import lanczos3_interpolate
xx,yy = np.meshgrid(np.arange(0., pw-0.51, 0.5, dtype=np.float32),
np.arange(0., ph-0.51, 0.5, dtype=np.float32))
xx = xx.ravel()
yy = yy.ravel()
ix = xx.astype(np.int32)
iy = yy.astype(np.int32)
dx = (xx - ix).astype(np.float32)
dy = (yy - iy).astype(np.float32)
psfimg = psfimg.astype(np.float32)
rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg])
if plots:
plt.clf()
plt.imshow(psfimg, interpolation='nearest', origin='lower')
plt.title('Original PSF model')
ps.savefig()
plt.clf()
plt.imshow(subpsf, interpolation='nearest', origin='lower')
plt.title('Subsampled PSF model')
ps.savefig()
psfimg = subpsf
del xx, yy, ix, iy, dx, dy
from tractor.psf import PixelizedPSF
psfimg /= psfimg.sum()
fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0}
psfimg *= fluxrescales[band]
tim.psf = PixelizedPSF(psfimg)
if psf_broadening is not None and not pixelized_psf:
# psf_broadening is a factor by which the PSF FWHMs
# should be scaled; the PSF is a little wider
# post-reactivation.
psf = tim.getPsf()
from tractor import GaussianMixturePSF
if isinstance(psf, GaussianMixturePSF):
#
print('Broadening PSF: from', psf)
p0 = psf.getParams()
pnames = psf.getParamNames()
p1 = [p * psf_broadening**2 if 'var' in name else p
for (p, name) in zip(p0, pnames)]
psf.setParams(p1)
print('Broadened PSF:', psf)
else:
print('WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf))
wcs = tim.wcs.wcs
ok,x,y = wcs.radec2pixelxy(ra, dec)
x = np.round(x - 1.).astype(int)
y = np.round(y - 1.).astype(int)
good = (x >= 0) * (x < tw) * (y >= 0) * (y < th)
# Which sources are in this brick's unique area?
usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
I, = np.nonzero(good * usrc)
nexp[I] = tim.nuims[y[I], x[I]]
if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
mjd[I] = (tim.mjdmin + tim.mjdmax) / 2.
phot.wise_coadd_id[I] = tile.coadd_id
central_flux[I] = tim.getImage()[y[I], x[I]]
del x,y,good,usrc
# PSF norm for depth
psf = tim.getPsf()
h,w = tim.shape
patch = psf.getPointSourcePatch(h//2, w//2).patch
psfnorm = np.sqrt(np.sum(patch**2))
# To handle zero-depth, we return 1/nanomaggies^2 units rather than mags.
psfdepth = 1. / (tim.sig1 / psfnorm)**2
phot.get(wband + '_psfdepth')[I] = psfdepth
tim.tile = tile
tims.append(tim)
if plots:
plt.clf()
mn,mx = 0.1, 20000
plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100,
range=(np.log10(mn), np.log10(mx)))
logt = np.arange(0, 5)
plt.xticks(logt, ['%i' % i for i in 10.**logt])
plt.title('Central fluxes (W%i)' % band)
plt.axvline(np.log10(20000), color='k')
plt.axvline(np.log10(1000), color='k')
ps.savefig()
# Eddie's non-secret recipe:
#- central pixel <= 1000: 19x19 pix box size
#- central pixel in 1000 - 20000: 59x59 box size
#- central pixel > 20000 or saturated: 149x149 box size
#- object near "bright star": 299x299 box size
nbig = nmedium = nsmall = 0
for src,cflux in zip(cat, central_flux):
if cflux > 20000:
R = 100
nbig += 1
elif cflux > 1000:
R = 30
nmedium += 1
else:
R = 15
nsmall += 1
if isinstance(src, PointSource):
src.fixedRadius = R
else:
### FIXME -- sizes for galaxies..... can we set PSF size separately?
galrad = 0
# RexGalaxy is a subclass of ExpGalaxy
if isinstance(src, (ExpGalaxy, DevGalaxy)):
galrad = src.shape.re
elif isinstance(src, FixedCompositeGalaxy):
galrad = max(src.shapeExp.re, src.shapeDev.re)
pixscale = 2.75
src.halfsize = int(np.hypot(R, galrad * 5 / pixscale))
#print('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small')
minsb = 0.
fitsky = False
tractor = Tractor(tims, cat)
if use_ceres:
from tractor.ceres_optimizer import CeresOptimizer
tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block)
tractor.freezeParamsRecursive('*')
tractor.thawPathsTo(wanyband)
kwa = dict(fitstat_extras=[('pronexp', [tim.nims for tim in tims])])
t0 = Time()
R = tractor.optimize_forced_photometry(
minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True,
variance=True, shared_params=False,
wantims=wantims, **kwa)
print('unWISE forced photometry took', Time() - t0)
if use_ceres:
term = R.ceres_status['termination']
# Running out of memory can cause failure to converge
# and term status = 2.
# Fail completely in this case.
if term != 0:
print('Ceres termination status:', term)
raise RuntimeError(
'Ceres terminated with status %i' % term)
if wantims:
ims1 = R.ims1
flux_invvars = R.IV
if R.fitstats is not None:
for k in fskeys:
x = getattr(R.fitstats, k)
fitstats[k] = np.array(x).astype(np.float32)
if save_fits:
for i,tim in enumerate(tims):
tile = tim.tile
(dat, mod, ie, chi, roi) = ims1[i]
wcshdr = fitsio.FITSHDR()
tim.wcs.wcs.add_to_header(wcshdr)
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' %
tag, dat, clobber=True, header=wcshdr)
fitsio.write('%s-mod.fits' % tag, mod,
clobber=True, header=wcshdr)
fitsio.write('%s-chi.fits' % tag, chi,
clobber=True, header=wcshdr)
if plots:
# Create models for just the brightest sources
bright_cat = [src for src in cat
if src.getBrightness().getBand(wanyband) > 1000]
print('Bright soures:', len(bright_cat))
btr = Tractor(tims, bright_cat)
for tim in tims:
mod = btr.getModelImage(tim)
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
sig1 = tim.sig1
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: bright-star models' % tag)
ps.savefig()
if get_models:
for i,tim in enumerate(tims):
tile = tim.tile
(dat, mod, ie, chi, roi) = ims1[i]
models[(tile.coadd_id, band)] = (mod, dat, ie, tim.roi, tim.wcs.wcs)
if plots:
for i,tim in enumerate(tims):
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
(dat, mod, ie, chi, roi) = ims1[i]
sig1 = tim.sig1
plt.clf()
plt.imshow(dat, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi, interpolation='nearest', origin='lower',
cmap='gray', vmin=-5, vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = flux_invvars
# Sources out of bounds, eg, never change from their default
# (1-sigma or whatever) initial fluxes. Zero them out instead.
nm[nm_ivar == 0] = 0.
phot.set(wband + '_nanomaggies', nm.astype(np.float32))
phot.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32))
dnm = np.zeros(len(nm_ivar), np.float32)
okiv = (nm_ivar > 0)
dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32)
okflux = (nm > 0)
mag = np.zeros(len(nm), np.float32)
mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])
).astype(np.float32)
dmag = np.zeros(len(nm), np.float32)
ok = (okiv * okflux)
dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok])
).astype(np.float32)
mag[np.logical_not(okflux)] = np.nan
dmag[np.logical_not(ok)] = np.nan
phot.set(wband + '_mag', mag)
phot.set(wband + '_mag_err', dmag)
for k in fskeys:
phot.set(wband + '_' + k, fitstats[k])
phot.set(wband + '_nexp', nexp)
if not np.all(mjd == 0):
phot.set(wband + '_mjd', mjd)
rtn = wphotduck()
rtn.phot = phot
rtn.models = None
rtn.maskmap = None
if get_models:
rtn.models = models
if get_masks:
rtn.maskmap = maskmap
return rtn
def main():
ps = PlotSequence('cov')
survey = LegacySurveyData()
ra, dec = 242.0, 10.2
fn = 'coverage-ccds.fits'
if not os.path.exists(fn):
ccds = survey.get_ccds()
ccds.cut(ccds.filter == 'r')
ccds.cut(ccds.propid == '2014B-0404')
ccds.cut(np.hypot(ccds.ra_bore - ra, ccds.dec_bore - dec) < 2.5)
print(np.unique(ccds.expnum), 'unique exposures')
print('propids', np.unique(ccds.propid))
ccds.writeto(fn)
else:
ccds = fits_table(fn)
plt.clf()
for e in np.unique(ccds.expnum):
I = np.flatnonzero(ccds.expnum == e)
plt.plot(ccds.ra[I], ccds.dec[I], '.')
ps.savefig()
degw = 3.0
pixscale = 10.
W = degw * 3600 / 10.
H = W
hi = 6
cmap = cmap_discretize('jet', hi + 1)
wcs = Tan(ra, dec, W / 2. + 0.5, H / 2. + 0.5, -pixscale / 3600., 0., 0.,
pixscale / 3600., float(W), float(H))
r0, d0 = wcs.pixelxy2radec(1, 1)
r1, d1 = wcs.pixelxy2radec(W, H)
extent = [min(r0, r1), max(r0, r1), min(d0, d1), max(d0, d1)]
for expnums in [
[348666],
[348666, 348710, 348686],
[348659, 348667, 348658, 348666, 348665, 348669, 348668],
None,
[
348683, 348687, 347333, 348686, 348685, 348692, 348694, 348659,
348667, 348658, 348666, 348665, 348669, 348668, 348707, 348709,
348708, 348710, 348711, 348716, 348717
],
]:
nexp = np.zeros((H, W), np.uint8)
for ccd in ccds:
if expnums is not None and not ccd.expnum in expnums:
continue
ccdwcs = survey.get_approx_wcs(ccd)
r, d = ccdwcs.pixelxy2radec(1, 1)
ok, x0, y0 = wcs.radec2pixelxy(r, d)
r, d = ccdwcs.pixelxy2radec(ccd.width, ccd.height)
ok, x1, y1 = wcs.radec2pixelxy(r, d)
xlo = np.clip(int(np.round(min(x0, x1))) - 1, 0, W - 1)
xhi = np.clip(int(np.round(max(x0, x1))) - 1, 0, W - 1)
ylo = np.clip(int(np.round(min(y0, y1))) - 1, 0, H - 1)
yhi = np.clip(int(np.round(max(y0, y1))) - 1, 0, H - 1)
nexp[ylo:yhi + 1, xlo:xhi + 1] += 1
plt.clf()
plt.imshow(nexp,
interpolation='nearest',
origin='lower',
vmin=-0.5,
vmax=hi + 0.5,
cmap=cmap,
extent=extent)
plt.colorbar(ticks=np.arange(hi + 1))
ps.savefig()
O = fits_table('obstatus/decam-tiles_obstatus.fits')
O.cut(np.hypot(O.ra - ra, O.dec - dec) < 2.5)
for p in [1, 2, 3]:
print('Pass', p, 'exposures:', O.r_expnum[O.get('pass') == p])
O.cut(O.get('pass') == 2)
print(len(O), 'pass 2 nearby')
d = np.hypot(O.ra - ra, O.dec - dec)
print('Dists:', d)
I = np.flatnonzero(d < 0.5)
assert (len(I) == 1)
ocenter = O[I[0]]
print('Center expnum', ocenter.r_expnum)
I = np.flatnonzero(d >= 0.5)
O.cut(I)
#center = ccds[ccds.expnum == ocenter.r_expnum]
#p2 = ccds[ccds.
ok, xc, yc = wcs.radec2pixelxy(ocenter.ra, ocenter.dec)
xx, yy = np.meshgrid(np.arange(W) + 1, np.arange(H) + 1)
c_d2 = (xc - xx)**2 + (yc - yy)**2
best = np.ones((H, W), bool)
for o in O:
ok, x, y = wcs.radec2pixelxy(o.ra, o.dec)
d2 = (x - xx)**2 + (y - yy)**2
best[d2 < c_d2] = False
del d2
del c_d2, xx, yy
# plt.clf()
# plt.imshow(best, interpolation='nearest', origin='lower', cmap='gray',
# vmin=0, vmax=1)
# ps.savefig()
plt.clf()
plt.imshow(nexp * best,
interpolation='nearest',
origin='lower',
vmin=-0.5,
vmax=hi + 0.5,
cmap=cmap,
extent=extent)
plt.colorbar(ticks=np.arange(hi + 1))
ps.savefig()
plt.clf()
n, b, p = plt.hist(np.clip(nexp[best], 0, hi),
range=(-0.5, hi + 0.5),
bins=hi + 1)
plt.xlim(-0.5, hi + 0.5)
ps.savefig()
print('b', b)
print('n', n)
print('fracs', np.array(n) / np.sum(n))
print('pcts',
', '.join(['%.1f' % f for f in 100. * np.array(n) / np.sum(n)]))
def main():
global survey
init()
ps = PlotSequence('sky')
# export LEGACY_SURVEY_DIR=/scratch1/scratchdirs/desiproc/DRs/dr4-bootes/legacypipe-dir/
#survey = LegacySurveyData()
survey = get_survey('dr4v2')
ccds = survey.get_ccds_readonly()
print(len(ccds), 'CCDs')
ccds = ccds[ccds.camera == 'mosaic']
print(len(ccds), 'Mosaic CCDs')
# plt.clf()
# plt.hist(ccds.mjd_obs % 1.0, bins=50)
# plt.xlabel('MJD mod 1')
# ps.savefig()
ccds.imjd = np.floor(ccds.mjd_obs).astype(int)
mjds = np.unique(ccds.imjd)
print(len(mjds), 'unique MJDs')
mp = multiproc(nthreads=8, init=init)
allvals = []
medmjds = []
args = []
for kk, mjd in enumerate(mjds):
I = np.flatnonzero(ccds.imjd == mjd)
print('MJD', mjd, ' (%i of %i):' % (kk + 1, len(mjds)), len(I), 'CCDs')
if len(I) == 0:
continue
# pick one near the middle
#i = I[len(I)/2]
for i in [I[len(I) / 4], I[len(I) / 2], I[3 * len(I) / 4]]:
ccd = ccds[i]
key = dict(expnum=ccd.expnum, ccdname=ccd.ccdname)
oldvals = key
vals = cache.find(key)
print('Got', vals.count(), 'cache hits for', key)
gotone = False
for val in vals:
#if 'median_adu' in val:
if 'mode_adu' in val:
print('cache hit:', val)
allvals.append(val)
medmjds.append(mjd)
gotone = True
break
else:
print('partial cache hit:', val)
oldvals = val
###!
cache.delete_one(val)
if gotone:
continue
print('args: key', key, 'oldvals', oldvals)
args.append((mjd, key, oldvals, ccd))
# plt.clf()
# plt.hist(tim.getImage().ravel(), range=(-0.1, 0.1), bins=50,
# histtype='step', color='b')
# plt.axvline(med, color='b', alpha=0.3, lw=2)
# plt.axvline(0., color='k', alpha=0.3, lw=2)
# plt.xlabel('Sky-subtracted pixel values')
# plt.title('Date ' + ccd.date_obs + ': ' + str(im))
# ps.savefig()
if len(args):
meds = mp.map(read_sky_val, args)
print('Medians:', meds)
for (mjd, key, oldvals, ccd), val in zip(args, meds):
if val is None:
continue
allvals.append(val)
medmjds.append(mjd)
medians = []
median_adus = []
mode_adus = []
skyadus = []
keepmjds = []
for mjd, val in zip(medmjds, allvals):
madu = val['median_adu']
if madu is None:
continue
if 'median' in val:
medians.append(val['median'])
else:
from tractor.brightness import NanoMaggies
zpscale = NanoMaggies.zeropointToScale(val['ccdzpt'])
med = (val['median_adu'] - val['skyadu']) / zpscale
print('Computed median diff:', med, 'nmgy')
medians.append(med)
keepmjds.append(mjd)
median_adus.append(val['median_adu'])
mode_adus.append(val['mode_adu'])
skyadus.append(val['skyadu'])
medmjds = keepmjds
median_adus = np.array(median_adus)
mode_adus = np.array(mode_adus)
skyadus = np.array(skyadus)
medmjds = np.array(medmjds)
medians = np.array(medians)
plt.clf()
plt.plot(medmjds, median_adus - skyadus, 'b.')
plt.xlabel('MJD')
#plt.ylabel('Image median after sky subtraction (nmgy)')
plt.ylabel('Image median - SKYADU (ADU)')
#plt.ylim(-0.03, 0.03)
#plt.ylim(-10, 10)
plt.ylim(-1, 1)
plt.axhline(0, color='k', lw=2, alpha=0.5)
ps.savefig()
plt.clf()
plt.plot(medmjds, mode_adus - skyadus, 'b.')
plt.xlabel('MJD')
plt.ylabel('Image mode - SKYADU (ADU)')
plt.ylim(-1, 1)
plt.axhline(0, color='k', lw=2, alpha=0.5)
ps.savefig()
print('Median pcts:', np.percentile(medians, [0, 2, 50, 98, 100]))
mlo, mhi = np.percentile(medians, [2, 98])
I = np.flatnonzero((medians > mlo) * (medians < mhi))
d0 = np.mean(medmjds)
dmjd = medmjds[I] - d0
A = np.zeros((len(dmjd), 2))
A[:, 0] = 1.
A[:, 1] = dmjd
b = np.linalg.lstsq(A, medians[I])[0]
print('Lstsq:', b)
offset = b[0]
slope = b[1]
xx = np.array([dmjd.min(), dmjd.max()])
print('Offset', offset)
print('Slope', slope)
plt.clf()
plt.plot(medmjds, medians, 'b.')
ax = plt.axis()
plt.plot(xx + d0, offset + slope * xx, 'b-')
plt.axis(ax)
plt.xlabel('MJD')
plt.ylabel('Image median after sky subtraction (nmgy)')
plt.ylim(-0.03, 0.03)
plt.axhline(0, color='k', lw=2, alpha=0.5)
ps.savefig()
plt.clf()
plt.plot(median_adus, skyadus, 'b.')
ax = plt.axis()
lo = min(ax[0], ax[2])
hi = max(ax[1], ax[3])
plt.plot([lo, hi], [lo, hi], 'k-', alpha=0.5)
plt.xlabel('Median image ADU')
plt.ylabel('SKYADU')
plt.axis(ax)
ps.savefig()
plt.clf()
plt.plot(medmjds, skyadus / median_adus, 'b.')
plt.xlabel('MJD')
plt.ylim(0.98, 1.02)
plt.axhline(1.0, color='k', alpha=0.5)
plt.ylabel('SKYADU / median image ADU')
ps.savefig()
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--name1', help='Name for first data set')
parser.add_argument('--name2', help='Name for second data set')
parser.add_argument('--plot-prefix', default='compare',
help='Prefix for plot filenames; default "%default"')
parser.add_argument('--match', default=1.0,
help='Astrometric cross-match distance in arcsec')
parser.add_argument('dir1', help='First directory to compare')
parser.add_argument('dir2', help='Second directory to compare')
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = 'Comparing %s to %s' % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile('tractor-.*.fits')
cat1,cat2 = [],[]
for basedir,cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath,dirnames,filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print('Skipping', fn, 'due to filename')
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), 'from', fn)
cat.append(t)
cat1 = merge_tables(cat1, columns='fillzero')
cat2 = merge_tables(cat2, columns='fillzero')
print('Total of', len(cat1), 'from', name1)
print('Total of', len(cat2), 'from', name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)
cat1.cut((cat1.decam_anymask[:,1] == 0) *
(cat1.decam_anymask[:,2] == 0) *
(cat1.decam_anymask[:,4] == 0))
cat2.cut((cat2.decam_anymask[:,1] == 0) *
(cat2.decam_anymask[:,2] == 0) *
(cat2.decam_anymask[:,4] == 0))
print('Total of', len(cat1), 'unmasked from', name1)
print('Total of', len(cat2), 'unmasked from', name2)
I,J,d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match/3600.,
nearest=True)
print(len(I), 'matched')
plt.clf()
plt.hist(d * 3600., 100)
plt.xlabel('Match distance (arcsec)')
plt.title(tt)
ps.savefig()
matched1 = cat1[I]
matched2 = cat2[J]
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
K = np.flatnonzero((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
print('Median mw_trans', band, 'is',
np.median(matched1.decam_mw_transmission[:,iband]))
plt.clf()
plt.errorbar(matched1.decam_flux[K,iband],
matched2.decam_flux[K,iband],
fmt='.', color=cc,
xerr=1./np.sqrt(matched1.decam_flux_ivar[K,iband]),
yerr=1./np.sqrt(matched2.decam_flux_ivar[K,iband]),
alpha=0.1,
)
plt.xlabel('%s flux: %s' % (name1, band))
plt.ylabel('%s flux: %s' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.)
plt.axis([-100, 1000, -100, 1000])
plt.title(tt)
ps.savefig()
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
K = np.flatnonzero(good)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
P = np.flatnonzero(good * psf1 * psf2)
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1./iv1 + 1./iv2)
plt.clf()
plt.plot(mag1[K],
(matched2.decam_flux[K,iband] - matched1.decam_flux[K,iband]) / std[K],
'.', alpha=0.1, color=cc)
plt.plot(mag1[P],
(matched2.decam_flux[P,iband] - matched1.decam_flux[P,iband]) / std[P],
'.', alpha=0.1, color='k')
plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' % (name2, name1, band))
plt.xlabel('%s mag: %s' % (name1, band))
plt.axhline(0, color='k', alpha=0.5)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
plt.clf()
lp,lt = [],[]
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
#good = True
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1./iv1 + 1./iv2)
#std = np.hypot(std, 0.01)
G = np.flatnonzero(good * psf1 * psf2 *
np.isfinite(mag1) *
(mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band]))
n,b,p = plt.hist((matched2.decam_flux[G,iband] -
matched1.decam_flux[G,iband]) / std[G],
range=(-4, 4), bins=50, histtype='step', color=cc,
normed=True)
sig = (matched2.decam_flux[G,iband] -
matched1.decam_flux[G,iband]) / std[G]
print('Raw mean and std of points:', np.mean(sig), np.std(sig))
med = np.median(sig)
rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
print('Median and percentile-based sigma:', med, rsigma)
lp.append(p[0])
lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))
bins = []
gaussint = []
for blo,bhi in zip(b, b[1:]):
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
#bins.extend([blo,bhi])
#gaussint.extend([c,c])
bins.append((blo+bhi)/2.)
gaussint.append(c)
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.title(tt)
plt.xlabel('Flux difference / error (sigma)')
plt.axvline(0, color='k', alpha=0.1)
plt.ylim(0, 0.45)
plt.legend(lp, lt, loc='upper right')
ps.savefig()
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
plt.clf()
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
matched2.decam_flux[:,iband], matched2.decam_flux_ivar[:,iband])
meanmag = NanoMaggies.nanomaggiesToMag((
matched1.decam_flux[:,iband] + matched2.decam_flux[:,iband]) / 2.)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0) *
np.isfinite(mag1) * np.isfinite(mag2))
K = np.flatnonzero(good)
P = np.flatnonzero(good * psf1 * psf2)
plt.errorbar(mag1[K], mag2[K], fmt='.', color=cc,
xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s (mag)' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.)
plt.axis([24, 16, 24, 16])
plt.title(tt)
ps.savefig()
plt.clf()
plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt='.', color=cc,
xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -1, 1])
plt.title(tt)
ps.savefig()
magbins = np.arange(16, 24.001, 0.5)
plt.clf()
plt.plot(mag1[K], (mag2[K]-mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
'.', color=cc, alpha=0.1)
plt.plot(mag1[P], (mag2[P]-mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
plt.clf()
plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)
midmag = []
vals = np.zeros((len(magbins)-1, 5))
median_err1 = []
iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)
# FIXME -- should we do some stats after taking off the mean difference?
for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
midmag.append((mlo+mhi)/2.)
median_err1.append(np.median(magerr1[I]))
if len(I) == 0:
continue
# median and +- 1 sigma quantiles
ybin = y[I]
vals[bini,0] = np.percentile(ybin, 16)
vals[bini,1] = np.median(ybin)
vals[bini,2] = np.percentile(ybin, 84)
# +- 2 sigma quantiles
vals[bini,3] = np.percentile(ybin, 2.3)
vals[bini,4] = np.percentile(ybin, 97.7)
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
'vs expected for Gaussian;', len(ybin), 'points')
# if iqd > iqd_gauss:
# # What error adding in quadrature would you need to make the IQD match?
# err = median_err1[-1]
# target_err = err * (iqd / iqd_gauss)
# sys_err = np.sqrt(target_err**2 - err**2)
# print('--> add systematic error', sys_err)
# ~ Johan's cuts
mlo = 21.
mhi = dict(g=24., r=23.5, z=22.5)[band]
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
ybin = y[I]
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', mlo, mhi, 'band', band, ': IQD is factor',
iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points')
if iqd > iqd_gauss:
# What error adding in quadrature would you need to make
# the IQD match?
err = np.median(np.hypot(magerr1[I], magerr2[I]))
print('Median error (hypot):', err)
target_err = err * (iqd / iqd_gauss)
print('Target:', target_err)
sys_err = np.sqrt((target_err**2 - err**2) / 2.)
print('--> add systematic error', sys_err)
# check...
err_sys = np.hypot(np.hypot(magerr1, sys_err),
np.hypot(magerr2, sys_err))
ysys = (mag2 - mag1) / err_sys
ysys = ysys[I]
print('Resulting median error:', np.median(err_sys[I]))
iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
# Hmmm, this doesn't work... totally overshoots.
plt.errorbar(midmag, vals[:,1], fmt='o', color='b',
yerr=(vals[:,1]-vals[:,0], vals[:,2]-vals[:,1]),
capthick=3, zorder=20)
plt.errorbar(midmag, vals[:,1], fmt='o', color='b',
yerr=(vals[:,1]-vals[:,3], vals[:,4]-vals[:,1]),
capthick=2, zorder=20)
plt.axhline( 1., color='b', alpha=0.2)
plt.axhline(-1., color='b', alpha=0.2)
plt.axhline( 2., color='b', alpha=0.2)
plt.axhline(-2., color='b', alpha=0.2)
for mag,err,y in zip(midmag, median_err1, vals[:,3]):
if not np.isfinite(err):
continue
if y < -6:
continue
plt.text(mag, y-0.1, '%.3f' % err, va='top', ha='center', color='k',
fontsize=10)
plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axvline(21, color='k', alpha=0.3)
plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)
plt.axis([24.1, 16, -6, 6])
plt.title(tt)
ps.savefig()
#magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
if band == 'g':
magbins = [20, 24]
elif band == 'r':
magbins = [20, 23.5]
elif band == 'z':
magbins = [20, 22.5]
slo,shi = -5,5
plt.clf()
ha = dict(bins=25, range=(slo,shi), histtype='step', normed=True)
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
midmag = []
nn = []
rgbs = []
lt,lp = [],[]
for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
if len(I) == 0:
continue
ybin = y[I]
rgb = [0.,0.,0.]
rgb[0] = float(bini) / (len(magbins)-1)
rgb[2] = 1. - rgb[0]
n,b,p = plt.hist(ybin, color=rgb, **ha)
lt.append('mag %g to %g' % (mlo,mhi))
lp.append(p[0])
midmag.append((mlo+mhi)/2.)
nn.append(n)
rgbs.append(rgb)
bins = []
gaussint = []
for blo,bhi in zip(b, b[1:]):
#midbin.append((blo+bhi)/2.)
#gaussint.append(scipy.stats.norm.cdf(bhi) -
# scipy.stats.norm.cdf(blo))
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
bins.extend([blo,bhi])
gaussint.extend([c,c])
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo,shi)
ps.savefig()
bincenters = b[:-1] + (b[1]-b[0])/2.
plt.clf()
lp = []
for n,rgb,mlo,mhi in zip(nn, rgbs, magbins, magbins[1:]):
p = plt.plot(bincenters, n, '-', color=rgb)
lp.append(p[0])
plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo,shi)
ps.savefig()
def stage0(**kwargs):
ps = PlotSequence('cfht')
decals = CfhtDecals()
B = decals.get_bricks()
print 'Bricks:'
B.about()
ra,dec = 190.0, 11.0
#bands = 'ugri'
bands = 'gri'
B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec)))
print 'Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5]
brick = B[0]
pixscale = 0.186
#W,H = 1024,1024
#W,H = 2048,2048
#W,H = 3600,3600
W,H = 4800,4800
targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H)
ccdfn = 'cfht-ccds.fits'
if os.path.exists(ccdfn):
T = fits_table(ccdfn)
else:
T = get_ccd_list()
T.writeto(ccdfn)
print len(T), 'CCDs'
T.cut(ccds_touching_wcs(targetwcs, T))
print len(T), 'CCDs touching brick'
T.cut(np.array([b in bands for b in T.filter]))
print len(T), 'in bands', bands
ims = []
for t in T:
im = CfhtImage(t)
# magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME'])
# fwhm = t.seeing / (pixscale * 3600)
# print '-> FWHM', fwhm, 'pix'
im.seeing = t.seeing
im.pixscale = t.pixscale
print 'seeing', t.seeing
print 'pixscale', im.pixscale*3600, 'arcsec/pix'
im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height)
ims.append(im)
# Read images, clip to ROI
targetrd = np.array([targetwcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
keepims = []
tims = []
for im in ims:
print
print 'Reading expnum', im.expnum, 'name', im.extname, 'band', im.band, 'exptime', im.exptime
band = im.band
wcs = im.read_wcs()
imh,imw = wcs.imageh,wcs.imagew
imgpoly = [(1,1),(1,imh),(imw,imh),(imw,1)]
ok,tx,ty = wcs.radec2pixelxy(targetrd[:-1,0], targetrd[:-1,1])
tpoly = zip(tx,ty)
clip = clip_polygon(imgpoly, tpoly)
clip = np.array(clip)
#print 'Clip', clip
if len(clip) == 0:
continue
x0,y0 = np.floor(clip.min(axis=0)).astype(int)
x1,y1 = np.ceil (clip.max(axis=0)).astype(int)
slc = slice(y0,y1+1), slice(x0,x1+1)
## FIXME -- it seems I got lucky and the cross product is
## negative == clockwise, as required by clip_polygon. One
## could check this and reverse the polygon vertex order.
# dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0]
# dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1]
# cross = dx0*dy1 - dx1*dy0
# print 'Cross:', cross
print 'Image slice: x [%i,%i], y [%i,%i]' % (x0,x1, y0,y1)
print 'Reading image from', im.imgfn, 'HDU', im.hdu
img,imghdr = im.read_image(header=True, slice=slc)
goodpix = (img != 0)
print 'Number of pixels == 0:', np.sum(img == 0)
print 'Number of pixels != 0:', np.sum(goodpix)
if np.sum(goodpix) == 0:
continue
# print 'Image shape', img.shape
print 'Image range', img.min(), img.max()
print 'Goodpix image range:', (img[goodpix]).min(), (img[goodpix]).max()
if img[goodpix].min() == img[goodpix].max():
print 'No dynamic range in image'
continue
# print 'Reading invvar from', im.wtfn, 'HDU', im.hdu
# invvar = im.read_invvar(slice=slc)
# # print 'Invvar shape', invvar.shape
# # print 'Invvar range:', invvar.min(), invvar.max()
# invvar[goodpix == 0] = 0.
# if np.all(invvar == 0.):
# print 'Skipping zero-invvar image'
# continue
# assert(np.all(np.isfinite(img)))
# assert(np.all(np.isfinite(invvar)))
# assert(not(np.all(invvar == 0.)))
# # Estimate per-pixel noise via Blanton's 5-pixel MAD
# slice1 = (slice(0,-5,10),slice(0,-5,10))
# slice2 = (slice(5,None,10),slice(5,None,10))
# # print 'sliced shapes:', img[slice1].shape, img[slice2].shape
# # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape
# # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2])
# # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape
# mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel())
# sig1 = 1.4826 * mad / np.sqrt(2.)
# print 'MAD sig1:', sig1
# # invvar was 1 or 0
# invvar *= (1./(sig1**2))
# medsky = np.median(img[goodpix])
# Read full image for sig1 and sky estimate
fullimg = im.read_image()
fullgood = (fullimg != 0)
# Estimate per-pixel noise via Blanton's 5-pixel MAD
slice1 = (slice(0,-5,10),slice(0,-5,10))
slice2 = (slice(5,None,10),slice(5,None,10))
mad = np.median(np.abs(fullimg[slice1] - fullimg[slice2])[fullgood[slice1] * fullgood[slice2]].ravel())
sig1 = 1.4826 * mad / np.sqrt(2.)
print 'MAD sig1:', sig1
medsky = np.median(fullimg[fullgood])
invvar = np.zeros_like(img)
invvar[goodpix] = 1./sig1**2
# Median-smooth sky subtraction
plt.clf()
dimshow(np.round((img-medsky) / sig1), vmin=-3, vmax=5)
plt.title('Scalar median: %s' % im.name)
ps.savefig()
# medsky = np.zeros_like(img)
# # astrometry.util.util
# median_smooth(img, np.logical_not(goodpix), 256, medsky)
fullmed = np.zeros_like(fullimg)
median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed)
fullmed += medsky
medimg = fullmed[slc]
plt.clf()
dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5)
plt.title('Median filtered: %s' % im.name)
ps.savefig()
#print 'Subtracting median:', medsky
#img -= medsky
img -= medimg
primhdr = im.read_image_primary_header()
magzp = decals.get_zeropoint_for(im)
print 'magzp', magzp
zpscale = NanoMaggies.zeropointToScale(magzp)
print 'zpscale', zpscale
# Scale images to Nanomaggies
img /= zpscale
sig1 /= zpscale
invvar *= zpscale**2
orig_zpscale = zpscale
zpscale = 1.
assert(np.sum(invvar > 0) > 0)
print 'After scaling:'
print 'sig1', sig1
print 'invvar range', invvar.min(), invvar.max()
print 'image range', img.min(), img.max()
assert(np.all(np.isfinite(img)))
assert(np.all(np.isfinite(invvar)))
assert(np.isfinite(sig1))
plt.clf()
lo,hi = -5*sig1, 10*sig1
n,b,p = plt.hist(img[goodpix].ravel(), 100, range=(lo,hi), histtype='step', color='k')
xx = np.linspace(lo, hi, 200)
plt.plot(xx, max(n)*np.exp(-xx**2 / (2.*sig1**2)), 'r-')
plt.xlim(lo,hi)
plt.title('Pixel histogram: %s' % im.name)
ps.savefig()
twcs = ConstantFitsWcs(wcs)
if x0 or y0:
twcs.setX0Y0(x0,y0)
info = im.get_image_info()
fullh,fullw = info['dims']
# read fit PsfEx model
psfex = PsfEx.fromFits(im.psffitfn)
print 'Read', psfex
# HACK -- highly approximate PSF here!
#psf_fwhm = imghdr['FWHM']
#psf_fwhm = im.seeing
psf_fwhm = im.seeing / (im.pixscale * 3600)
print 'PSF FWHM', psf_fwhm, 'pixels'
psf_sigma = psf_fwhm / 2.35
psf = NCircularGaussianPSF([psf_sigma],[1.])
print 'img type', img.dtype
tim = Image(img, invvar=invvar, wcs=twcs, psf=psf,
photocal=LinearPhotoCal(zpscale, band=band),
sky=ConstantSky(0.), name=im.name + ' ' + band)
tim.zr = [-3. * sig1, 10. * sig1]
tim.sig1 = sig1
tim.band = band
tim.psf_fwhm = psf_fwhm
tim.psf_sigma = psf_sigma
tim.sip_wcs = wcs
tim.x0,tim.y0 = int(x0),int(y0)
tim.psfex = psfex
tim.imobj = im
mn,mx = tim.zr
tim.ima = dict(interpolation='nearest', origin='lower', cmap='gray',
vmin=mn, vmax=mx)
tims.append(tim)
keepims.append(im)
ims = keepims
print 'Computing resampling...'
# save resampling params
for tim in tims:
wcs = tim.sip_wcs
subh,subw = tim.shape
subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh)
tim.subwcs = subwcs
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(targetwcs, subwcs, [], 2)
except OverlapError:
print 'No overlap'
continue
if len(Yo) == 0:
continue
tim.resamp = (Yo,Xo,Yi,Xi)
print 'Creating coadds...'
# Produce per-band coadds, for plots
coimgs = []
cons = []
for ib,band in enumerate(bands):
coimg = np.zeros((H,W), np.float32)
con = np.zeros((H,W), np.uint8)
for tim in tims:
if tim.band != band:
continue
(Yo,Xo,Yi,Xi) = tim.resamp
if len(Yo) == 0:
continue
nn = (tim.getInvvar()[Yi,Xi] > 0)
coimg[Yo,Xo] += tim.getImage ()[Yi,Xi] * nn
con [Yo,Xo] += nn
# print
# print 'tim', tim.name
# print 'number of resampled pix:', len(Yo)
# reim = np.zeros_like(coimg)
# ren = np.zeros_like(coimg)
# reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn
# ren[Yo,Xo] = nn
# print 'number of resampled pix with positive invvar:', ren.sum()
# plt.clf()
# plt.subplot(2,2,1)
# mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]]
# print 'Percentiles:', mn,mx
# dimshow(reim, vmin=mn, vmax=mx)
# plt.colorbar()
# plt.subplot(2,2,2)
# dimshow(con)
# plt.colorbar()
# plt.subplot(2,2,3)
# dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1])
# plt.colorbar()
# plt.subplot(2,2,4)
# plt.hist(reim.ravel(), 100, histtype='step', color='b')
# plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r')
# plt.suptitle('%s: %s' % (band, tim.name))
# ps.savefig()
coimg /= np.maximum(con,1)
coimgs.append(coimg)
cons .append(con)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ps.savefig()
plt.clf()
for i,b in enumerate(bands):
plt.subplot(2,2,i+1)
dimshow(cons[i], ticks=False)
plt.title('%s band' % b)
plt.colorbar()
plt.suptitle('Number of exposures')
ps.savefig()
print 'Grabbing SDSS sources...'
bandlist = [b for b in bands]
cat,T = get_sdss_sources(bandlist, targetwcs)
# record coordinates in target brick image
ok,T.tx,T.ty = targetwcs.radec2pixelxy(T.ra, T.dec)
T.tx -= 1
T.ty -= 1
T.itx = np.clip(np.round(T.tx).astype(int), 0, W-1)
T.ity = np.clip(np.round(T.ty).astype(int), 0, H-1)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5)
plt.axis(ax)
plt.title('SDSS sources')
ps.savefig()
print 'Detmaps...'
# Render the detection maps
detmaps = dict([(b, np.zeros((H,W), np.float32)) for b in bands])
detivs = dict([(b, np.zeros((H,W), np.float32)) for b in bands])
for tim in tims:
iv = tim.getInvvar()
psfnorm = 1./(2. * np.sqrt(np.pi) * tim.psf_sigma)
detim = tim.getImage().copy()
detim[iv == 0] = 0.
detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2
detsig1 = tim.sig1 / psfnorm
subh,subw = tim.shape
detiv = np.zeros((subh,subw), np.float32) + (1. / detsig1**2)
detiv[iv == 0] = 0.
(Yo,Xo,Yi,Xi) = tim.resamp
detmaps[tim.band][Yo,Xo] += detiv[Yi,Xi] * detim[Yi,Xi]
detivs [tim.band][Yo,Xo] += detiv[Yi,Xi]
rtn = dict()
for k in ['T', 'coimgs', 'cons', 'detmaps', 'detivs',
'targetrd', 'pixscale', 'targetwcs', 'W','H',
'bands', 'tims', 'ps', 'brick', 'cat']:
rtn[k] = locals()[k]
return rtn
