def run_one_brick(X):
brick, ibrick, nbricks, plots, kwargs = X
survey = LegacySurveyData()
print()
print()
print('Brick', (ibrick + 1), 'of', nbricks, ':', brick.brickname)
dirnm = os.path.join('depthcuts', brick.brickname[:3])
outfn = os.path.join(dirnm, 'ccds-%s.fits' % brick.brickname)
if os.path.exists(outfn):
print('Exists:', outfn)
return 0
H, W = 3600, 3600
pixscale = 0.262
bands = ['g', 'r', 'z']
# Get WCS object describing brick
targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
targetrd = np.array([
targetwcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
gitver = get_git_version()
ccds = survey.ccds_touching_wcs(targetwcs)
if ccds is None:
print('No CCDs actually touching brick')
return 0
print(len(ccds), 'CCDs actually touching brick')
ccds.cut(np.in1d(ccds.filter, bands))
print('Cut on filter:', len(ccds), 'CCDs remain.')
if 'ccd_cuts' in ccds.get_columns():
norig = len(ccds)
ccds.cut(ccds.ccd_cuts == 0)
print(len(ccds), 'of', norig, 'CCDs pass cuts')
else:
print('No CCD cuts')
if len(ccds) == 0:
print('No CCDs left')
return 0
ps = None
if plots:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('depth-%s' % brick.brickname)
splinesky = True
gaussPsf = False
pixPsf = True
do_calibs = False
normalizePsf = True
get_depth_maps = kwargs.pop('get_depth_maps', False)
try:
D = make_depth_cut(survey,
ccds,
bands,
targetrd,
brick,
W,
H,
pixscale,
plots,
ps,
splinesky,
gaussPsf,
pixPsf,
normalizePsf,
do_calibs,
gitver,
targetwcs,
get_depth_maps=get_depth_maps,
**kwargs)
if get_depth_maps:
keep, overlapping, depthmaps = D
else:
keep, overlapping = D
except:
print('Failed to make_depth_cut():')
import traceback
traceback.print_exc()
return -1
print(np.sum(overlapping), 'CCDs overlap the brick')
print(np.sum(keep), 'CCDs passed depth cut')
ccds.overlapping = overlapping
ccds.passed_depth_cut = keep
if not os.path.exists(dirnm):
try:
os.makedirs(dirnm)
except:
pass
if get_depth_maps:
for band, depthmap in depthmaps:
doutfn = os.path.join(dirnm,
'depth-%s-%s.fits' % (brick.brickname, band))
hdr = fitsio.FITSHDR()
# Plug the WCS header cards into these images
targetwcs.add_to_header(hdr)
hdr.delete('IMAGEW')
hdr.delete('IMAGEH')
hdr.add_record(dict(name='EQUINOX', value=2000.))
hdr.add_record(dict(name='FILTER', value=band))
fitsio.write(doutfn, depthmap, header=hdr)
print('Wrote', doutfn)
tmpfn = os.path.join(os.path.dirname(outfn),
'tmp-' + os.path.basename(outfn))
ccds.writeto(tmpfn)
os.rename(tmpfn, outfn)
print('Wrote', outfn)
return 0
catfn = args[2]
outfn = args[3]
if os.path.exists(outfn):
print 'Ouput file exists:', outfn
sys.exit(0)
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
T = exposure_metadata([filename], hdus=[hdu])
print 'Metadata:'
T.about()
decals = Decals()
im = DecamImage(decals, T[0])
tim = im.get_tractor_image(slc=zoomslice, const2psf=True)
print 'Got tim:', tim
if catfn == 'DR1':
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, decals=decals)
plt.savefig('diffrd.png')
plt.clf()
p1 = plt.plot(A.expnum, A.dx * 0.262, 'b.')
p2 = plt.plot(A.expnum, A.dy * 0.262, 'r.')
p3 = plt.plot(A.expnum, A.ra_offset, 'g.')
p4 = plt.plot(A.expnum, A.dec_offset, 'm.')
plt.legend([p1[0], p2[0], p3[0], p4[0]], ['dx', 'dy', 'dRA', 'dDec'],
'lower right')
plt.xlabel('expnum')
plt.savefig('difft.png')
# Now, look at each copilot extension vs im16.
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('astrom')
C.affdx = C.affine[:, 2]
C.affdy = C.affine[:, 5]
refext = 'im16'
Cref = C[C.extension == refext]
print(len(Cref), 'im16 exposures')
ref_expnum = dict([(expnum, i) for i, expnum in enumerate(Cref.expnum)])
plt.clf()
p1 = plt.plot(C.expnum, C.affine[:, 3] - 1, 'b.')
p2 = plt.plot(C.expnum, C.affine[:, 4], 'r.')
plt.plot(Cref.expnum, Cref.affine[:, 4], 'k-')
p3 = plt.plot(C.expnum, C.affine[:, 6], 'c.')
p4 = plt.plot(C.expnum, C.affine[:, 7] - 1, 'm.')
from tractor import *
from common import *
def subplot_grid(ny, nx, i):
y = i / nx
x = i % nx
plt.subplot(ny, nx, 1 + ((ny - 1) - y) * nx + x)
if __name__ == '__main__':
decals = Decals()
ps = PlotSequence('psf')
#B = decals.get_bricks()
T = decals.get_ccds()
T.cut(T.extname == 'S1')
print 'Cut to', len(T)
#print 'Expnums:', T[:10]
T.cut(T.expnum == 348233)
print 'Cut to', len(T)
T.about()
band = T.filter[0]
print 'Band:', band
im = DecamImage(T[0])
def main():
max_radius = 4.
rstep = 0.001
radii = np.arange(rstep / 2., max_radius, rstep)
# emcee
# init_04 = ([25.35, 8.269, -25.22], [0.5045, 0.3587, 0.4013])
# sersic = 0.4
# target = ser_model(radii, sersic)
# initamps,initvars = init_04
# pars = np.append(initamps, initvars)
# sample_sersic(radii, target, pars)
# return
### Need to fit parameters for both N-component mixtures at the
### transition indices (plus some overlap!)
#dser = 0.01
#dser = 0.05
#all_sersics = np.arange(0.6, 0.3-dser/2., -dser)
#all_sersics = np.arange(1.0, 0.8-dser/2., -dser)
#all_sersics = np.arange(0.85, 0.60-dser/2., -dser)
#all_sersics = np.arange(0.75, 0.50-dser/2., -dser)
#all_sersics = np.arange(0.4, 0.25-dser/2., -dser)
# Approaching 0.25, |amps| become large -- exceeding 100
all_sersics = []
all_amps = []
all_vars = []
all_loglikes = []
#all_logprobs = []
sersics7 = np.array([6.1, 6.2, 6.3])
init_7 = ([
9.78462,
5.92346,
3.08624,
1.37584,
0.528399,
0.17315,
0.0471669,
0.0111288,
], [
1.52991,
0.132595,
0.0169103,
0.00245378,
0.000366884,
5.27404e-05,
6.76674e-06,
6.00242e-07,
])
sersics6 = np.array([6.0, 5.5, 5.0, 4.5, 4.0, 3.5, 3.0, 2.7])
init_6 = ([
0.0113067, 0.04898785, 0.18195408, 0.55939775, 1.46288372, 3.28556791,
6.27896305, 9.86946446
], [
6.07125356e-07, 7.02153046e-06, 5.60375312e-05, 3.98494081e-04,
2.72853912e-03, 1.93601976e-02, 1.58544866e-01, 1.95149972e+00
])
sersics4 = np.array([3.5, 3.4, 3.3, 3.2, 3.1, 3.0])
init4 = [
8.2659,
5.31123,
2.79165,
1.16263,
0.383863,
0.0977581,
0.0176055,
0.00172958,
], [
2.04792,
0.283875,
0.051887,
0.00977245,
0.0017372,
0.000267685,
3.11052e-05,
1.88165e-06,
]
n = [None] * 7
priors4 = [
n + [1e-1], n + [1e-2], n + [1e-3], n + [1e-4], n + [1e-5], n + [1e-6]
]
sersics3 = np.array([3.2, 3.1, 3., 2.5, 2., 1.9, 1.8, 1.7, 1.6, 1.5])
init3 = ([7.872, 5.073, 2.661, 1.112, 0.3659, 0.09262, 0.01655],
[2.095, 0.3306, 0.06875, 0.01458, 0.002892, 0.0004967, 6.458e-05])
n = [None] * 6
priors3 = [None] * 5 + [
n + [1e-4], n + [3e-4], n + [1e-5], n + [3e-5], n + [1e-6]
]
sersics15 = np.array([1.55, 1.51, 1.5, 1.4, 1.3, 1.2, 1.1, 1.0])
init15 = ([6.653, 4.537, 1.776, 0.5341, 0.121,
0.01932], [1.83, 0.4273, 0.1187, 0.03255, 0.007875, 0.001491])
# 5->6 ~ 0.70
sersicsA = np.array(
[1.0, 0.95, 0.9, 0.85, 0.8, 0.75, 0.74, 0.73, 0.72, 0.71])
initA = ([6.0, 4.34, 1.18, 0.223, 0.0308,
0.00235], [1.5, 0.461, 0.14, 0.0391, 0.00885, 0.0012])
n = [None] * 5
priorsA = [None] * 6 + [n + [1e-3], n + [3e-4], n + [1e-4], n + [3e-5]]
# 4->5 ~ 0.62
sersicsB = np.array([0.8, 0.75, 0.71, 0.7, 0.65, 0.64, 0.63, 0.62])
initB = ([5.818, 4.1, 0.7976, 0.0947,
0.006378], [1.272, 0.4728, 0.1475, 0.03664, 0.005635])
priorsB = [None] * 4 + [
[None, None, None, None, 0.01],
[None, None, None, None, 0.003],
[None, None, None, None, 0.001],
[None, None, None, None, 0.0003],
]
# 3->4 ~ 0.57
#sersicsC = np.array([0.75, 0.7, 0.63, 0.62, 0.6, 0.58, 0.575, 0.57])
#initC = ([ 6.06735, 3.75046, 0.485251, 0.0287147, ],
# [ 1.12455, 0.455509, 0.12943, 0.0216252, ])
sersicsC = np.array([0.63, 0.62, 0.6, 0.58, 0.575, 0.57])
initC = [
6.24609,
3.29044,
0.320663,
0.016228,
], [
0.995781,
0.471937,
0.144342,
0.0256225,
]
priorsC = [None] * 3 + [
[None, None, None, 1e-3],
[None, None, None, 3e-4],
[None, None, None, 1e-4],
]
### USE THIS 2->3 ~ 0.53
sersicsC2 = np.array([0.57, 0.56, 0.55, 0.54, 0.53, 0.52, 0.515, 0.51])
initC2 = [6.44901, 2.83301, 0.218781], [0.892218, 0.508317, 0.17374]
priorsC2 = [None] * 4 + [[None, None, 0.1], [None, None, 0.01],
[None, None, 0.003], [None, None, 0.001]]
dser = 0.01
sersicsD = np.arange(0.61, 0.50 + dser / 2., -dser)
init_055 = ([7.747, 1.589], [0.8195, 0.4178])
sersicsE = np.array([0.5])
init_05 = ([9.065], [0.7213])
dser = 0.01
sersicsF = np.arange(0.5 - dser, 0.4 - dser / 2., -dser)
init_049 = ([9.24, -0.23], [0.71, 0.33])
sersicsG = np.arange(0.42, 0.29 - dser / 2., -dser)
init_04 = ([
13.5581,
0.433216,
-5.44774,
], [
0.575113,
0.304652,
0.385431,
])
# Ramp between N=3 and N=2 around n~0.4
sersicsFG = np.array([0.39, 0.40, 0.41, 0.42])
initFG = ([
15.4473,
1.43828,
-8.47514,
], [
0.540383,
0.28262,
0.365116,
])
priorsFG = [[None, 1., None], [None, 0.5, None], [None, 0.25, None],
[None, 0.125, None]]
# Ramp between N=4 and N=2, n~0.6
sersicsCD = np.array([0.6, 0.59, 0.58, 0.57, 0.57,
0.57]) #, 0.56, 0.55, 0.54])
initCD = ([
6.39787,
3.02775,
0.256118,
0.0121954,
], [
0.942441,
0.480798,
0.152128,
0.027722,
])
priorsCD = [
[None, None, None, 0.01],
[None, None, None, 0.003],
[None, None, None, 0.001],
[None, None, None, 0.0003],
[None, None, None, 0.0001],
[None, None, None, 0.00003],
]
# priorsCD = [ [None,None, 0.3, 0.01],
# [None,None, 0.1, 0.003],
# [None,None, 0.03, 0.001],
# [None,None, 0.01, 0.0003],
# [None,None, 0.003, 0.0001],
# [None,None, 0.001, 0.00003],
# [None,None, 0.0003, 0.00003],
# ]
sersicsCD2 = np.array([0.57, 0.56, 0.55, 0.54])
initCD2 = [7.82365, 1.63757, 0.0411003], [0.840165, 0.373703, 0.0649509]
priorsCD2 = [
[None, None, 0.3],
[None, None, 0.1],
[None, None, 0.03],
[None, None, 0.01],
# [None,None, 0.003],
# [None,None, 0.001],
# [None,None, 0.0003],
]
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('fit')
for sersics, ini, priors, patch in [
# (sersics7, init_7, None, False),
# (sersics6, init_6, None, False),
(sersics4, init4, priors4, False),
#(sersics3, init3, priors3, False),
#(sersics15, init15, None, False),
#(sersicsA, initA, priorsA, False),
#(sersicsB, initB, priorsB, False),
#(sersicsC, initC, priorsC, False),
# (sersicsD, init_055, None, False),
# (sersicsE, init_05, None, False),
# (sersicsF, init_049, None, True),
# (sersicsF, init_049, None, False),
# (sersicsG, init_04, None, False),
#(sersicsFG, initFG, priorsFG, False),
#(sersicsC, init_07, None, False),
#(sersicsC2, initC2, priorsC2, False),
#(sersicsD, init_055, None, False),
#(sersicsCD, initCD, priorsCD, False),
#(sersicsCD2, initCD2, priorsCD2, False),
]:
amps, vars, loglikes = fit_range(sersics,
radii,
ini,
priors=priors,
ps=ps)
# amps2,vars2,loglikes2 = fit_range(reversed(sersics), radii,
# (amps[-1], vars[-1]))
#
# amps3,vars3,loglikes3 = fit_range(sersics, radii,
# (amps2[-1], vars2[-1]))
#
# amps4,vars4,loglikes4 = fit_range(reversed(sersics), radii,
# (amps3[-1], vars3[-1]))
# amps,vars = amps4,vars4
print('After fitting range forward:', sersics)
#print('amps:', amps)
#print('vars:', vars)
print('log-likes:', loglikes)
# print('After fitting range backwards:', sersics)
# #print('amps:', amps2)
# #print('vars:', vars2)
# print('log-likes:', loglikes2)
#
# print('After fitting range forwards again:', sersics)
# print('log-likes:', loglikes3)
#
# print('After fitting range backwards again:', sersics)
# print('log-likes:', loglikes4)
for sersic, fitamps, fitvars in reversed(list(zip(sersics, amps,
vars))):
print('(%.2f' % sersic, ', [',
''.join(['%g, ' % a for a in fitamps]), '], [',
''.join(['%g, ' % v for v in fitvars]), ']),')
# the one after 0.5: patch across 0.5
if patch:
amps_before = all_amps[-2]
vars_before = all_vars[-2]
amps_after = amps[0]
vars_after = vars[0]
amps_half = all_amps[-1]
vars_half = all_vars[-1]
assert (len(amps_before) == len(amps_after))
#print('vars before:', vars_before)
#print('vars after:', vars_after)
#print('vars half:', vars_half)
patchvars = np.append([vars_half[0]],
np.sqrt(vars_before[1:] * vars_after[1:]))
#print('Patch variances:', patchvars)
patchamps = np.array([amps_half[0]] + [0.] * (len(amps_after) - 1))
#print('Patch amps:', patchamps)
#all_sersics.append(sersics[0])
#all_amps.append(patchamps)
#all_vars.append(patchvars)
#all_loglikes.append(loglikes[0])
# Replace 0.5 fit
all_amps[-1] = patchamps
all_vars[-1] = patchvars
all_sersics.extend(sersics)
all_amps.extend(amps)
all_vars.extend(vars)
all_loglikes.extend(loglikes)
print()
print('All fit results:')
print()
for sersic, fitamps, fitvars in reversed(
list(zip(all_sersics, all_amps, all_vars))):
print('(%.3g' % sersic, ', [', ''.join(['%g, ' % a for a in fitamps]),
'], [', ''.join(['%g, ' % v for v in fitvars]), ']),')
all_sersics = np.array(all_sersics)
plt.figure(figsize=(12, 6))
plt.subplot(1, 2, 1)
#for i in range(3):
for i in range(8):
J, = np.nonzero([len(a) > i for a in all_amps])
plt.plot(all_sersics[J], np.array([all_amps[j][i] for j in J]), 'o-')
plt.xlabel('Sersic index')
plt.ylabel('Mixture amplitudes')
plt.axhline(0., color='k')
plt.yscale('symlog', linthreshy=1e-3)
plt.subplot(1, 2, 2)
#for i in range(3):
for i in range(8):
J, = np.nonzero([len(a) > i for a in all_vars])
plt.plot(all_sersics[J], np.array([all_vars[j][i] for j in J]), 'o-')
plt.xlabel('Sersic index')
plt.ylabel('Mixture variances')
plt.yscale('log')
plt.suptitle('Gaussian Mixture approximations to Sersic profiles')
plt.savefig('/tmp/mix.png')
plt.xscale('log')
xt = [6, 5, 4, 3, 2, 1, 0.5, 0.4, 0.3]
plt.xticks(xt, ['%g' % x for x in xt])
plt.subplot(1, 2, 1)
plt.xscale('log')
plt.xticks(xt, ['%g' % x for x in xt])
plt.savefig('/tmp/mix2.png')
plt.clf()
plt.subplot(1, 2, 1)
for s, a, v in zip(all_sersics, all_amps, all_vars):
plt.plot(s + np.zeros_like(a), a, '-')
plt.xlabel('Sersic index')
plt.ylabel('Mixture amplitudes')
plt.axhline(0., color='k')
plt.yscale('symlog', linthreshy=1e-3)
plt.xscale('log')
xt = [6, 5, 4, 3, 2, 1, 0.5, 0.4, 0.3]
plt.xticks(xt, ['%g' % x for x in xt])
plt.subplot(1, 2, 2)
for s, a, v in zip(all_sersics, all_amps, all_vars):
plt.plot(s + np.zeros_like(v), v, '.')
plt.xlabel('Sersic index')
plt.ylabel('Mixture variances')
plt.yscale('log')
plt.xscale('log')
xt = [6, 5, 4, 3, 2, 1, 0.5, 0.4, 0.3]
plt.xticks(xt, ['%g' % x for x in xt])
plt.savefig('/tmp/mix3.png')
return
#all_sersics = np.arange(1.0, 0.25-dser/2, -dser)
# (Re-)Initializations: iser -> (amps, vars)
inits = dict([
# 1.0 -- paper 1 values for lux, 6 components
# (0, ([2.34853813e-03, 3.07995260e-02, 2.23364214e-01,
# 1.17949102e+00, 4.33873750e+00, 5.99820770e+00],
# [1.20078965e-03, 8.84526493e-03, 3.91463084e-02,
# 1.39976817e-01, 4.60962500e-01, 1.50159566e+00])),
(np.argmin(np.abs(all_sersics - 1.0)),
([6.0, 4.34, 1.18, 0.223, 0.0308,
0.00235], [1.5, 0.461, 0.14, 0.0391, 0.00885, 0.0012])),
# 0.85 -- one amplitude goes negative. Smallest amplitude ~ 1e-3,
# variances reasonably spaced
# 5 components
(np.argmin(np.abs(all_sersics - 0.85)),
([5.81, 4.2, 0.946, 0.148,
0.018], [1.35, 0.475, 0.152, 0.044, 0.0104])),
# 0.6
#(np.argmin(np.abs(all_sersics - 0.6)),
# ([8.5, 1.15], [0.83, 0.25])),
# Nothing wrong with the 5-component fit at 0.6; smallest component ~ 1e-3
(np.argmin(np.abs(all_sersics - 0.6)),
([6.236, 3.169, 0.3216, 0.02589], [0.9646, 0.4898, 0.1714, 0.04615])),
# Pattern changes at 0.55; all amps still positive, smallest ~ 1e-2.5
(np.argmin(np.abs(all_sersics - 0.55)), ([6.522, 2.721, 0.1968],
[0.873, 0.5162, 0.1796])),
# 0.5: one component only
(np.argmin(np.abs(all_sersics - 0.5)), ([9.], [0.72])),
# First one below 0.5: flip one amp negative.
(np.argmin(np.abs(all_sersics - (0.5 - dser))), ([9.24,
-0.23], [0.71,
0.33])),
# Switch to 3 components
#(np.argmin(np.abs(all_sersics - 0.37)),
# ([33.85, -35.89, 10.25], [0.46, 0.37, 0.31])),
#([15, -15, 10], [0.5, 0.35, 0.3])),
#(np.argmin(np.abs(all_sersics - 0.39)),
# ([31.13, -31.29, 8.43], [0.47, 0.38, 0.32])),
(np.argmin(np.abs(all_sersics - 0.40)), ([31.34, -31.92,
8.92], [0.48, 0.39, 0.34])),
#(np.argmin(np.abs(all_sersics - 0.41)),
# ([25.57, -25.94, 8.78], [0.50, 0.40, 0.36])),
# They cross over!
#(np.argmin(np.abs(all_sersics - 0.42)),
# ([24.05, -18.77, 3.19], [0.52, 0.43, 0.38])),
])
for iser, sersic in enumerate(all_sersics):
#for iser,sersic in enumerate(np.append(all_sersics, list(reversed(all_sersics[:-1])))):
#all_ser2.append(sersic)
target = ser_model(radii, sersic)
if iser in inits:
a, v = inits[iser]
initamps = np.array(a)
initvars = np.array(v)
print('Sersic index %.2f' % sersic,
': resetting initialization to', initamps, initvars)
print('Sersic %.2f' % sersic, 'initialized at amps',
', '.join(['%.4g' % a for a in initamps]), 'vars',
', '.join(['%.4g' % v for v in initvars]))
pars = np.append(initamps, initvars)
pars = optimize_mixture(pars, radii, target)
fitamps = pars[:len(pars) // 2].copy()
fitvars = pars[len(pars) // 2:].copy()
all_amps.append(fitamps)
all_vars.append(fitvars)
#all_logprobs.append(mixture_logprob(pars, radii, target))
all_loglikes.append(mixture_loglikelihood(pars, radii, target))
initamps = fitamps
initvars = fitvars
print('Sersic %.2f' % sersic, 'amps',
', '.join(['%.4g' % a for a in fitamps]), 'vars',
', '.join(['%.4g' % v for v in fitvars]))
#print('Sersic %.2f' % sersic, 'amps', ', '.join(['%g'%a for a in fitamps]), 'vars', ', '.join(['%g'%v for v in fitvars]))
all_amps = np.array(all_amps)
all_vars = np.array(all_vars)
#all_logprobs = np.array(all_logprobs)
all_loglikes = np.array(all_loglikes)
plt.figure(figsize=(12, 6))
plt.subplot(1, 2, 1)
#for i in range(3):
for i in range(6):
J, = np.nonzero([len(a) > i for a in all_amps])
plt.plot(all_sersics[J], np.array([all_amps[j][i] for j in J]), 'o-')
plt.xlabel('Sersic index')
plt.ylabel('Mixture amplitudes')
plt.axhline(0., color='k')
plt.yscale('symlog', linthreshy=1e-3)
plt.subplot(1, 2, 2)
#for i in range(3):
for i in range(6):
J, = np.nonzero([len(a) > i for a in all_vars])
plt.plot(all_sersics[J], np.array([all_vars[j][i] for j in J]), 'o-')
plt.xlabel('Sersic index')
plt.ylabel('Mixture variances')
plt.yscale('log')
plt.suptitle('Gaussian Mixture approximations to Sersic profiles')
plt.savefig('/tmp/mix.png')
plt.clf()
plt.plot(all_sersics, all_loglikes, 'ko-')
plt.xlabel('Sersic index')
plt.ylabel('Mixture log-likelihood')
plt.suptitle('Gaussian Mixture approximations to Sersic profiles')
mx = max(all_loglikes)
plt.ylim(mx - 200, mx)
plt.savefig('/tmp/loglike.png')
plt.figure(figsize=(16, 4))
xx = radii[radii <= 4]
for i, (n, a, v) in enumerate(zip(all_sersics, all_amps, all_vars)):
plt.clf()
plot_one_row(n, a, v, xx)
#plt.suptitle('Logprob %.2f' % lnp)
plt.savefig('/tmp/ser-%02i.png' % i)
def main():
ps = PlotSequence('conv')
S = 51
center = S/2
print('Center', center)
#for psf_sigma in [2., 1.5, 1.]:
for psf_sigma in [2.]:
rms2 = []
x = np.arange(S)
y = np.arange(S)
xx,yy = np.meshgrid(x, y)
scale = 1.5 / psf_sigma
pixpsf = render_airy((scale, center), x, y)
psf = (scale,center)
eval_psf = render_airy
plt.clf()
plt.subplot(2,1,1)
plt.plot(x, pixpsf[center,:], 'b-')
plt.plot(x, pixpsf[:,center], 'r-')
plt.subplot(2,1,2)
plt.plot(x, np.maximum(1e-16, pixpsf[center,:]), 'b-')
plt.plot(x, np.maximum(1e-16, pixpsf[:,center]), 'r-')
plt.yscale('log')
ps.savefig()
plt.clf()
plt.imshow(pixpsf, interpolation='nearest', origin='lower')
ps.savefig()
plt.clf()
plt.imshow(np.log10(np.maximum(1e-16, pixpsf)),
interpolation='nearest', origin='lower')
plt.colorbar()
plt.title('log PSF')
ps.savefig()
# psf
#psf = scipy.stats.norm(loc=center + 0.5, scale=psf_sigma)
# plt.clf()
# plt.imshow(Pcdf, interpolation='nearest', origin='lower')
# ps.savefig()
# #Pcdf = psf.cdf(xx) * psf.cdf(yy)
# #pixpsf = integrate_gaussian(psf, xx, yy)
#
# padpsf = np.zeros((S*2-1, S*2-1))
# ph,pw = pixpsf.shape
# padpsf[S/2:S/2+ph, S/2:S/2+pw] = pixpsf
# Fpsf = np.fft.rfft2(padpsf)
#
# padh,padw = padpsf.shape
# v = np.fft.rfftfreq(padw)
# w = np.fft.fftfreq(padh)
# fmax = max(max(np.abs(v)), max(np.abs(w)))
# cut = fmax / 2. * 1.000001
# #print('Frequence cut:', cut)
# Ffiltpsf = Fpsf.copy()
# #print('Ffiltpsf', Ffiltpsf.shape)
# #print((np.abs(w) < cut).shape)
# #print((np.abs(v) < cut).shape)
# Ffiltpsf[np.abs(w) > cut, :] = 0.
# Ffiltpsf[:, np.abs(v) > cut] = 0.
# #print('pad v', v)
# #print('pad w', w)
#
# filtpsf = np.fft.irfft2(Ffiltpsf, s=(padh,padw))
#
# print('filtered PSF real', np.max(np.abs(filtpsf.real)))
# print('filtered PSF imag', np.max(np.abs(filtpsf.imag)))
#
# plt.clf()
# plt.subplot(2,3,1)
# dimshow(Fpsf.real)
# plt.colorbar()
# plt.title('Padded PSF real')
# plt.subplot(2,3,4)
# dimshow(Fpsf.imag)
# plt.colorbar()
# plt.title('Padded PSF imag')
#
# plt.subplot(2,3,2)
# dimshow(Ffiltpsf.real)
# plt.colorbar()
# plt.title('Filt PSF real')
# plt.subplot(2,3,5)
# dimshow(Ffiltpsf.imag)
# plt.colorbar()
# plt.title('Filt PSF imag')
#
# plt.subplot(2,3,3)
# dimshow(filtpsf.real)
# plt.title('PSF real')
# plt.colorbar()
#
# plt.subplot(2,3,6)
# dimshow(filtpsf.imag)
# plt.title('PSF imag')
# plt.colorbar()
#
# ps.savefig()
#
#
# pixpsf = filtpsf
gal_sigmas = [2, 1, 0.5, 0.25]
for gal_sigma in gal_sigmas:
# plt.clf()
# plt.imshow(Gcdf, interpolation='nearest', origin='lower')
# plt.savefig('dcdf.png')
# plt.clf()
# plt.imshow(np.exp(-0.5 * ((xx-center)**2 + (yy-center)**2)/2.**2),
# interpolation='nearest', origin='lower')
# plt.savefig('g.png')
# my convolution
pixscale = 1.
cd = pixscale * np.eye(2) / 3600.
P,FG,Gmine,v,w = galaxy_psf_convolution(
gal_sigma, 0., 0., GaussianGalaxy, cd,
0., 0., pixpsf, debug=True)
#print('v:', v)
#print('w:', w)
#print('P:', P.shape)
print()
print('PSF %g, Gal %g' % (psf_sigma, gal_sigma))
rmax = np.argmax(np.abs(w))
cmax = np.argmax(np.abs(v))
l2_rmax = np.sqrt(np.sum(P[rmax,:].real**2 + P[rmax,:].imag**2))
l2_cmax = np.sqrt(np.sum(P[:,cmax].real**2 + P[:,cmax].imag**2))
print('PSF L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax)
l2_rmax = np.sqrt(np.sum(FG[rmax,:].real**2 + FG[rmax,:].imag**2))
l2_cmax = np.sqrt(np.sum(FG[:,cmax].real**2 + FG[:,cmax].imag**2))
print('Gal L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax)
C = P * FG
l2_rmax = np.sqrt(np.sum(C[rmax,:].real**2 + C[rmax,:].imag**2))
l2_cmax = np.sqrt(np.sum(C[:,cmax].real**2 + C[:,cmax].imag**2))
print('PSF*Gal L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax)
print()
Fpsf, Fgal = compare_subsampled(
S, 1, ps, psf, pixpsf, Gmine,v,w,
gal_sigma, psf_sigma, cd, get_ffts=True, eval_psf=eval_psf)
plt.clf()
plt.subplot(2,4,1)
dimshow(P.real)
plt.colorbar()
plt.title('PSF real')
plt.subplot(2,4,5)
dimshow(P.imag)
plt.colorbar()
plt.title('PSF imag')
plt.subplot(2,4,2)
dimshow(FG.real)
plt.colorbar()
plt.title('Gal real')
plt.subplot(2,4,6)
dimshow(FG.imag)
plt.colorbar()
plt.title('Gal imag')
plt.subplot(2,4,3)
dimshow((P * FG).real)
plt.colorbar()
plt.title('P*Gal real')
plt.subplot(2,4,7)
dimshow((P * FG).imag)
plt.colorbar()
plt.title('P*Gal imag')
plt.subplot(2,4,4)
dimshow((Fgal).real)
plt.colorbar()
plt.title('pixGal real')
plt.subplot(2,4,8)
dimshow((Fgal).imag)
plt.colorbar()
plt.title('pixGal imag')
plt.suptitle('PSF %g, Gal %g' % (psf_sigma, gal_sigma))
ps.savefig()
subsample = [1,2,4]
rms1 = []
for s in subsample:
rms = compare_subsampled(S, s, ps, psf, pixpsf, Gmine,v,w, gal_sigma, psf_sigma, cd, eval_psf=eval_psf)
rms1.append(rms)
rms2.append(rms1)
print()
print('PSF sigma =', psf_sigma)
print('RMSes:')
for rms1,gal_sigma in zip(rms2, gal_sigmas):
print('Gal sigma', gal_sigma, 'rms:',
', '.join(['%.3g' % r for r in rms1]))
def main(survey=None, opt=None, args=None):
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if args is None:
args = sys.argv[1:]
print('forced_photom.py', ' '.join(args))
if opt is None:
parser = get_parser()
opt = parser.parse_args(args)
import logging
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
t0 = Time()
if survey is None:
survey = LegacySurveyData(survey_dir=opt.survey_dir,
cache_dir=opt.cache_dir,
output_dir=opt.out_dir)
if opt.skip:
if opt.out is not None:
outfn = opt.out
else:
outfn = survey.find_file('forced',
output=True,
camera=opt.camera,
expnum=opt.expnum)
if os.path.exists(outfn):
print('Ouput file exists:', outfn)
return 0
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
return -1
if opt.out is None and opt.out_dir is None:
print('Must supply either --out or --out-dir')
return -1
if opt.expnum is None and opt.out is None:
print('If no --expnum is given, must supply --out filename')
return -1
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Cache CCDs files before the find_ccds call...
# Copy required files into the cache?
if opt.pre_cache:
def copy_files_to_cache(fns):
for fn in fns:
cachefn = fn.replace(survey.survey_dir, survey.cache_dir)
if not cachefn.startswith(survey.cache_dir):
print('Skipping', fn)
continue
outdir = os.path.dirname(cachefn)
trymakedirs(outdir)
print('Copy', fn)
print(' to', cachefn)
shutil.copyfile(fn, cachefn)
assert (survey.cache_dir is not None)
fnset = set()
fn = survey.find_file('bricks')
fnset.add(fn)
fns = survey.find_file('ccd-kds')
fnset.update(fns)
copy_files_to_cache(fnset)
# Read metadata from survey-ccds.fits table
ccds = survey.find_ccds(camera=opt.camera,
expnum=opt.expnum,
ccdname=opt.ccdname)
print(len(ccds), 'with camera', opt.camera, 'and expnum', opt.expnum,
'and ccdname', opt.ccdname)
# sort CCDs
ccds.cut(np.lexsort((ccds.ccdname, ccds.expnum, ccds.camera)))
# If there is only one catalog survey_dir, we pass it to get_catalog_in_wcs
# as the northern survey.
catsurvey_north = survey
catsurvey_south = None
if opt.catalog_dir_north is not None:
assert (opt.catalog_dir_south is not None)
assert (opt.catalog_resolve_dec_ngc is not None)
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
elif opt.catalog_dir is not None:
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)
# Copy required CCD & calib files into the cache?
if opt.pre_cache:
assert (survey.cache_dir is not None)
fnset = set()
for ccd in ccds:
im = survey.get_image_object(ccd)
for key in im.get_cacheable_filename_variables():
fn = getattr(im, key)
if fn is None or not (os.path.exists(fn)):
continue
fnset.add(fn)
copy_files_to_cache(fnset)
args = []
for ccd in ccds:
args.append((survey, catsurvey_north, catsurvey_south,
opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))
if opt.threads:
from astrometry.util.multiproc import multiproc
from astrometry.util.timingpool import TimingPool, TimingPoolMeas
pool = TimingPool(opt.threads)
poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(poolmeas)
mp = multiproc(None, pool=pool)
tm = Time()
FF = mp.map(bounce_one_ccd, args)
print('Multi-processing forced-phot:', Time() - tm)
del mp
Time.measurements.remove(poolmeas)
del poolmeas
pool.close()
pool.join()
del pool
else:
FF = map(bounce_one_ccd, args)
FF = [F for F in FF if F is not None]
if len(FF) == 0:
print('No photometry results to write.')
return 0
# Keep only the first header
_, version_hdr, _, _ = FF[0]
# unpack results
outlier_masks = [m for _, _, m, _ in FF]
outlier_hdrs = [h for _, _, _, h in FF]
FF = [F for F, _, _, _ in FF]
F = merge_tables(FF)
if len(ccds):
version_hdr.delete('CPHDU')
version_hdr.delete('CCDNAME')
from legacypipe.utils import add_bits
from legacypipe.bits import DQ_BITS
add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
for i, ap in enumerate(apertures_arcsec):
version_hdr.add_record(
dict(name='APRAD%i' % i,
value=ap,
comment='(optical) Aperture radius, in arcsec'))
unitmap = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2',
'apflux': 'nanomaggy',
'apflux_ivar': '1/nanomaggy^2',
'psfdepth': '1/nanomaggy^2',
'galdepth': '1/nanomaggy^2',
'sky': 'nanomaggy/arcsec^2',
'psfsize': 'arcsec',
'fwhm': 'pixels',
'ccdrarms': 'arcsec',
'ccddecrms': 'arcsec',
'ra': 'deg',
'dec': 'deg',
'skyrms': 'counts/sec',
'dra': 'arcsec',
'ddec': 'arcsec',
'dra_ivar': '1/arcsec^2',
'ddec_ivar': '1/arcsec^2'
}
columns = F.get_columns()
order = [
'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'fwhm', 'ccd_cuts',
'airmass', 'sky', 'skyrms', 'psfdepth', 'galdepth', 'ccdzpt',
'ccdrarms', 'ccddecrms', 'ccdphrms', 'ra', 'dec', 'flux', 'flux_ivar',
'fracflux', 'rchisq', 'fracmasked', 'fracin', 'apflux', 'apflux_ivar',
'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar'
]
columns = [c for c in order if c in columns]
units = [unitmap.get(c, '') for c in columns]
if opt.out is not None:
outdir = os.path.dirname(opt.out)
if len(outdir):
trymakedirs(outdir)
tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.out))
fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
F.writeto(tmpfn, units=units, append=True, columns=columns)
os.rename(tmpfn, opt.out)
print('Wrote', opt.out)
else:
with survey.write_output('forced',
camera=opt.camera,
expnum=opt.expnum) as out:
F.writeto(None,
fits_object=out.fits,
primheader=version_hdr,
units=units,
columns=columns)
print('Wrote', out.real_fn)
if opt.outlier_mask is not None:
# Add outlier bit meanings to the primary header
version_hdr.add_record(
dict(name='COMMENT', value='Outlier mask bit meanings'))
version_hdr.add_record(
dict(name='OUTL_POS',
value=1,
comment='Outlier mask bit for Positive outlier'))
version_hdr.add_record(
dict(name='OUTL_NEG',
value=2,
comment='Outlier mask bit for Negative outlier'))
if opt.outlier_mask == 'default':
outdir = os.path.join(opt.out_dir, 'outlier-masks')
camexp = set(zip(ccds.camera, ccds.expnum))
for c, e in camexp:
I = np.flatnonzero((ccds.camera == c) * (ccds.expnum == e))
ccd = ccds[I[0]]
imfn = ccd.image_filename.strip()
outfn = os.path.join(outdir, imfn.replace('.fits',
'-outlier.fits'))
trymakedirs(outfn, dir=True)
tempfn = outfn.replace('.fits', '-tmp.fits')
with fitsio.FITS(tempfn, 'rw', clobber=True) as fits:
fits.write(None, header=version_hdr)
for i in I:
mask = outlier_masks[i]
_, _, _, meth, tile = survey.get_compression_args(
'outliers_mask', shape=mask.shape)
fits.write(mask,
header=outlier_hdrs[i],
extname=ccds.ccdname[i],
compress=meth,
tile_dims=tile)
os.rename(tempfn, outfn)
print('Wrote', outfn)
elif opt.outlier_mask is not None:
with fitsio.FITS(opt.outlier_mask, 'rw', clobber=True) as F:
F.write(None, header=version_hdr)
for i, (hdr, mask) in enumerate(zip(outlier_hdrs, outlier_masks)):
_, _, _, meth, tile = survey.get_compression_args(
'outliers_mask', shape=mask.shape)
F.write(mask,
header=hdr,
extname=ccds.ccdname[i],
compress=meth,
tile_dims=tile)
print('Wrote', opt.outlier_mask)
tnow = Time()
print('Total:', tnow - t0)
return 0
if is_galaxy and t.type == 1:
# deV
src = DevGalaxy(pos, bright, shape)
elif is_galaxy and t.type == 2:
# exp
src = ExpGalaxy(pos, bright, shape)
else:
src = PointSource(pos, bright)
srcs.append(src)
return srcs
if __name__ == '__main__':
ps = PlotSequence('cfht')
if False:
# Don't start looking at the ACS image. Don't.
imgfn = 'cfht/acs_I_030mas_065_sci.VISRES.fits'
wtfn = 'cfht/acs_I_030mas_065_wht.VISRES.fits'
flagfn = 'cfht/acs_I_030mas_065_flg.VISRES.fits'
catfn = 'cfht/acs_I_030mas_065_sci.VISRES.ldac'
cat = fits_table(catfn, hdu=2)
cat.ra = cat.alpha_j2000
cat.dec = cat.delta_j2000
img, imghdr = fitsio.read(imgfn, ext=ext, header=True)
print('Read image', img.shape, img.dtype)
img = img.astype(np.float32)
plt.xlabel('Depth: %s band' % band)
plt.ylabel('Number of pixels')
plt.title('Depth: Galaxy, %s' % band)
ps.savefig()
if __name__ == '__main__':
north = False
if north == True:
outfn = 'dr8-north-depth-concat.fits'
summaryfn = 'dr8-north-depth-summary.fits'
allfn = 'dr8-north-depth.fits'
basedir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8/north'
summarize_depths(basedir, outfn, summaryfn, allfn)
ps = PlotSequence('depth')
summary_plots(summaryfn, ps, 'BASS+MzLS DR8')
else:
outfn = 'dr8-south-depth-concat.fits'
summaryfn = 'dr8-south-depth-summary.fits'
allfn = 'dr8-south-depth.fits'
basedir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8/south'
summarize_depths(basedir, outfn, summaryfn, allfn)
ps = PlotSequence('depth')
summary_plots(summaryfn, ps, 'DECaLS DR8')
import sys
sys.exit(0)
import matplotlib
matplotlib.use('Agg')
import pylab as plt
import numpy as np
import fitsio
from astrometry.util.fits import fits_table, merge_tables
from astrometry.util.util import wcs_pv2sip_hdr
from astrometry.util.plotutils import PlotSequence
if __name__ == '__main__':
import astrometry
ps = PlotSequence('pretty')
catdir = os.path.join(os.path.dirname(astrometry.__file__), 'catalogs')
fn = os.path.join(catdir, 'ngc2000.fits')
NGC = fits_table(fn)
NGC.name = np.array(['NGC %i' % ngcnum for ngcnum in NGC.ngcnum])
#NGC.delete_column('ngcnum')
IC = fits_table(os.path.join(catdir, 'ic2000.fits'))
IC.name = np.array(['IC %i' % icnum for icnum in IC.icnum])
#IC.delete_column('icnum')
UGC = fits_table(os.path.join(catdir, 'ugc.fits'))
UGC.name = np.array(['UGC %i' % ugcnum for ugcnum in UGC.ugcnum])
#UGC.delete_column('ugcnum')
def main():
global survey
init()
ps = PlotSequence('sky')
# export LEGACY_SURVEY_DIR=/scratch1/scratchdirs/desiproc/DRs/dr4-bootes/legacypipe-dir/
#survey = LegacySurveyData()
survey = get_survey('dr4v2')
ccds = survey.get_ccds_readonly()
print(len(ccds), 'CCDs')
ccds = ccds[ccds.camera == 'mosaic']
print(len(ccds), 'Mosaic CCDs')
# plt.clf()
# plt.hist(ccds.mjd_obs % 1.0, bins=50)
# plt.xlabel('MJD mod 1')
# ps.savefig()
ccds.imjd = np.floor(ccds.mjd_obs).astype(int)
mjds = np.unique(ccds.imjd)
print(len(mjds), 'unique MJDs')
mp = multiproc(nthreads=8, init=init)
allvals = []
medmjds = []
args = []
for kk, mjd in enumerate(mjds):
I = np.flatnonzero(ccds.imjd == mjd)
print('MJD', mjd, ' (%i of %i):' % (kk + 1, len(mjds)), len(I), 'CCDs')
if len(I) == 0:
continue
# pick one near the middle
#i = I[len(I)/2]
for i in [I[len(I) / 4], I[len(I) / 2], I[3 * len(I) / 4]]:
ccd = ccds[i]
key = dict(expnum=ccd.expnum, ccdname=ccd.ccdname)
oldvals = key
vals = cache.find(key)
print('Got', vals.count(), 'cache hits for', key)
gotone = False
for val in vals:
#if 'median_adu' in val:
if 'mode_adu' in val:
print('cache hit:', val)
allvals.append(val)
medmjds.append(mjd)
gotone = True
break
else:
print('partial cache hit:', val)
oldvals = val
###!
cache.delete_one(val)
if gotone:
continue
print('args: key', key, 'oldvals', oldvals)
args.append((mjd, key, oldvals, ccd))
# plt.clf()
# plt.hist(tim.getImage().ravel(), range=(-0.1, 0.1), bins=50,
# histtype='step', color='b')
# plt.axvline(med, color='b', alpha=0.3, lw=2)
# plt.axvline(0., color='k', alpha=0.3, lw=2)
# plt.xlabel('Sky-subtracted pixel values')
# plt.title('Date ' + ccd.date_obs + ': ' + str(im))
# ps.savefig()
if len(args):
meds = mp.map(read_sky_val, args)
print('Medians:', meds)
for (mjd, key, oldvals, ccd), val in zip(args, meds):
if val is None:
continue
allvals.append(val)
medmjds.append(mjd)
medians = []
median_adus = []
mode_adus = []
skyadus = []
keepmjds = []
for mjd, val in zip(medmjds, allvals):
madu = val['median_adu']
if madu is None:
continue
if 'median' in val:
medians.append(val['median'])
else:
from tractor.brightness import NanoMaggies
zpscale = NanoMaggies.zeropointToScale(val['ccdzpt'])
med = (val['median_adu'] - val['skyadu']) / zpscale
print('Computed median diff:', med, 'nmgy')
medians.append(med)
keepmjds.append(mjd)
median_adus.append(val['median_adu'])
mode_adus.append(val['mode_adu'])
skyadus.append(val['skyadu'])
medmjds = keepmjds
median_adus = np.array(median_adus)
mode_adus = np.array(mode_adus)
skyadus = np.array(skyadus)
medmjds = np.array(medmjds)
medians = np.array(medians)
plt.clf()
plt.plot(medmjds, median_adus - skyadus, 'b.')
plt.xlabel('MJD')
#plt.ylabel('Image median after sky subtraction (nmgy)')
plt.ylabel('Image median - SKYADU (ADU)')
#plt.ylim(-0.03, 0.03)
#plt.ylim(-10, 10)
plt.ylim(-1, 1)
plt.axhline(0, color='k', lw=2, alpha=0.5)
ps.savefig()
plt.clf()
plt.plot(medmjds, mode_adus - skyadus, 'b.')
plt.xlabel('MJD')
plt.ylabel('Image mode - SKYADU (ADU)')
plt.ylim(-1, 1)
plt.axhline(0, color='k', lw=2, alpha=0.5)
ps.savefig()
print('Median pcts:', np.percentile(medians, [0, 2, 50, 98, 100]))
mlo, mhi = np.percentile(medians, [2, 98])
I = np.flatnonzero((medians > mlo) * (medians < mhi))
d0 = np.mean(medmjds)
dmjd = medmjds[I] - d0
A = np.zeros((len(dmjd), 2))
A[:, 0] = 1.
A[:, 1] = dmjd
b = np.linalg.lstsq(A, medians[I])[0]
print('Lstsq:', b)
offset = b[0]
slope = b[1]
xx = np.array([dmjd.min(), dmjd.max()])
print('Offset', offset)
print('Slope', slope)
plt.clf()
plt.plot(medmjds, medians, 'b.')
ax = plt.axis()
plt.plot(xx + d0, offset + slope * xx, 'b-')
plt.axis(ax)
plt.xlabel('MJD')
plt.ylabel('Image median after sky subtraction (nmgy)')
plt.ylim(-0.03, 0.03)
plt.axhline(0, color='k', lw=2, alpha=0.5)
ps.savefig()
plt.clf()
plt.plot(median_adus, skyadus, 'b.')
ax = plt.axis()
lo = min(ax[0], ax[2])
hi = max(ax[1], ax[3])
plt.plot([lo, hi], [lo, hi], 'k-', alpha=0.5)
plt.xlabel('Median image ADU')
plt.ylabel('SKYADU')
plt.axis(ax)
ps.savefig()
plt.clf()
plt.plot(medmjds, skyadus / median_adus, 'b.')
plt.xlabel('MJD')
plt.ylim(0.98, 1.02)
plt.axhline(1.0, color='k', alpha=0.5)
plt.ylabel('SKYADU / median image ADU')
ps.savefig()
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None,
use_ceres=True, ceres_block=8,
save_fits=False, get_models=False, ps=None,
psf_broadening=None,
pixelized_psf=False,
get_masks=None,
move_crpix=False,
modelsky_dir=None):
'''
Given a list of tractor sources *cat*
and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
runs forced photometry, returning a FITS table the same length as *cat*.
*get_masks*: the WCS to resample mask bits into.
'''
from tractor import NanoMaggies, PointSource, Tractor, ExpGalaxy, DevGalaxy, FixedCompositeGalaxy
if not pixelized_psf and psf_broadening is None:
# PSF broadening in post-reactivation data, by band.
# Newer version from Aaron's email to decam-chatter, 2018-06-14.
broadening = { 1: 1.0405, 2: 1.0346, 3: None, 4: None }
psf_broadening = broadening[band]
if False:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('wise-forced-w%i' % band)
plots = (ps is not None)
if plots:
import pylab as plt
wantims = (plots or save_fits or get_models)
wanyband = 'w'
if get_models:
models = {}
wband = 'w%i' % band
fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix',
'pronexp']
Nsrcs = len(cat)
phot = fits_table()
# Filled in based on unique tile overlap
phot.wise_coadd_id = np.array([' '] * Nsrcs)
phot.set(wband + '_psfdepth', np.zeros(len(phot), np.float32))
ra = np.array([src.getPosition().ra for src in cat])
dec = np.array([src.getPosition().dec for src in cat])
nexp = np.zeros(Nsrcs, np.int16)
mjd = np.zeros(Nsrcs, np.float64)
central_flux = np.zeros(Nsrcs, np.float32)
fitstats = {}
tims = []
if get_masks:
mh,mw = get_masks.shape
maskmap = np.zeros((mh,mw), np.uint32)
for tile in tiles:
print('Reading WISE tile', tile.coadd_id, 'band', band)
tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band,
bandname=wanyband, roiradecbox=roiradecbox)
if tim is None:
print('Actually, no overlap with tile', tile.coadd_id)
continue
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data' % tag)
ps.savefig()
plt.clf()
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
range=(-5,10), bins=100)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.title(tag)
ps.savefig()
if move_crpix and band in [1, 2]:
realwcs = tim.wcs.wcs
x,y = realwcs.crpix
tile_crpix = tile.get('crpix_w%i' % band)
dx = tile_crpix[0] - 1024.5
dy = tile_crpix[1] - 1024.5
realwcs.set_crpix(x+dx, y+dy)
#print('CRPIX', x,y, 'shift by', dx,dy, 'to', realwcs.crpix)
if modelsky_dir and band in [1, 2]:
fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band))
if not os.path.exists(fn):
raise RuntimeError('WARNING: does not exist:', fn)
x0,x1,y0,y1 = tim.roi
bg = fitsio.FITS(fn)[2][y0:y1, x0:x1]
#print('Read background map:', bg.shape, bg.dtype, 'vs image', tim.shape)
if plots:
plt.clf()
plt.subplot(1,2,1)
plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=5 * sig1)
plt.subplot(1,2,2)
plt.imshow(bg, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=5 * sig1)
tag = '%s W%i' % (tile.coadd_id, band)
plt.suptitle(tag)
ps.savefig()
plt.clf()
ha = dict(range=(-5,10), bins=100, histtype='step')
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
color='b', label='Original', **ha)
plt.hist(((tim.getImage()-bg) * tim.inverr)[tim.inverr > 0].ravel(),
color='g', label='Minus Background', **ha)
plt.axvline(0, color='k', alpha=0.5)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.legend()
plt.title(tag + ': background')
ps.savefig()
# Actually subtract the background!
tim.data -= bg
# Floor the per-pixel variances
if band in [1,2]:
# in Vega nanomaggies per pixel
floor_sigma = {1: 0.5, 2: 2.0}
with np.errstate(divide='ignore'):
new_ie = 1. / np.hypot(1./tim.inverr, floor_sigma[band])
new_ie[tim.inverr == 0] = 0.
if plots:
plt.clf()
plt.plot((1. / tim.inverr[tim.inverr>0]).ravel(), (1./new_ie[tim.inverr>0]).ravel(), 'b.')
plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band))
plt.xlabel('original')
plt.ylabel('floored')
ps.savefig()
tim.inverr = new_ie
# Read mask file?
if get_masks:
from astrometry.util.resample import resample_with_wcs, OverlapError
# unwise_dir can be a colon-separated list of paths
tilemask = None
for d in tile.unwise_dir.split(':'):
fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id,
'unwise-%s-msk.fits.gz' % tile.coadd_id)
if os.path.exists(fn):
print('Reading unWISE mask file', fn)
x0,x1,y0,y1 = tim.roi
tilemask = fitsio.FITS(fn)[0][y0:y1,x0:x1]
break
if tilemask is None:
print('unWISE mask file for tile', tile.coadd_id, 'does not exist')
else:
try:
tanwcs = tim.wcs.wcs
assert(tanwcs.shape == tilemask.shape)
Yo,Xo,Yi,Xi,_ = resample_with_wcs(get_masks, tanwcs, intType=np.int16)
# Only deal with mask pixels that are set.
I, = np.nonzero(tilemask[Yi,Xi] > 0)
# Trim to unique area for this tile
rr,dd = get_masks.pixelxy2radec(Yo[I]+1, Xo[I]+1)
good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
I = I[good]
maskmap[Yo[I],Xo[I]] = tilemask[Yi[I], Xi[I]]
except OverlapError:
# Shouldn't happen by this point
print('No overlap between WISE tile', tile.coadd_id, 'and brick')
# The tiles have some overlap, so zero out pixels outside the
# tile's unique area.
th,tw = tim.shape
xx,yy = np.meshgrid(np.arange(tw), np.arange(th))
rr,dd = tim.wcs.wcs.pixelxy2radec(xx+1, yy+1)
unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
#print(np.sum(unique), 'of', (th*tw), 'pixels in this tile are unique')
tim.inverr[unique == False] = 0.
del xx,yy,rr,dd,unique
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage() * (tim.inverr > 0),
interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data (unique)' % tag)
ps.savefig()
if pixelized_psf:
import unwise_psf
if (band == 1) or (band == 2):
# we only have updated PSFs for W1 and W2
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id,
modelname='neo4_unwisecat')
else:
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id)
if band == 4:
# oversample (the unwise_psf models are at native W4 5.5"/pix,
# while the unWISE coadds are made at 2.75"/pix.
ph,pw = psfimg.shape
subpsf = np.zeros((ph*2-1, pw*2-1), np.float32)
from astrometry.util.util import lanczos3_interpolate
xx,yy = np.meshgrid(np.arange(0., pw-0.51, 0.5, dtype=np.float32),
np.arange(0., ph-0.51, 0.5, dtype=np.float32))
xx = xx.ravel()
yy = yy.ravel()
ix = xx.astype(np.int32)
iy = yy.astype(np.int32)
dx = (xx - ix).astype(np.float32)
dy = (yy - iy).astype(np.float32)
psfimg = psfimg.astype(np.float32)
rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg])
if plots:
plt.clf()
plt.imshow(psfimg, interpolation='nearest', origin='lower')
plt.title('Original PSF model')
ps.savefig()
plt.clf()
plt.imshow(subpsf, interpolation='nearest', origin='lower')
plt.title('Subsampled PSF model')
ps.savefig()
psfimg = subpsf
del xx, yy, ix, iy, dx, dy
from tractor.psf import PixelizedPSF
psfimg /= psfimg.sum()
fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0}
psfimg *= fluxrescales[band]
tim.psf = PixelizedPSF(psfimg)
if psf_broadening is not None and not pixelized_psf:
# psf_broadening is a factor by which the PSF FWHMs
# should be scaled; the PSF is a little wider
# post-reactivation.
psf = tim.getPsf()
from tractor import GaussianMixturePSF
if isinstance(psf, GaussianMixturePSF):
#
print('Broadening PSF: from', psf)
p0 = psf.getParams()
pnames = psf.getParamNames()
p1 = [p * psf_broadening**2 if 'var' in name else p
for (p, name) in zip(p0, pnames)]
psf.setParams(p1)
print('Broadened PSF:', psf)
else:
print('WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf))
wcs = tim.wcs.wcs
ok,x,y = wcs.radec2pixelxy(ra, dec)
x = np.round(x - 1.).astype(int)
y = np.round(y - 1.).astype(int)
good = (x >= 0) * (x < tw) * (y >= 0) * (y < th)
# Which sources are in this brick's unique area?
usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
I, = np.nonzero(good * usrc)
nexp[I] = tim.nuims[y[I], x[I]]
if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
mjd[I] = (tim.mjdmin + tim.mjdmax) / 2.
phot.wise_coadd_id[I] = tile.coadd_id
central_flux[I] = tim.getImage()[y[I], x[I]]
del x,y,good,usrc
# PSF norm for depth
psf = tim.getPsf()
h,w = tim.shape
patch = psf.getPointSourcePatch(h//2, w//2).patch
psfnorm = np.sqrt(np.sum(patch**2))
# To handle zero-depth, we return 1/nanomaggies^2 units rather than mags.
psfdepth = 1. / (tim.sig1 / psfnorm)**2
phot.get(wband + '_psfdepth')[I] = psfdepth
tim.tile = tile
tims.append(tim)
if plots:
plt.clf()
mn,mx = 0.1, 20000
plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100,
range=(np.log10(mn), np.log10(mx)))
logt = np.arange(0, 5)
plt.xticks(logt, ['%i' % i for i in 10.**logt])
plt.title('Central fluxes (W%i)' % band)
plt.axvline(np.log10(20000), color='k')
plt.axvline(np.log10(1000), color='k')
ps.savefig()
# Eddie's non-secret recipe:
#- central pixel <= 1000: 19x19 pix box size
#- central pixel in 1000 - 20000: 59x59 box size
#- central pixel > 20000 or saturated: 149x149 box size
#- object near "bright star": 299x299 box size
nbig = nmedium = nsmall = 0
for src,cflux in zip(cat, central_flux):
if cflux > 20000:
R = 100
nbig += 1
elif cflux > 1000:
R = 30
nmedium += 1
else:
R = 15
nsmall += 1
if isinstance(src, PointSource):
src.fixedRadius = R
else:
### FIXME -- sizes for galaxies..... can we set PSF size separately?
galrad = 0
# RexGalaxy is a subclass of ExpGalaxy
if isinstance(src, (ExpGalaxy, DevGalaxy)):
galrad = src.shape.re
elif isinstance(src, FixedCompositeGalaxy):
galrad = max(src.shapeExp.re, src.shapeDev.re)
pixscale = 2.75
src.halfsize = int(np.hypot(R, galrad * 5 / pixscale))
#print('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small')
minsb = 0.
fitsky = False
tractor = Tractor(tims, cat)
if use_ceres:
from tractor.ceres_optimizer import CeresOptimizer
tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block)
tractor.freezeParamsRecursive('*')
tractor.thawPathsTo(wanyband)
kwa = dict(fitstat_extras=[('pronexp', [tim.nims for tim in tims])])
t0 = Time()
R = tractor.optimize_forced_photometry(
minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True,
variance=True, shared_params=False,
wantims=wantims, **kwa)
print('unWISE forced photometry took', Time() - t0)
if use_ceres:
term = R.ceres_status['termination']
# Running out of memory can cause failure to converge
# and term status = 2.
# Fail completely in this case.
if term != 0:
print('Ceres termination status:', term)
raise RuntimeError(
'Ceres terminated with status %i' % term)
if wantims:
ims1 = R.ims1
flux_invvars = R.IV
if R.fitstats is not None:
for k in fskeys:
x = getattr(R.fitstats, k)
fitstats[k] = np.array(x).astype(np.float32)
if save_fits:
for i,tim in enumerate(tims):
tile = tim.tile
(dat, mod, ie, chi, roi) = ims1[i]
wcshdr = fitsio.FITSHDR()
tim.wcs.wcs.add_to_header(wcshdr)
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' %
tag, dat, clobber=True, header=wcshdr)
fitsio.write('%s-mod.fits' % tag, mod,
clobber=True, header=wcshdr)
fitsio.write('%s-chi.fits' % tag, chi,
clobber=True, header=wcshdr)
if plots:
# Create models for just the brightest sources
bright_cat = [src for src in cat
if src.getBrightness().getBand(wanyband) > 1000]
print('Bright soures:', len(bright_cat))
btr = Tractor(tims, bright_cat)
for tim in tims:
mod = btr.getModelImage(tim)
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
sig1 = tim.sig1
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: bright-star models' % tag)
ps.savefig()
if get_models:
for i,tim in enumerate(tims):
tile = tim.tile
(dat, mod, ie, chi, roi) = ims1[i]
models[(tile.coadd_id, band)] = (mod, dat, ie, tim.roi, tim.wcs.wcs)
if plots:
for i,tim in enumerate(tims):
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
(dat, mod, ie, chi, roi) = ims1[i]
sig1 = tim.sig1
plt.clf()
plt.imshow(dat, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi, interpolation='nearest', origin='lower',
cmap='gray', vmin=-5, vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = flux_invvars
# Sources out of bounds, eg, never change from their default
# (1-sigma or whatever) initial fluxes. Zero them out instead.
nm[nm_ivar == 0] = 0.
phot.set(wband + '_nanomaggies', nm.astype(np.float32))
phot.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32))
dnm = np.zeros(len(nm_ivar), np.float32)
okiv = (nm_ivar > 0)
dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32)
okflux = (nm > 0)
mag = np.zeros(len(nm), np.float32)
mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])
).astype(np.float32)
dmag = np.zeros(len(nm), np.float32)
ok = (okiv * okflux)
dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok])
).astype(np.float32)
mag[np.logical_not(okflux)] = np.nan
dmag[np.logical_not(ok)] = np.nan
phot.set(wband + '_mag', mag)
phot.set(wband + '_mag_err', dmag)
for k in fskeys:
phot.set(wband + '_' + k, fitstats[k])
phot.set(wband + '_nexp', nexp)
if not np.all(mjd == 0):
phot.set(wband + '_mjd', mjd)
rtn = wphotduck()
rtn.phot = phot
rtn.models = None
rtn.maskmap = None
if get_models:
rtn.models = models
if get_masks:
rtn.maskmap = maskmap
return rtn
ps.savefig()
imx = np.argmax(m22)
diff = np.diff(m22[:imx + 1])
# Assert monotonic up to the peak (from the left)
# (low-level jitter/wrap-around is allowed)
self.assertTrue(np.all(np.logical_or(np.abs(diff) < 1e-9, diff > 0)))
diff = np.diff(m22[imx:])
# Assert monotonic decreasing after to the peak
# (low-level jitter/wrap-around is allowed)
self.assertTrue(np.all(np.logical_or(np.abs(diff) < 1e-9, diff < 0)))
# Assert that wrap-around exists for PixelizedPsf model
diff = np.diff(m21[:imx + 1])
self.assertFalse(np.all(np.logical_or(np.abs(diff) < 1e-9, diff > 0)))
diff = np.diff(m21[imx:])
self.assertFalse(np.all(np.logical_or(np.abs(diff) < 1e-9, diff < 0)))
if __name__ == '__main__':
import sys
if '--plots' in sys.argv:
sys.argv.remove('--plots')
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('gal')
unittest.main()
def main(passnum, threads):
global udecs
global P3
global T
global tilewcs
global exps
global bad_expids
global tileid_to_depth
ps = PlotSequence('covfill-p%i' % passnum)
retirablefn = 'retirable-p%i.fits' % passnum
depthsfn = 'all-depths-p%i.fits' % passnum
if os.path.exists(retirablefn):
R = fits_table(retirablefn)
pcts = np.arange(0, 101)
target = 22.5
req_pcts = [0, 2, 2, 5, 5, 10, 10, 100]
req_depths = [0, 0, target-0.6, target-0.6,
target-0.3, target-0.3, target, target]
maglo, maghi = 21,23
plt.clf()
for depths in R.depths:
plt.plot(pcts, np.clip(depths, maglo, maghi), 'b-',
alpha=0.1)
plt.plot(req_pcts, np.clip(req_depths, maglo, maghi), 'k-',
lw=2, alpha=0.5)
plt.ylim(maglo, maghi)
plt.xlim(0, 100)
plt.xlabel('Coverage fraction')
plt.ylabel('Existing depth')
plt.suptitle('MzLS: retirable pass-%i tiles: %i' % (passnum,len(R)))
ps.savefig()
# Where are they on the sky?
T = fits_table('obstatus/mosaic-tiles_obstatus.fits')
T.cut(T.in_desi == 1)
T.cut(T.get('pass') <= 3)
tileid_to_index = np.zeros(T.tileid.max()+1, int)
tileid_to_index[T.tileid] = np.arange(len(T))
R.ra = T.ra [tileid_to_index[R.tileid]]
R.dec = T.dec[tileid_to_index[R.tileid]]
plt.clf()
plt.plot(T.ra, T.dec, 'k.', alpha=0.02)
I = (T.z_done == 1)
plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.1)
plt.plot(R.ra, R.dec, 'b.')
ax = [310,80,30,85]
#xl,xh = plt.xlim()
#plt.xlim(xh,xl)
plt.xlabel('RA (deg)')
plt.ylabel('Dec (deg)')
plt.title('MzLS: retirable pass-%i tiles' % passnum)
plt.axis(ax)
ps.savefig()
for p in [1,2,3]:
plt.clf()
plt.plot(T.ra, T.dec, 'k.', alpha=0.02)
#plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.1)
I = np.flatnonzero((T.get('pass') == p) * (T.z_done == 1)
* (T.z_depth > 1) * (T.z_depth < 30))
plt.scatter(T.ra[I], T.dec[I], c=T.z_depth[I],
vmin=20, vmax=23, s=4)
I = np.flatnonzero((T.get('pass') == p) * (T.z_done == 1)
* (T.z_depth == 30))
plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.5)
plt.colorbar()
plt.title('MzLS: Finished tiles in pass %i' % p)
plt.axis(ax)
ps.savefig()
sys.exit(0)
# NERSC: export LEGACY_SURVEY_DIR=/global/cscratch1/sd/dstn/dr4plus
# (dstn laptop: export LEGACY_SURVEY_DIR=~/legacypipe-dir-mzls/)
survey = LegacySurveyData()
ccds = survey.get_annotated_ccds()
print('Annotated CCDs:', len(ccds))
ccds.cut(ccds.camera == 'mosaic')
print(len(ccds), 'Mosaic')
print('Unique exposures:', len(np.unique(ccds.expnum)))
ccds.cut(ccds.exptime > 60)
print('Exptime > 60 sec:', len(ccds))
nccds = Counter(ccds.expnum)
for k,v in nccds.most_common():
if v <= 4:
break
print('Expnum', k, 'appears', v, 'times')
print('Tile pass numbers:', Counter(ccds.tilepass).most_common())
# Fix parsing of OBJECT field to tileid...
from obsbot import get_tile_id_from_name
tileids = []
for o in ccds.object:
tid = get_tile_id_from_name(o.strip())
if tid is None:
tid = 0
tileids.append(tid)
tileids = np.array(tileids)
print(len(np.unique(tileids)), 'unique tile ids in annotated file, from OBJECT')
print(len(np.unique(ccds.tileid)), 'unique tile ids in ann file from TILEID')
D = np.flatnonzero(tileids != ccds.tileid)
print(len(D), 'different tileids')
print('From OBJECT:', tileids[D])
print('From TILEID:', ccds.tileid[D])
ccds.tileid = tileids
T = fits_table('obstatus/mosaic-tiles_obstatus.fits')
f = open('obstatus/bad_expid.txt')
bad_expids = []
for line in f:
line = line.strip()
if len(line) == 0:
continue
if line[0] == '#':
continue
words = line.split()
try:
expnum = int(words[0])
except:
print('Skipping line:', line)
continue
bad_expids.append(expnum)
print('Read', len(bad_expids), 'bad exposure numbers')
# Update ccds.tilepass from ccds.tileid
tileidtopass = dict(zip(T.tileid, T.get('pass')))
tileidtoebv = dict(zip(T.tileid, T.ebv_med))
ccds.tilepass = np.array([tileidtopass.get(tileid, 0)
for tileid in ccds.tileid])
ccds.tileebv = np.array([tileidtoebv.get(tileid, 0)
for tileid in ccds.tileid])
print('Tile pass numbers after update:', Counter(ccds.tilepass).most_common())
e,I = np.unique(ccds.expnum, return_index=True)
exps = ccds[I]
#print('Object names,exptimes for tilepass==0:', zip(exps.object[exps.tilepass == 0], exps.exptime[exps.tilepass == 0]))
# Average the depth per exposure
for j,expnum in enumerate(exps.expnum):
I = np.flatnonzero(ccds.expnum == expnum)
if len(I) != 4:
print('Exposure', expnum, 'has', len(I), 'CCD entries')
continue
# Don't include zeros in computing average depths!
Igood = I[(ccds.galdepth[I] > 0) * (ccds.ccdzpt[I] < 30)]
if len(Igood) > 0:
exps.galdepth[j] = np.mean(ccds.galdepth[Igood])
else:
exps.galdepth[j] = 0.
# CCDs-table-based mapping from tileid to depth.
I = np.flatnonzero((exps.tilepass > 0) * (exps.galdepth > 0) *
(exps.tileid > 0))
tileid_to_depth = dict(zip(exps.tileid[I], exps.galdepth[I]))
T.cut(T.in_desi == 1)
T.cut(T.get('pass') <= 3)
# The tiles we'll examine
P3 = T[T.get('pass') == passnum]
print(len(P3), 'pass', passnum, 'and in DESI')
todo = P3[P3.z_done == 0]
print(len(todo), 'pass', passnum, 'tiles to do (Z_DONE=0)')
# Tiles with measured depths
T.cut((T.z_depth > 15) * (T.z_depth < 30))
print(len(T), 'tiles with measured depths')
# Passes other than 3... they ~ only barely overlap, so don't
# contribute significant depth.
#T.cut(T.get('pass') < 3)
udecs = np.unique(P3.dec)
print(len(udecs), 'unique Dec values in pass', passnum)
# Grab an arbitrary weight-map image and use that as a proxy!
wtfn = 'k4m_170501_112501_oow_zd_v1.fits.fz'
F = fitsio.FITS(wtfn)
tilewcs = []
# Read the WCS headers for each chip.
# They make the CRVAL be the boresight for all 4 chips... perfect!
# (because this means we can just set CRVAL = RA,Dec to shift the WCSes)
for i in range(1, len(F)):
hdr = F[i].read_header()
wcs = wcs_pv2sip_hdr(hdr)
tilewcs.append(wcs)
mp = multiproc(threads)
#args = [(i,t,udecs,P3,T,tilewcs,exps,bad_expids,tileid_to_depth)
args = [(i,t)
for i,t in enumerate(todo)]
thedepths = mp.map(one_tile, args)
alldepths = []
retirable = []
for arg,depths in zip(args, thedepths):
if depths is None:
continue
itile,tile = arg[:2]
target = 22.5
req_pcts = [0, 2, 2, 5, 5, 10, 10, 100]
req_depths = [0, 0, target-0.6, target-0.6,
target-0.3, target-0.3, target, target]
print(' Depths at 2, 5, and 10th percentile vs target:',
'%.2f' % (depths[2] - (target - 0.6)),
'%.2f' % (depths[5] - (target - 0.3)),
'%.2f' % (depths[10] - target))
alldepths.append((tile.tileid, depths))
if not ((depths[2] > target - 0.6) and
(depths[5] > target - 0.3) and
(depths[10] > target)):
continue
retirable.append((tile.tileid, depths))
#if len(retirable) == 10:
# break
# if ps.ploti >= 100:
# continue
#
# maglo, maghi = 21,23
#
# plt.clf()
# #plt.subplot(1,2,1)
# plt.subplot2grid((2,2), (0,0))
# plt.imshow(depth, interpolation='nearest', origin='lower',
# vmin=maglo, vmax=maghi)
# plt.colorbar(ticks=[np.arange(maglo, maghi+0.01, 0.5)])
# plt.xticks([]); plt.yticks([])
# #plt.subplot(1,2,2)
# plt.subplot2grid((2,2), (1,0))
# plt.imshow(nexp, interpolation='nearest', origin='lower',
# vmin=0, vmax=4)
# plt.colorbar(ticks=[0,1,2,3,4])
# plt.xticks([]); plt.yticks([])
#
# ax = plt.subplot2grid((2,2), (0,1), rowspan=2)
# plt.plot(req_pcts, np.clip(req_depths, maglo, maghi), 'k-',
# lw=2, alpha=0.5)
# plt.plot(pcts, np.clip(depths, maglo, maghi), 'b-')
# ax.yaxis.tick_right()
# plt.ylim(maglo, maghi)
# plt.xlim(0, 100)
# plt.xlabel('Coverage fraction')
# plt.ylabel('Existing depth')
# plt.suptitle('Tile %i' % tile.tileid)
# ps.savefig()
#if ps.ploti == 100:
# break
# print('Tiles that could be retired:')
# print('# Tileid 0th-percentile-extcorr-depth 1st-pctile 2nd-pctile ...')
# for tileid, depths in retirable:
# print(tileid, ' '.join(['%.3f' % d for d in depths]))
R = fits_table()
R.tileid = np.array ([t for t,d in alldepths])
R.depths = np.vstack([d for t,d in alldepths])
R.writeto(depthsfn)
if len(retirable):
R = fits_table()
R.tileid = np.array([t for t,d in retirable])
R.depths = np.vstack([d for t,d in retirable])
R.writeto(retirablefn)
else:
print('No tiles in pass', passnum, 'are retirable')
def main():
ps = PlotSequence('cov')
survey = LegacySurveyData()
ra, dec = 242.0, 10.2
fn = 'coverage-ccds.fits'
if not os.path.exists(fn):
ccds = survey.get_ccds()
ccds.cut(ccds.filter == 'r')
ccds.cut(ccds.propid == '2014B-0404')
ccds.cut(np.hypot(ccds.ra_bore - ra, ccds.dec_bore - dec) < 2.5)
print(np.unique(ccds.expnum), 'unique exposures')
print('propids', np.unique(ccds.propid))
ccds.writeto(fn)
else:
ccds = fits_table(fn)
plt.clf()
for e in np.unique(ccds.expnum):
I = np.flatnonzero(ccds.expnum == e)
plt.plot(ccds.ra[I], ccds.dec[I], '.')
ps.savefig()
degw = 3.0
pixscale = 10.
W = degw * 3600 / 10.
H = W
hi = 6
cmap = cmap_discretize('jet', hi + 1)
wcs = Tan(ra, dec, W / 2. + 0.5, H / 2. + 0.5, -pixscale / 3600., 0., 0.,
pixscale / 3600., float(W), float(H))
r0, d0 = wcs.pixelxy2radec(1, 1)
r1, d1 = wcs.pixelxy2radec(W, H)
extent = [min(r0, r1), max(r0, r1), min(d0, d1), max(d0, d1)]
for expnums in [
[348666],
[348666, 348710, 348686],
[348659, 348667, 348658, 348666, 348665, 348669, 348668],
None,
[
348683, 348687, 347333, 348686, 348685, 348692, 348694, 348659,
348667, 348658, 348666, 348665, 348669, 348668, 348707, 348709,
348708, 348710, 348711, 348716, 348717
],
]:
nexp = np.zeros((H, W), np.uint8)
for ccd in ccds:
if expnums is not None and not ccd.expnum in expnums:
continue
ccdwcs = survey.get_approx_wcs(ccd)
r, d = ccdwcs.pixelxy2radec(1, 1)
ok, x0, y0 = wcs.radec2pixelxy(r, d)
r, d = ccdwcs.pixelxy2radec(ccd.width, ccd.height)
ok, x1, y1 = wcs.radec2pixelxy(r, d)
xlo = np.clip(int(np.round(min(x0, x1))) - 1, 0, W - 1)
xhi = np.clip(int(np.round(max(x0, x1))) - 1, 0, W - 1)
ylo = np.clip(int(np.round(min(y0, y1))) - 1, 0, H - 1)
yhi = np.clip(int(np.round(max(y0, y1))) - 1, 0, H - 1)
nexp[ylo:yhi + 1, xlo:xhi + 1] += 1
plt.clf()
plt.imshow(nexp,
interpolation='nearest',
origin='lower',
vmin=-0.5,
vmax=hi + 0.5,
cmap=cmap,
extent=extent)
plt.colorbar(ticks=np.arange(hi + 1))
ps.savefig()
O = fits_table('obstatus/decam-tiles_obstatus.fits')
O.cut(np.hypot(O.ra - ra, O.dec - dec) < 2.5)
for p in [1, 2, 3]:
print('Pass', p, 'exposures:', O.r_expnum[O.get('pass') == p])
O.cut(O.get('pass') == 2)
print(len(O), 'pass 2 nearby')
d = np.hypot(O.ra - ra, O.dec - dec)
print('Dists:', d)
I = np.flatnonzero(d < 0.5)
assert (len(I) == 1)
ocenter = O[I[0]]
print('Center expnum', ocenter.r_expnum)
I = np.flatnonzero(d >= 0.5)
O.cut(I)
#center = ccds[ccds.expnum == ocenter.r_expnum]
#p2 = ccds[ccds.
ok, xc, yc = wcs.radec2pixelxy(ocenter.ra, ocenter.dec)
xx, yy = np.meshgrid(np.arange(W) + 1, np.arange(H) + 1)
c_d2 = (xc - xx)**2 + (yc - yy)**2
best = np.ones((H, W), bool)
for o in O:
ok, x, y = wcs.radec2pixelxy(o.ra, o.dec)
d2 = (x - xx)**2 + (y - yy)**2
best[d2 < c_d2] = False
del d2
del c_d2, xx, yy
# plt.clf()
# plt.imshow(best, interpolation='nearest', origin='lower', cmap='gray',
# vmin=0, vmax=1)
# ps.savefig()
plt.clf()
plt.imshow(nexp * best,
interpolation='nearest',
origin='lower',
vmin=-0.5,
vmax=hi + 0.5,
cmap=cmap,
extent=extent)
plt.colorbar(ticks=np.arange(hi + 1))
ps.savefig()
plt.clf()
n, b, p = plt.hist(np.clip(nexp[best], 0, hi),
range=(-0.5, hi + 0.5),
bins=hi + 1)
plt.xlim(-0.5, hi + 0.5)
ps.savefig()
print('b', b)
print('n', n)
print('fracs', np.array(n) / np.sum(n))
print('pcts',
', '.join(['%.1f' % f for f in 100. * np.array(n) / np.sum(n)]))
def main(decals=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for decals.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if decals is None:
decals = Decals()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from decals-ccds.fits table
T = decals.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(np.array([f.strip() == opt.filename for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert(len(T) == 1)
im = decals.get_image_object(T[0])
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True)
print('Got tim:', tim)
if opt.catfn in ['DR1', 'DR2']:
if opt.catalog_path is None:
opt.catalog_path = opt.catfn.lower()
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, decals=decals)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3],
'tractor-%s.fits' % b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
TT.append(T)
T = merge_tables(TT)
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T
T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T
cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE)
# print('Got cat:', cat)
print('Forced photom...')
opti = None
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T))
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap
ap = 1./(np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap
if opt.forced:
kwa = {}
if opt.plots is None:
kwa.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True, fitstats=True,
shared_params=False, **kwa)
if opt.plots:
(data,mod,ie,chi,roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([src.getBrightness().getFlux(tim.band)
for src in cat]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2 .astype(np.float32)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, decals.decals_dir)
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
fname = os.path.join(d2, d1, fname)
print('Trimmed filename to', fname)
#version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file'))
version_hdr.add_record(dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext'))
version_hdr.add_record(dict(name='CAMERA', value='DECam', comment='Dark Energy Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {'mjd':'sec', 'exptime':'sec', 'flux':'nanomaggy',
'flux_ivar':'1/nanomaggy^2'}
columns = F.get_columns()
for i,col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
print('Finished forced phot:', Time()-t0)
return 0
def main(survey=None, opt=None):
print(' '.join(sys.argv))
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = tlast = Time()
if opt.skip and os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing first arg as exposure number (otherwise, it's a filename)
try:
expnum = int(opt.expnum)
filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
filename = opt.expnum
# Try parsing HDU: "all" or HDU name or HDU number.
all_hdus = (opt.ccdname == 'all')
hdu = -1
ccdname = None
if not all_hdus:
try:
hdu = int(opt.ccdname)
except:
ccdname = opt.ccdname
if survey is None:
survey = LegacySurveyData(survey_dir=opt.survey_dir)
catsurvey_north = survey
catsurvey_south = None
if opt.catalog_dir_north is not None:
assert (opt.catalog_dir_south is not None)
assert (opt.catalog_resolve_dec_ngc is not None)
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
if opt.catalog_dir is not None:
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)
if filename is not None and hdu >= 0:
# FIXME -- try looking up in CCDs file?
# Read metadata from file
print('Warning: faking metadata from file contents')
T = exposure_metadata([filename], hdus=[hdu])
print('Metadata:')
T.about()
if not 'ccdzpt' in T.columns():
phdr = fitsio.read_header(filename)
T.ccdzpt = np.array([phdr['MAGZERO']])
print('WARNING: using header MAGZERO')
T.ccdraoff = np.array([0.])
T.ccddecoff = np.array([0.])
print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and ccdname', ccdname)
if hdu >= 0:
T.cut(T.image_hdu == hdu)
print(len(T), 'with HDU', hdu)
if filename is not None:
T.cut(np.array([f.strip() == filename for f in T.image_filename]))
print(len(T), 'with filename', filename)
if opt.camera is not None:
T.cut(T.camera == opt.camera)
print(len(T), 'with camera', opt.camera)
if not all_hdus:
assert (len(T) == 1)
args = []
for ccd in T:
args.append((survey, catsurvey_north, catsurvey_south,
opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))
if opt.threads:
from astrometry.util.multiproc import multiproc
from astrometry.util.timingpool import TimingPool, TimingPoolMeas
pool = TimingPool(opt.threads)
poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(poolmeas)
mp = multiproc(None, pool=pool)
tm = Time()
FF = mp.map(bounce_one_ccd, args)
print('Multi-processing forced-phot:', Time() - tm)
else:
FF = map(bounce_one_ccd, args)
FF = [F for F in FF if F is not None]
if len(FF) == 0:
print('No photometry results to write.')
return 0
# Keep only the first header
_, version_hdr = FF[0]
FF = [F for F, hdr in FF]
F = merge_tables(FF)
if all_hdus:
version_hdr.delete('CPHDU')
version_hdr.delete('CCDNAME')
units = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2',
'apflux': 'nanomaggy',
'apflux_ivar': '1/nanomaggy^2',
'psfdepth': '1/nanomaggy^2',
'galdepth': '1/nanomaggy^2',
'sky': 'nanomaggy/arcsec^2',
'psfsize': 'arcsec'
}
if opt.derivs:
units.update({
'dra': 'arcsec',
'ddec': 'arcsec',
'dra_ivar': '1/arcsec^2',
'ddec_ivar': '1/arcsec^2'
})
columns = F.get_columns()
order = [
'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'ccd_cuts',
'airmass', 'sky', 'psfdepth', 'galdepth', 'ra', 'dec', 'flux',
'flux_ivar', 'fracflux', 'rchisq', 'fracmasked', 'apflux',
'apflux_ivar', 'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar',
'ddec_ivar'
]
columns = [c for c in order if c in columns]
# Set units headers (must happen after column ordering is set!)
hdr = fitsio.FITSHDR()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.outfn))
fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
F.writeto(tmpfn, header=hdr, append=True, columns=columns)
os.rename(tmpfn, opt.outfn)
print('Wrote', opt.outfn)
tnow = Time()
print('Total:', tnow - t0)
return 0
def process_image(fn, ext, nom, sfd, opt, obs, tiles):
db = opt.db
print('Reading', fn)
if sfd is None:
sfd = gSFD
# Read primary FITS header
phdr = fitsio.read_header(fn)
obstype = phdr.get('OBSTYPE','').strip()
print('obstype:', obstype)
exptime = phdr.get('EXPTIME', 0)
expnum = phdr.get('EXPNUM', 0)
filt = phdr.get('FILTER', None)
if filt is not None:
filt = filt.strip()
filt = filt.split()[0]
if filt is None:
filt = ''
airmass = phdr.get('AIRMASS', 0.)
ra = hmsstring2ra (phdr.get('RA', '0'))
dec = dmsstring2dec(phdr.get('DEC', '0'))
# Write QA plots to files named by the exposure number
print('Exposure number:', expnum)
skip = False
if obstype in ['zero', 'focus', 'dome flat', '']:
print('Skipping obstype =', obstype)
skip = True
if exptime == 0:
print('Exposure time EXPTIME in header =', exptime)
skip = True
if expnum == '':
print('No expnum in header')
skip = True
if filt == 'solid':
print('Solid (block) filter.')
skip = True
if skip and not db:
return None
if db:
import obsdb
if ext is None:
ext = get_default_extension(fn)
m,created = obsdb.MeasuredCCD.objects.get_or_create(
filename=fn, extension=ext)
m.obstype = obstype
m.camera = camera_name(phdr)
m.expnum = expnum
m.exptime = exptime
m.mjd_obs = phdr.get('MJD-OBS', 0.)
m.airmass = airmass
m.rabore = ra
m.decbore = dec
m.band = filt
m.bad_pixcnt = ('PIXCNT1' in phdr)
m.readtime = phdr.get('READTIME', 0.)
if opt.focus and obstype == 'focus' and m.camera == 'mosaic3':
from mosaic_focus import Mosaic3FocusMeas
show_plot = opt.show
if show_plot:
import pylab as plt
plt.figure(2, figsize=(8,10))
if ext is None:
ext = get_default_extension(fn)
meas = Mosaic3FocusMeas(fn, ext, nom)
focusfn = 'focus.png'
meas.run(ps=None, plotfn=focusfn)
print('Wrote', focusfn)
if show_plot:
plt.draw()
plt.show(block=False)
plt.pause(0.001)
plt.figure(1)
if skip:
m.save()
return None
if opt.doplots:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('qa-%i' % expnum)
ps.printfn = False
else:
ps = None
# Measure the new image
kwa = {}
if ext is not None:
kwa.update(ext=ext)
if opt.n_fwhm is not None:
kwa.update(n_fwhm=opt.n_fwhm)
M = measure_raw(fn, ps=ps, **kwa)
if opt.doplots:
from glob import glob
# Gather all the QAplots into a single pdf and clean them up.
qafile = 'qa-%i.pdf' % expnum
pnglist = sorted(glob('qa-%i-??.png' % expnum))
cmd = 'convert {} {}'.format(' '.join(pnglist), qafile)
print('Writing out {}'.format(qafile))
#print(cmd)
os.system(cmd)
if not opt.keep_plots:
[os.remove(png) for png in pnglist]
# (example results for testig)
#M = {'seeing': 1.4890481099577366, 'airmass': 1.34,
#'skybright': 18.383479116033314, 'transparency': 0.94488537276869045,
#'band': 'z', 'zp': 26.442847814941093}
#print('Measurements:', M)
trans = M.get('transparency', 0)
band = M['band']
# Look up E(B-V) in SFD map
ebv = sfd.ebv(ra, dec)[0]
print('E(B-V): %.3f' % ebv)
if trans > 0:
fid = nom.fiducial_exptime(band)
expfactor = exposure_factor(fid, nom,
airmass, ebv, M['seeing'], M['skybright'],
trans)
print('Exposure factor: %6.3f' % expfactor)
t_exptime = expfactor * fid.exptime
print('Target exposure time: %6.1f' % t_exptime)
t_exptime = np.clip(t_exptime, fid.exptime_min, fid.exptime_max)
print('Clipped exposure time: %6.1f' % t_exptime)
if band == 'z':
t_sat = nom.saturation_time(band, M['skybright'])
if t_exptime > t_sat:
t_exptime = t_sat
print('Reduced exposure time to avoid z-band saturation: %.1f' % t_exptime)
print
print('Actual exposure time taken: %6.1f' % exptime)
print('Depth (exposure time) factor: %6.3f' % (exptime / t_exptime))
# If you were going to re-plan, you would run with these args:
plandict = dict(seeing=M['seeing'], transparency=trans)
# Assume the sky is as much brighter than canonical in each band... unlikely
dsky = M['skybright'] - nom.sky(M['band'])
for b in 'grz':
plandict['sb'+b] = nom.sky(b) + dsky
# Note that nightlystrategy.py takes UTC dates.
start = datenow()
# Start the strategy 5 minutes from now.
start += datetime.timedelta(0, 5*60)
d = start.date()
plandict['startdate'] = '%04i-%02i-%02i' % (d.year, d.month, d.day)
t = start.time()
plandict['starttime'] = t.strftime('%H:%M:%S')
# Make an hour-long plan
end = start + datetime.timedelta(0, 3600)
d = end.date()
plandict['enddate'] = '%04i-%02i-%02i' % (d.year, d.month, d.day)
t = end.time()
plandict['endtime'] = t.strftime('%H:%M:%S')
# Set "--date" to be the UTC date at previous sunset.
# (nightlystrategy will ask for the next setting of the sun below
# -18-degrees from that date to define the sn_18). We could
# probably also get away with subtracting, like, 12 hours from
# now()...
sun = ephem.Sun()
obs.date = datenow()
# not the proper horizon, but this doesn't matter -- just need it to
# be before -18-degree twilight.
obs.horizon = 0.
sunset = obs.previous_setting(sun)
# pyephem's Date.tuple() splits a date into y,m,d,h,m,s
d = sunset.tuple()
#print('Date at sunset, UTC:', d)
year,month,day = d[:3]
plandict['date'] = '%04i-%02i-%02i' % (year, month, day)
# Decide the pass.
goodseeing = plandict['seeing'] < 1.3
photometric = plandict['transparency'] > 0.9
if goodseeing and photometric:
passnum = 1
elif goodseeing or photometric:
passnum = 2
else:
passnum = 3
plandict['pass'] = passnum
## ??
plandict['portion'] = 1.0
print('Replan command:')
print()
print('python2.7 nightlystrategy.py --seeg %(seeing).3f --seer %(seeing).3f --seez %(seeing).3f --sbg %(sbg).3f --sbr %(sbr).3f --sbz %(sbz).3f --transparency %(transparency).3f --start-date %(startdate)s --start-time %(starttime)s --end-date %(enddate)s --end-time %(endtime)s --date %(date)s --portion %(portion)f --pass %(pass)i' % plandict)
print()
else:
plandict = None
expfactor = 0.
rtn = (M, plandict, expnum)
if not db:
return rtn
m.racenter = M['ra_ccd']
m.deccenter = M['dec_ccd']
m.ebv = ebv
zp = M.get('zp', 0.)
if zp is None:
zp = 0.
m.zeropoint = zp
m.transparency = trans
m.seeing = M.get('seeing', 0.)
m.sky = M['skybright']
m.expfactor = expfactor
m.dx = M.get('dx', 0)
m.dy = M.get('dy', 0)
m.nmatched = M.get('nmatched',0)
img = fitsio.read(fn, ext=1)
cheaphash = np.sum(img)
# cheaphash becomes an int64.
m.md5sum = cheaphash
set_tile_fields(m, phdr, tiles)
m.save()
return rtn
def main():
ps = PlotSequence('cov')
survey = LegacySurveyData()
ra,dec = 242.0, 10.2
fn = 'coverage-ccds.fits'
if not os.path.exists(fn):
ccds = survey.get_ccds()
ccds.cut(ccds.filter == 'r')
ccds.cut(ccds.propid == '2014B-0404')
ccds.cut(np.hypot(ccds.ra_bore - ra, ccds.dec_bore - dec) < 2.5)
print(np.unique(ccds.expnum), 'unique exposures')
print('propids', np.unique(ccds.propid))
ccds.writeto(fn)
else:
ccds = fits_table(fn)
plt.clf()
for e in np.unique(ccds.expnum):
I = np.flatnonzero(ccds.expnum == e)
plt.plot(ccds.ra[I], ccds.dec[I], '.')
ps.savefig()
degw = 3.0
pixscale = 10.
W = degw * 3600 / 10.
H = W
hi = 6
cmap = cmap_discretize('jet', hi+1)
wcs = Tan(ra, dec, W/2.+0.5, H/2.+0.5,
-pixscale/3600., 0., 0., pixscale/3600., float(W), float(H))
r0,d0 = wcs.pixelxy2radec(1,1)
r1,d1 = wcs.pixelxy2radec(W,H)
extent = [min(r0,r1),max(r0,r1), min(d0,d1),max(d0,d1)]
for expnums in [ [348666], [348666,348710, 348686],
[348659, 348667, 348658, 348666, 348665, 348669, 348668],
None,
[348683, 348687, 347333, 348686, 348685, 348692, 348694,
348659, 348667, 348658, 348666, 348665, 348669, 348668,
348707, 348709, 348708, 348710, 348711, 348716, 348717],
]:
nexp = np.zeros((H,W), np.uint8)
for ccd in ccds:
if expnums is not None and not ccd.expnum in expnums:
continue
ccdwcs = survey.get_approx_wcs(ccd)
r,d = ccdwcs.pixelxy2radec(1, 1)
ok,x0,y0 = wcs.radec2pixelxy(r, d)
r,d = ccdwcs.pixelxy2radec(ccd.width, ccd.height)
ok,x1,y1 = wcs.radec2pixelxy(r, d)
xlo = np.clip(int(np.round(min(x0,x1))) - 1, 0, W-1)
xhi = np.clip(int(np.round(max(x0,x1))) - 1, 0, W-1)
ylo = np.clip(int(np.round(min(y0,y1))) - 1, 0, H-1)
yhi = np.clip(int(np.round(max(y0,y1))) - 1, 0, H-1)
nexp[ylo:yhi+1, xlo:xhi+1] += 1
plt.clf()
plt.imshow(nexp, interpolation='nearest', origin='lower',
vmin=-0.5, vmax=hi+0.5, cmap=cmap, extent=extent)
plt.colorbar(ticks=np.arange(hi+1))
ps.savefig()
O = fits_table('obstatus/decam-tiles_obstatus.fits')
O.cut(np.hypot(O.ra - ra, O.dec - dec) < 2.5)
for p in [1,2,3]:
print('Pass', p, 'exposures:', O.r_expnum[O.get('pass') == p])
O.cut(O.get('pass') == 2)
print(len(O), 'pass 2 nearby')
d = np.hypot(O.ra - ra, O.dec - dec)
print('Dists:', d)
I = np.flatnonzero(d < 0.5)
assert(len(I) == 1)
ocenter = O[I[0]]
print('Center expnum', ocenter.r_expnum)
I = np.flatnonzero(d >= 0.5)
O.cut(I)
#center = ccds[ccds.expnum == ocenter.r_expnum]
#p2 = ccds[ccds.
ok,xc,yc = wcs.radec2pixelxy(ocenter.ra, ocenter.dec)
xx,yy = np.meshgrid(np.arange(W)+1, np.arange(H)+1)
c_d2 = (xc - xx)**2 + (yc - yy)**2
best = np.ones((H,W), bool)
for o in O:
ok,x,y = wcs.radec2pixelxy(o.ra, o.dec)
d2 = (x - xx)**2 + (y - yy)**2
best[d2 < c_d2] = False
del d2
del c_d2,xx,yy
# plt.clf()
# plt.imshow(best, interpolation='nearest', origin='lower', cmap='gray',
# vmin=0, vmax=1)
# ps.savefig()
plt.clf()
plt.imshow(nexp * best, interpolation='nearest', origin='lower',
vmin=-0.5, vmax=hi+0.5, cmap=cmap, extent=extent)
plt.colorbar(ticks=np.arange(hi+1))
ps.savefig()
plt.clf()
n,b,p = plt.hist(np.clip(nexp[best], 0, hi), range=(-0.5,hi+0.5), bins=hi+1)
plt.xlim(-0.5, hi+0.5)
ps.savefig()
print('b', b)
print('n', n)
print('fracs', np.array(n) / np.sum(n))
print('pcts', ', '.join(['%.1f' % f for f in 100. * np.array(n)/np.sum(n)]))
t1 = Time()
print 'Stage tims:', t1 - t0
R = stage_srcs(**P)
P.update(R)
t2 = Time()
print 'Stage srcs:', t2 - t1
R = stage_fitblobs(**P)
P.update(R)
t2b = Time()
print 'Stage fitblobs:', t2b - t2
P.update(catalogfn=catalogfn)
stage_writecat(**P)
t3 = Time()
print 'Stage writecat:', t3 - t2b
# Plots
print
print 'plots:', P.keys()
print
ps = PlotSequence(pspat)
P.update(ps=ps, outdir='pipebrick-plots')
stage_fitplots(**P)
t4 = Time()
print 'Stage fitplots:', t4 - t3
print 'Total:', t4 - t0
def main():
# ps = PlotSequence('pro')
# star_profiles(ps)
# sys.exit(0)
#survey_dir = '/project/projectdirs/desiproc/dr3'
#survey = LegacySurveyData(survey_dir=survey_dir)
survey = LegacySurveyData()
ralo, rahi = 240, 245
declo, dechi = 5, 12
ps = PlotSequence('comp')
bands = 'grz'
ccdfn = 'ccds-forced.fits'
if not os.path.exists(ccdfn):
ccds = survey.get_annotated_ccds()
ccds.cut((ccds.ra > ralo) * (ccds.ra < rahi) * (ccds.dec > declo) *
(ccds.dec < dechi))
print(len(ccds), 'CCDs')
ccds.path = np.array([
os.path.join( #'dr3',
'forced', ('%08i' % e)[:5], '%08i' % e,
'decam-%08i-%s-forced.fits' % (e, n.strip()))
for e, n in zip(ccds.expnum, ccds.ccdname)
])
I, = np.nonzero([os.path.exists(fn) for fn in ccds.path])
print(len(I), 'CCDs with forced photometry')
ccds.cut(I)
#ccds = ccds[:500]
#e,I = np.unique(ccds.expnum, return_index=True)
#print(len(I), 'unique exposures')
#ccds.cut(I)
FF = read_forcedphot_ccds(ccds, survey)
FF.writeto('forced-all-matches.fits')
# - z band -- no trend w/ PS1 mag (brighter-fatter)
ccds.writeto(ccdfn)
ccdfn2 = 'ccds-forced-2.fits'
if not os.path.exists(ccdfn2):
ccds = fits_table(ccdfn)
# Split into brighter/fainter halves
FF = fits_table('forced-all-matches.fits')
print(len(FF), 'forced measurements')
FF.cut(FF.masked == False)
print(len(FF), 'forced measurements not masked')
ccds.brightest_mdiff = np.zeros(len(ccds))
ccds.brightest_mscatter = np.zeros(len(ccds))
ccds.bright_mdiff = np.zeros(len(ccds))
ccds.bright_mscatter = np.zeros(len(ccds))
ccds.faint_mdiff = np.zeros(len(ccds))
ccds.faint_mscatter = np.zeros(len(ccds))
for iccd in range(len(ccds)):
I = np.flatnonzero(FF.iforced == iccd)
if len(I) == 0:
continue
if len(I) < 10:
continue
F = FF[I]
b = np.percentile(F.psmag, 10)
m = np.median(F.psmag)
print(len(F), 'matches for CCD', iccd, 'median mag', m, '10th pct',
b)
J = np.flatnonzero(F.psmag < b)
diff = F.mag[J] - F.psmag[J]
ccds.brightest_mdiff[iccd] = np.median(diff)
ccds.brightest_mscatter[iccd] = (np.percentile(diff, 84) -
np.percentile(diff, 16)) / 2.
J = np.flatnonzero(F.psmag < m)
diff = F.mag[J] - F.psmag[J]
ccds.bright_mdiff[iccd] = np.median(diff)
ccds.bright_mscatter[iccd] = (np.percentile(diff, 84) -
np.percentile(diff, 16)) / 2.
J = np.flatnonzero(F.psmag > m)
diff = F.mag[J] - F.psmag[J]
ccds.faint_mdiff[iccd] = np.median(diff)
ccds.faint_mscatter[iccd] = (np.percentile(diff, 84) -
np.percentile(diff, 16)) / 2.
ccds.writeto(ccdfn2)
ccds = fits_table(ccdfn2)
plt.clf()
plt.hist(ccds.nforced, bins=100)
plt.title('nforced')
ps.savefig()
plt.clf()
plt.hist(ccds.nmatched, bins=100)
plt.title('nmatched')
ps.savefig()
#ccds.cut(ccds.nmatched >= 150)
ccds.cut(ccds.nmatched >= 50)
print('Cut to', len(ccds), 'with >50 matched')
ccds.cut(ccds.photometric)
print('Cut to', len(ccds), 'photometric')
neff = 1. / ccds.psfnorm_mean**2
# Narcsec is in arcsec**2
narcsec = neff * ccds.pixscale_mean**2
# to arcsec
narcsec = np.sqrt(narcsec)
# Correction factor to get back to equivalent of Gaussian sigma
narcsec /= (2. * np.sqrt(np.pi))
# Conversion factor to FWHM (2.35)
narcsec *= 2. * np.sqrt(2. * np.log(2.))
ccds.psfsize = narcsec
for band in bands:
I = np.flatnonzero(
(ccds.filter == band) * (ccds.photometric) * (ccds.blacklist_ok))
mlo, mhi = -0.01, 0.05
plt.clf()
plt.plot(ccds.ccdzpt[I], ccds.exptime[I], 'k.', alpha=0.1)
J = np.flatnonzero((ccds.filter == band) * (ccds.photometric == False))
plt.plot(ccds.ccdzpt[J], ccds.exptime[J], 'r.', alpha=0.1)
plt.xlabel('Zeropoint (mag)')
plt.ylabel('exptime')
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
plt.clf()
plt.plot(ccds.ccdzpt[I],
np.clip(ccds.mdiff[I], mlo, mhi),
'k.',
alpha=0.1)
plt.xlabel('Zeropoint (mag)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
#plt.axis([0, mxsee, mlo,mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
plt.clf()
plt.plot(ccds.ccdzpt[I], ccds.psfsize[I], 'k.', alpha=0.1)
plt.xlabel('Zeropoint (mag)')
plt.ylabel('PSF size (arcsec)')
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
# plt.clf()
# plt.plot(ccds.avsky[I],
# np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
# plt.xlabel('avsky')
# plt.ylabel('DECaLS PSF - PS1 (mag)')
# plt.axhline(0, color='k', alpha=0.2)
# plt.title('DR3: EDR region, Forced phot: %s band' % band)
# ps.savefig()
#
# plt.clf()
# plt.plot(ccds.meansky[I],
# np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
# plt.xlabel('meansky')
# plt.ylabel('DECaLS PSF - PS1 (mag)')
# plt.axhline(0, color='k', alpha=0.2)
# plt.title('DR3: EDR region, Forced phot: %s band' % band)
# ps.savefig()
plt.clf()
lo, hi = (-0.02, 0.05)
ha = dict(bins=50, histtype='step', range=(lo, hi))
n, b, p1 = plt.hist(ccds.brightest_mdiff[I], color='r', **ha)
n, b, p2 = plt.hist(ccds.bright_mdiff[I], color='g', **ha)
n, b, p3 = plt.hist(ccds.faint_mdiff[I], color='b', **ha)
plt.legend((p1[0], p2[0], p3[0]),
('Brightest 10%', 'Brightest 50%', 'Faintest 50%'))
plt.xlabel('DECaLS PSF - PS1 (mag)')
plt.ylabel('Number of CCDs')
plt.title('DR3: EDR region, Forced phot: %s band' % band)
plt.xlim(lo, hi)
ps.savefig()
for band in bands:
I = np.flatnonzero(ccds.filter == band)
mxsee = 4.
mlo, mhi = -0.01, 0.05
plt.clf()
plt.plot(np.clip(ccds.psfsize[I], 0, mxsee),
np.clip(ccds.mdiff[I], mlo, mhi),
'k.',
alpha=0.1)
# for p in [1,2,3]:
# J = np.flatnonzero(ccds.tilepass[I] == p)
# if len(J):
# plt.plot(np.clip(ccds.psfsize[I[J]], 0, mxsee),
# np.clip(ccds.mdiff[I[J]], mlo,mhi), '.', color='rgb'[p-1], alpha=0.2)
#plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.')
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
plt.axis([0, mxsee, mlo, mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
# Group by exposure
for band in bands:
I = np.flatnonzero(
(ccds.filter == band) * (ccds.photometric) * (ccds.blacklist_ok))
E, J = np.unique(ccds.expnum[I], return_index=True)
print(len(E), 'unique exposures in', band)
exps = ccds[I[J]]
print(len(exps), 'unique exposures in', band)
assert (len(np.unique(exps.expnum)) == len(exps))
exps.ddiff = np.zeros(len(exps))
exps.dsize = np.zeros(len(exps))
exps.nccds = np.zeros(len(exps), int)
exps.brightest_ddiff = np.zeros(len(exps))
exps.bright_ddiff = np.zeros(len(exps))
exps.faint_ddiff = np.zeros(len(exps))
for iexp, exp in enumerate(exps):
J = np.flatnonzero(ccds.expnum[I] == exp.expnum)
J = I[J]
print(len(J), 'CCDs in exposure', exp.expnum)
exps.brightest_mdiff[iexp] = np.median(ccds.brightest_mdiff[J])
exps.bright_mdiff[iexp] = np.median(ccds.bright_mdiff[J])
exps.faint_mdiff[iexp] = np.median(ccds.faint_mdiff[J])
exps.brightest_ddiff[iexp] = (
np.percentile(ccds.brightest_mdiff[J], 84) -
np.percentile(ccds.brightest_mdiff[J], 16)) / 2.
exps.bright_ddiff[iexp] = (
np.percentile(ccds.bright_mdiff[J], 84) -
np.percentile(ccds.bright_mdiff[J], 16)) / 2.
exps.faint_ddiff[iexp] = (
np.percentile(ccds.faint_mdiff[J], 84) -
np.percentile(ccds.faint_mdiff[J], 16)) / 2.
exps.mdiff[iexp] = np.median(ccds.mdiff[J])
exps.ddiff[iexp] = (np.percentile(ccds.mdiff[J], 84) -
np.percentile(ccds.mdiff[J], 16)) / 2.
exps.psfsize[iexp] = np.median(ccds.psfsize[J])
exps.dsize[iexp] = (np.percentile(ccds.psfsize[J], 84) -
np.percentile(ccds.psfsize[J], 16)) / 2.
exps.nccds[iexp] = len(J)
mxsee = 4.
mlo, mhi = -0.01, 0.05
exps.cut(exps.nccds >= 10)
plt.clf()
plt.errorbar(
np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.mdiff, mlo, mhi),
yerr=exps.ddiff,
#xerr=exps.dsize,
fmt='.',
color='k')
# plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.brightest_mdiff, mlo,mhi),
# yerr=exps.brightest_ddiff, fmt='r.')
# plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.bright_mdiff, mlo,mhi),
# yerr=exps.bright_ddiff, fmt='g.')
# plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.faint_mdiff, mlo,mhi),
# yerr=exps.faint_ddiff, fmt='b.')
# plt.plot(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.brightest_mdiff, mlo,mhi), 'r.')
# plt.plot(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.bright_mdiff, mlo,mhi), 'g.')
# plt.plot(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.faint_mdiff, mlo,mhi), 'b.')
#plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.')
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
plt.axis([0, mxsee, mlo, mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
plt.clf()
p1 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.brightest_mdiff, mlo, mhi),
'r.',
alpha=0.5)
p2 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.bright_mdiff, mlo, mhi),
'g.',
alpha=0.5)
p3 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.faint_mdiff, mlo, mhi),
'b.',
alpha=0.5)
plt.legend((p1[0], p2[0], p3[0]),
('Brightest 10%', 'Brightest 50%', 'Faintest 50%'))
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
plt.axis([0, mxsee, mlo, mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
J = np.argsort(-exps.mdiff)
for j in J:
print(' Photometric diff', exps.mdiff[j], 'PSF size',
exps.psfsize[j], 'expnum', exps.expnum[j])
sys.exit(0)
metavar='DECam-filename.fits.fz',
nargs='+',
help='DECam raw images to process')
args = parser.parse_args()
fns = args.images
#exts = args.ext
#if len(exts) == 0:
# exts = ['N4']
ext = args.ext
ps = None
if args.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(args.plots)
measargs = dict()
if args.aprad is not None:
measargs.update(aprad=args.aprad)
vals = {}
for fn in fns:
#for ext in exts:
print()
print('Measuring', fn, 'ext', ext)
d = measure_raw(fn, ext=ext, ps=ps, measargs=measargs)
d.update(filename=fn, ext=ext)
for k, v in d.items():
if not k in vals:
vals[k] = [v]
psfvar = np.zeros((3, 2, 2))
psfvar[0, 0, 0] = 1.2**2
psfvar[0, 1, 1] = psfvar[0, 0, 0]
psfvar[1, 0, 0] = 2.4**2
psfvar[1, 1, 1] = psfvar[1, 0, 0]
psfvar[2, 0, 0] = 3.6**2
psfvar[2, 1, 1] = psfvar[2, 0, 0]
psf = MixtureOfGaussians(psfamp, psfmean, psfvar)
functional_test_patch_maker('test_patch.png')
functional_test_patch_maker('test_psf_patch.png', psf=psf)
# functional test: c_gauss_2d_approx
from tractor.mix import c_gauss_2d_approx, c_gauss_2d_grid, c_gauss_2d_approx2, c_gauss_2d_approx3
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('approx')
for j in range(100):
print()
print('j =', j)
print()
x0, x1 = -50, 50
y0, y1 = -51, 51
W = x1 - x0
H = y1 - y0
result = np.zeros((H, W))
amp = np.array([1.0])
mean = np.array([[0.3, 0.7], ])
minval = 1e-3
def rbmain():
travis = 'travis' in sys.argv
extra_args = [
'--old-calibs-ok',
#'--verbose',
]
if travis:
extra_args.extend(
['--no-wise-ceres', '--no-gaia', '--no-large-galaxies'])
if 'ceres' in sys.argv:
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--ceres',
'--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres',
'--no-depth-cut'
])
sys.exit(0)
# demo RexGalaxy, with plots
if False:
from legacypipe.survey import RexGalaxy
from tractor import NanoMaggies, PixPos
from tractor import Image, GaussianMixturePSF, LinearPhotoCal
from legacypipe.survey import LogRadius
rex = RexGalaxy(PixPos(1., 2.), NanoMaggies(r=3.), LogRadius(0.))
print('Rex:', rex)
print('Rex params:', rex.getParams())
print('Rex nparams:', rex.numberOfParams())
H, W = 100, 100
tim = Image(data=np.zeros((H, W), np.float32),
inverr=np.ones((H, W), np.float32),
psf=GaussianMixturePSF(1., 0., 0., 4., 4., 0.),
photocal=LinearPhotoCal(1., band='r'))
derivs = rex.getParamDerivatives(tim)
print('Derivs:', len(derivs))
print('Rex params:', rex.getParamNames())
import pylab as plt
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('rex')
for d, nm in zip(derivs, rex.getParamNames()):
plt.clf()
plt.imshow(d.patch, interpolation='nearest', origin='lower')
plt.title('Derivative %s' % nm)
ps.savefig()
sys.exit(0)
# Test RexGalaxy
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6-rex'
main(args=[
'--brick',
'1102p240',
'--zoom',
'500',
'600',
'650',
'750',
'--force-all',
'--no-write',
'--no-wise',
#'--rex', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + extra_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0] == 'PSF ')
cmd = (
'(cd %s && sha256sum -c %s)' %
(outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
print(cmd)
rtn = os.system(cmd)
assert (rtn == 0)
# Test with a Tycho-2 star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
outdir = 'out-testcase6'
main(args=[
'--brick', '1102p240', '--zoom', '500', '600', '650', '750',
'--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir,
'--outdir', outdir
] + extra_args)
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 2)
print('Types:', T.type)
# Since there is a Tycho-2 star in the blob, forced to be PSF.
assert (T.type[0] == 'PSF ')
# Test that we can run splinesky calib if required...
from legacypipe.decam import DecamImage
DecamImage.splinesky_boxsize = 128
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4'
fn = os.path.join(surveydir, 'calib', 'decam', 'splinesky', '00431',
'00431608', 'decam-00431608-N3.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
'--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image',
'--no-hybrid-psf', '--survey-dir', surveydir, '--outdir', outdir
] + extra_args + ['-v'])
print('Checking for calib file', fn)
assert (os.path.exists(fn))
# Wrap-around, hybrid PSF
surveydir = os.path.join(os.path.dirname(__file__), 'testcase8')
outdir = 'out-testcase8'
main(args=[
'--brick',
'1209p050',
'--zoom',
'720',
'1095',
'3220',
'3500',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + extra_args)
# Test with a Tycho-2 star + another saturated star in the blob.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase7')
outdir = 'out-testcase7'
# remove --no-gaia
my_extra_args = [a for a in extra_args if a != '--no-gaia']
os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia-dr2')
os.environ['GAIA_CAT_VER'] = '2'
main(args=[
'--brick',
'1102p240',
'--zoom',
'250',
'350',
'1550',
'1650',
'--force-all',
'--no-write',
'--no-wise', #'--plots',
'--survey-dir',
surveydir,
'--outdir',
outdir
] + my_extra_args)
del os.environ['GAIA_CAT_DIR']
del os.environ['GAIA_CAT_VER']
fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 4)
# Check skipping blobs outside the brick's unique area.
# (this now doesn't detect any sources at all, reasonably)
# surveydir = os.path.join(os.path.dirname(__file__), 'testcase5')
# outdir = 'out-testcase5'
#
# fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits')
# if os.path.exists(fn):
# os.unlink(fn)
#
# main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150',
# '--force-all', '--no-write', '--coadd-bw',
# '--survey-dir', surveydir,
# '--early-coadds',
# '--outdir', outdir] + extra_args)
#
# assert(os.path.exists(fn))
# T = fits_table(fn)
# assert(len(T) == 1)
# Custom RA,Dec; blob ra,dec.
surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
outdir = 'out-testcase4b'
# Catalog written with one entry (--blobradec)
fn = os.path.join(outdir, 'tractor', 'cus',
'tractor-custom-186743p25461.fits')
if os.path.exists(fn):
os.unlink(fn)
main(args=[
'--radec', '186.743965', '25.461788', '--width', '250', '--height',
'250', '--force-all', '--no-write', '--no-wise', '--blobradec',
'186.740369', '25.453855', '--survey-dir', surveydir, '--outdir',
outdir
] + extra_args)
assert (os.path.exists(fn))
T = fits_table(fn)
assert (len(T) == 1)
surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
outdir = 'out-testcase3'
checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle')
if os.path.exists(checkpoint_fn):
os.unlink(checkpoint_fn)
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
] + extra_args)
# Read catalog into Tractor sources to test read_fits_catalog
from legacypipe.catalog import read_fits_catalog
from legacypipe.survey import LegacySurveyData, GaiaSource
from tractor.galaxy import DevGalaxy
from tractor import PointSource
survey = LegacySurveyData(survey_dir=outdir)
fn = survey.find_file('tractor', brick='2447p120')
print('Checking', fn)
T = fits_table(fn)
cat = read_fits_catalog(T, fluxPrefix='')
print('Read catalog:', cat)
assert (len(cat) == 2)
src = cat[0]
assert (type(src) == DevGalaxy)
assert (np.abs(src.pos.ra - 244.77973) < 0.00001)
assert (np.abs(src.pos.dec - 12.07233) < 0.00001)
src = cat[1]
print('Source', src)
assert (type(src) in [PointSource, GaiaSource])
assert (np.abs(src.pos.ra - 244.77830) < 0.00001)
assert (np.abs(src.pos.dec - 12.07250) < 0.00001)
# DevGalaxy(pos=RaDecPos[244.77975494973529, 12.072348111713127], brightness=NanoMaggies: g=19.2, r=17.9, z=17.1, shape=re=2.09234, e1=-0.198453, e2=0.023652,
# PointSource(RaDecPos[244.77833280764278, 12.072521274981987], NanoMaggies: g=25, r=23, z=21.7)
# Check that we can run again, using that checkpoint file.
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1', '--threads', '2'
] + extra_args)
# Assert...... something?
# Test --checkpoint without --threads
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', outdir, '--checkpoint', checkpoint_fn,
'--checkpoint-period', '1'
] + extra_args)
# MzLS + BASS data
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass')
# main(args=['--brick', '3521p002', '--zoom', '2400', '2450', '1200', '1250',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass'])
# From Kaylan's Bootes pre-DR4 run
# surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3')
# main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200',
# '--no-wise', '--force-all', '--no-write',
# '--survey-dir', surveydir2,
# '--outdir', 'out-mzlsbass3'] + extra_args)
# With plots!
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
'--outdir', 'out-testcase3', '--plots', '--nblobs', '1'
] + extra_args)
# Decals Image Simulations
# Uncomment WHEN galsim build for Travis
#os.environ["DECALS_SIM_DIR"]= os.path.join(os.path.dirname(__file__),'image_sims')
#brick= '2447p120'
#sim_main(args=['--brick', brick, '-n', '2', '-o', 'STAR', \
# '-ic', '1', '--rmag-range', '18', '26', '--threads', '1',\
# '--zoom', '1020', '1070', '2775', '2815'])
# Check if correct files written out
#rt_dir= os.path.join(os.getenv('DECALS_SIM_DIR'),brick,'star','001')
#assert( os.path.exists(os.path.join(rt_dir,'../','metacat-'+brick+'-star.fits')) )
#for fn in ['tractor-%s-star-01.fits' % brick,'simcat-%s-star-01.fits' % brick]:
# assert( os.path.exists(os.path.join(rt_dir,fn)) )
#for fn in ['image','model','resid','simscoadd']:
# assert( os.path.exists(os.path.join(rt_dir,'qa-'+brick+'-star-'+fn+'-01.jpg')) )
if not travis:
# With ceres
main(args=[
'--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
'--no-wise', '--force-all', '--no-write', '--ceres',
'--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres'
] + extra_args)
import numpy as np
import fitsio
from astrometry.util.fits import fits_table
from astrometry.util.plotutils import PlotSequence
from astrometry.util.multiproc import multiproc
from legacypipe.survey import *
from legacypipe.runs import get_survey
import photutils
import sys
import os
if __name__ == '__main__':
ps = PlotSequence('psfnorm')
survey = get_survey('dr4v2')
ccds = survey.get_ccds_readonly()
print(len(ccds), 'CCDs')
ccds = ccds[ccds.camera == '90prime']
print(len(ccds), 'Bok CCDs')
## Skip a few missing ones...
ccds = ccds[24:]
for i, ccd in enumerate(ccds):
print('Reading CCD', i)
try:
im = survey.get_image_object(ccd, makeNewWeightMap=False)
print('Reading', im)
p3 = plt.plot(dists, corrs_y, 'g.-')
if medians:
p4 = plt.plot(dists, rcorrs, 'b.--')
p5 = plt.plot(dists, rcorrs_x, 'r.--')
p6 = plt.plot(dists, rcorrs_y, 'g.--')
plt.xlabel('Pixel offset')
plt.ylabel('Correlation')
plt.axhline(0, color='k', alpha=0.3)
plt.legend([p1[0], p2[0], p3[0]], ['Diagonal', 'X', 'Y'],
loc='upper right')
return (dists, corrs, corrs_x, corrs_y, rcorrs, rcorrs_x, rcorrs_y, mads,
mads_x, mads_y, mad_random)
ps = PlotSequence('noise')
# fitsgetext -i /global/homes/d/dstn/cosmo/staging/decam/DECam_Raw/20160107/c4d_160107_213046_fri.fits.fz -e 0 -e 1 -o flat.fits
# img = fitsio.read('flat.fits').astype(np.float32)
# # trim the edges
# img = img[100:-100, 100:-100]
# img -= np.median(img)
# Pixel correlations in a flat image:
if False:
from obsbot.measure_raw import DECamMeasurer
from obsbot.decam import DecamNominalCalibration
nom = DecamNominalCalibration()
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--name1', help='Name for first data set')
parser.add_argument('--name2', help='Name for second data set')
parser.add_argument('--plot-prefix',
default='compare',
help='Prefix for plot filenames; default "%default"')
parser.add_argument('--match',
default=1.0,
help='Astrometric cross-match distance in arcsec')
parser.add_argument('dir1', help='First directory to compare')
parser.add_argument('dir2', help='Second directory to compare')
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = 'Comparing %s to %s' % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile('tractor-.*.fits')
cat1, cat2 = [], []
for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print('Skipping', fn, 'due to filename')
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), 'from', fn)
cat.append(t)
cat1 = merge_tables(cat1, columns='fillzero')
cat2 = merge_tables(cat2, columns='fillzero')
print('Total of', len(cat1), 'from', name1)
print('Total of', len(cat2), 'from', name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)
cat1.cut((cat1.decam_anymask[:, 1] == 0) *
(cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0))
cat2.cut((cat2.decam_anymask[:, 1] == 0) *
(cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0))
print('Total of', len(cat1), 'unmasked from', name1)
print('Total of', len(cat2), 'unmasked from', name2)
I, J, d = match_radec(cat1.ra,
cat1.dec,
cat2.ra,
cat2.dec,
opt.match / 3600.,
nearest=True)
print(len(I), 'matched')
plt.clf()
plt.hist(d * 3600., 100)
plt.xlabel('Match distance (arcsec)')
plt.title(tt)
ps.savefig()
matched1 = cat1[I]
matched2 = cat2[J]
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
K = np.flatnonzero((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
print('Median mw_trans', band, 'is',
np.median(matched1.decam_mw_transmission[:, iband]))
plt.clf()
plt.errorbar(
matched1.decam_flux[K, iband],
matched2.decam_flux[K, iband],
fmt='.',
color=cc,
xerr=1. / np.sqrt(matched1.decam_flux_ivar[K, iband]),
yerr=1. / np.sqrt(matched2.decam_flux_ivar[K, iband]),
alpha=0.1,
)
plt.xlabel('%s flux: %s' % (name1, band))
plt.ylabel('%s flux: %s' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
plt.axis([-100, 1000, -100, 1000])
plt.title(tt)
ps.savefig()
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
K = np.flatnonzero(good)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
P = np.flatnonzero(good * psf1 * psf2)
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1. / iv1 + 1. / iv2)
plt.clf()
plt.plot(
mag1[K],
(matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) /
std[K],
'.',
alpha=0.1,
color=cc)
plt.plot(
mag1[P],
(matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) /
std[P],
'.',
alpha=0.1,
color='k')
plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' %
(name2, name1, band))
plt.xlabel('%s mag: %s' % (name1, band))
plt.axhline(0, color='k', alpha=0.5)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
plt.clf()
lp, lt = [], []
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
#good = True
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1. / iv1 + 1. / iv2)
#std = np.hypot(std, 0.01)
G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) *
(mag1 >= 20) *
(mag1 < dict(g=24, r=23.5, z=22.5)[band]))
n, b, p = plt.hist(
(matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) /
std[G],
range=(-4, 4),
bins=50,
histtype='step',
color=cc,
normed=True)
sig = (matched2.decam_flux[G, iband] -
matched1.decam_flux[G, iband]) / std[G]
print('Raw mean and std of points:', np.mean(sig), np.std(sig))
med = np.median(sig)
rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
print('Median and percentile-based sigma:', med, rsigma)
lp.append(p[0])
lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))
bins = []
gaussint = []
for blo, bhi in zip(b, b[1:]):
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
#bins.extend([blo,bhi])
#gaussint.extend([c,c])
bins.append((blo + bhi) / 2.)
gaussint.append(c)
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.title(tt)
plt.xlabel('Flux difference / error (sigma)')
plt.axvline(0, color='k', alpha=0.1)
plt.ylim(0, 0.45)
plt.legend(lp, lt, loc='upper right')
ps.savefig()
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
plt.clf()
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband])
meanmag = NanoMaggies.nanomaggiesToMag(
(matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) /
2.)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0) * np.isfinite(mag1) *
np.isfinite(mag2))
K = np.flatnonzero(good)
P = np.flatnonzero(good * psf1 * psf2)
plt.errorbar(mag1[K],
mag2[K],
fmt='.',
color=cc,
xerr=magerr1[K],
yerr=magerr2[K],
alpha=0.1)
plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s (mag)' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
plt.axis([24, 16, 24, 16])
plt.title(tt)
ps.savefig()
plt.clf()
plt.errorbar(mag1[K],
mag2[K] - mag1[K],
fmt='.',
color=cc,
xerr=magerr1[K],
yerr=magerr2[K],
alpha=0.1)
plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -1, 1])
plt.title(tt)
ps.savefig()
magbins = np.arange(16, 24.001, 0.5)
plt.clf()
plt.plot(mag1[K],
(mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
'.',
color=cc,
alpha=0.1)
plt.plot(mag1[P],
(mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
'k.',
alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
plt.clf()
plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)
midmag = []
vals = np.zeros((len(magbins) - 1, 5))
median_err1 = []
iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)
# FIXME -- should we do some stats after taking off the mean difference?
for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
midmag.append((mlo + mhi) / 2.)
median_err1.append(np.median(magerr1[I]))
if len(I) == 0:
continue
# median and +- 1 sigma quantiles
ybin = y[I]
vals[bini, 0] = np.percentile(ybin, 16)
vals[bini, 1] = np.median(ybin)
vals[bini, 2] = np.percentile(ybin, 84)
# +- 2 sigma quantiles
vals[bini, 3] = np.percentile(ybin, 2.3)
vals[bini, 4] = np.percentile(ybin, 97.7)
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
'vs expected for Gaussian;', len(ybin), 'points')
# if iqd > iqd_gauss:
# # What error adding in quadrature would you need to make the IQD match?
# err = median_err1[-1]
# target_err = err * (iqd / iqd_gauss)
# sys_err = np.sqrt(target_err**2 - err**2)
# print('--> add systematic error', sys_err)
# ~ Johan's cuts
mlo = 21.
mhi = dict(g=24., r=23.5, z=22.5)[band]
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
ybin = y[I]
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', mlo, mhi, 'band', band,
': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;',
len(ybin), 'points')
if iqd > iqd_gauss:
# What error adding in quadrature would you need to make
# the IQD match?
err = np.median(np.hypot(magerr1[I], magerr2[I]))
print('Median error (hypot):', err)
target_err = err * (iqd / iqd_gauss)
print('Target:', target_err)
sys_err = np.sqrt((target_err**2 - err**2) / 2.)
print('--> add systematic error', sys_err)
# check...
err_sys = np.hypot(np.hypot(magerr1, sys_err),
np.hypot(magerr2, sys_err))
ysys = (mag2 - mag1) / err_sys
ysys = ysys[I]
print('Resulting median error:', np.median(err_sys[I]))
iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
# Hmmm, this doesn't work... totally overshoots.
plt.errorbar(midmag,
vals[:, 1],
fmt='o',
color='b',
yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]),
capthick=3,
zorder=20)
plt.errorbar(midmag,
vals[:, 1],
fmt='o',
color='b',
yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]),
capthick=2,
zorder=20)
plt.axhline(1., color='b', alpha=0.2)
plt.axhline(-1., color='b', alpha=0.2)
plt.axhline(2., color='b', alpha=0.2)
plt.axhline(-2., color='b', alpha=0.2)
for mag, err, y in zip(midmag, median_err1, vals[:, 3]):
if not np.isfinite(err):
continue
if y < -6:
continue
plt.text(mag,
y - 0.1,
'%.3f' % err,
va='top',
ha='center',
color='k',
fontsize=10)
plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axvline(21, color='k', alpha=0.3)
plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)
plt.axis([24.1, 16, -6, 6])
plt.title(tt)
ps.savefig()
#magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
if band == 'g':
magbins = [20, 24]
elif band == 'r':
magbins = [20, 23.5]
elif band == 'z':
magbins = [20, 22.5]
slo, shi = -5, 5
plt.clf()
ha = dict(bins=25, range=(slo, shi), histtype='step', normed=True)
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
midmag = []
nn = []
rgbs = []
lt, lp = [], []
for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
if len(I) == 0:
continue
ybin = y[I]
rgb = [0., 0., 0.]
rgb[0] = float(bini) / (len(magbins) - 1)
rgb[2] = 1. - rgb[0]
n, b, p = plt.hist(ybin, color=rgb, **ha)
lt.append('mag %g to %g' % (mlo, mhi))
lp.append(p[0])
midmag.append((mlo + mhi) / 2.)
nn.append(n)
rgbs.append(rgb)
bins = []
gaussint = []
for blo, bhi in zip(b, b[1:]):
#midbin.append((blo+bhi)/2.)
#gaussint.append(scipy.stats.norm.cdf(bhi) -
# scipy.stats.norm.cdf(blo))
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
bins.extend([blo, bhi])
gaussint.extend([c, c])
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo, shi)
ps.savefig()
bincenters = b[:-1] + (b[1] - b[0]) / 2.
plt.clf()
lp = []
for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]):
p = plt.plot(bincenters, n, '-', color=rgb)
lp.append(p[0])
plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo, shi)
ps.savefig()
cmd = 'wget -O %s "%s"' % (fn, url)
print(cmd)
os.system(cmd)
img = plt.imread(fn)
M = 20
img = img[M:-M, M:-M, :]
plt.subplot(NR, NC, j+1)
plt.imshow(img, interpolation='nearest', origin='lower')
plt.xticks([])
plt.yticks([])
if __name__ == '__main__':
ps = PlotSequence('morph')
from glob import glob
from astrometry.util.fits import merge_tables, fits_table
HST = fits_table('acs-gc.fits')
print(len(HST), 'ACS sources')
HST.cut(HST.imaging == 'COSMOS ')
print(len(HST), 'in COSMOS')
HST.about()
for dirnm in ['cosmos-50-rex', 'cosmos-51-rex', 'cosmos-52-rex']:
fns = glob(os.path.join(dirnm, 'metrics', '*', 'all-models-*.fits'))
T = merge_tables([fits_table(fn) for fn in fns])
print(len(T), 'sources')
T.about()
def stage0(**kwargs):
ps = PlotSequence('cfht')
decals = CfhtDecals()
B = decals.get_bricks()
print('Bricks:')
B.about()
ra, dec = 190.0, 11.0
#bands = 'ugri'
bands = 'gri'
B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec)))
print('Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5])
brick = B[0]
pixscale = 0.186
#W,H = 1024,1024
#W,H = 2048,2048
#W,H = 3600,3600
W, H = 4800, 4800
targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H)
ccdfn = 'cfht-ccds.fits'
if os.path.exists(ccdfn):
T = fits_table(ccdfn)
else:
T = get_ccd_list()
T.writeto(ccdfn)
print(len(T), 'CCDs')
T.cut(ccds_touching_wcs(targetwcs, T))
print(len(T), 'CCDs touching brick')
T.cut(np.array([b in bands for b in T.filter]))
print(len(T), 'in bands', bands)
ims = []
for t in T:
im = CfhtImage(t)
# magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME'])
# fwhm = t.seeing / (pixscale * 3600)
# print '-> FWHM', fwhm, 'pix'
im.seeing = t.seeing
im.pixscale = t.pixscale
print('seeing', t.seeing)
print('pixscale', im.pixscale * 3600, 'arcsec/pix')
im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height)
ims.append(im)
# Read images, clip to ROI
targetrd = np.array([
targetwcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
keepims = []
tims = []
for im in ims:
print()
print('Reading expnum', im.expnum, 'name', im.extname, 'band', im.band,
'exptime', im.exptime)
band = im.band
wcs = im.read_wcs()
imh, imw = wcs.imageh, wcs.imagew
imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)]
ok, tx, ty = wcs.radec2pixelxy(targetrd[:-1, 0], targetrd[:-1, 1])
tpoly = zip(tx, ty)
clip = clip_polygon(imgpoly, tpoly)
clip = np.array(clip)
#print 'Clip', clip
if len(clip) == 0:
continue
x0, y0 = np.floor(clip.min(axis=0)).astype(int)
x1, y1 = np.ceil(clip.max(axis=0)).astype(int)
slc = slice(y0, y1 + 1), slice(x0, x1 + 1)
## FIXME -- it seems I got lucky and the cross product is
## negative == clockwise, as required by clip_polygon. One
## could check this and reverse the polygon vertex order.
# dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0]
# dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1]
# cross = dx0*dy1 - dx1*dy0
# print 'Cross:', cross
print('Image slice: x [%i,%i], y [%i,%i]' % (x0, x1, y0, y1))
print('Reading image from', im.imgfn, 'HDU', im.hdu)
img, imghdr = im.read_image(header=True, slice=slc)
goodpix = (img != 0)
print('Number of pixels == 0:', np.sum(img == 0))
print('Number of pixels != 0:', np.sum(goodpix))
if np.sum(goodpix) == 0:
continue
# print 'Image shape', img.shape
print('Image range', img.min(), img.max())
print('Goodpix image range:', (img[goodpix]).min(),
(img[goodpix]).max())
if img[goodpix].min() == img[goodpix].max():
print('No dynamic range in image')
continue
# print 'Reading invvar from', im.wtfn, 'HDU', im.hdu
# invvar = im.read_invvar(slice=slc)
# # print 'Invvar shape', invvar.shape
# # print 'Invvar range:', invvar.min(), invvar.max()
# invvar[goodpix == 0] = 0.
# if np.all(invvar == 0.):
# print 'Skipping zero-invvar image'
# continue
# assert(np.all(np.isfinite(img)))
# assert(np.all(np.isfinite(invvar)))
# assert(not(np.all(invvar == 0.)))
# # Estimate per-pixel noise via Blanton's 5-pixel MAD
# slice1 = (slice(0,-5,10),slice(0,-5,10))
# slice2 = (slice(5,None,10),slice(5,None,10))
# # print 'sliced shapes:', img[slice1].shape, img[slice2].shape
# # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape
# # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2])
# # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape
# mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel())
# sig1 = 1.4826 * mad / np.sqrt(2.)
# print 'MAD sig1:', sig1
# # invvar was 1 or 0
# invvar *= (1./(sig1**2))
# medsky = np.median(img[goodpix])
# Read full image for sig1 and sky estimate
fullimg = im.read_image()
fullgood = (fullimg != 0)
# Estimate per-pixel noise via Blanton's 5-pixel MAD
slice1 = (slice(0, -5, 10), slice(0, -5, 10))
slice2 = (slice(5, None, 10), slice(5, None, 10))
mad = np.median(
np.abs(fullimg[slice1] -
fullimg[slice2])[fullgood[slice1] *
fullgood[slice2]].ravel())
sig1 = 1.4826 * mad / np.sqrt(2.)
print('MAD sig1:', sig1)
medsky = np.median(fullimg[fullgood])
invvar = np.zeros_like(img)
invvar[goodpix] = 1. / sig1**2
# Median-smooth sky subtraction
plt.clf()
dimshow(np.round((img - medsky) / sig1), vmin=-3, vmax=5)
plt.title('Scalar median: %s' % im.name)
ps.savefig()
# medsky = np.zeros_like(img)
# # astrometry.util.util
# median_smooth(img, np.logical_not(goodpix), 256, medsky)
fullmed = np.zeros_like(fullimg)
median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed)
fullmed += medsky
medimg = fullmed[slc]
plt.clf()
dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5)
plt.title('Median filtered: %s' % im.name)
ps.savefig()
#print 'Subtracting median:', medsky
#img -= medsky
img -= medimg
primhdr = im.read_image_primary_header()
magzp = decals.get_zeropoint_for(im)
print('magzp', magzp)
zpscale = NanoMaggies.zeropointToScale(magzp)
print('zpscale', zpscale)
# Scale images to Nanomaggies
img /= zpscale
sig1 /= zpscale
invvar *= zpscale**2
orig_zpscale = zpscale
zpscale = 1.
assert (np.sum(invvar > 0) > 0)
print('After scaling:')
print('sig1', sig1)
print('invvar range', invvar.min(), invvar.max())
print('image range', img.min(), img.max())
assert (np.all(np.isfinite(img)))
assert (np.all(np.isfinite(invvar)))
assert (np.isfinite(sig1))
plt.clf()
lo, hi = -5 * sig1, 10 * sig1
n, b, p = plt.hist(img[goodpix].ravel(),
100,
range=(lo, hi),
histtype='step',
color='k')
xx = np.linspace(lo, hi, 200)
plt.plot(xx, max(n) * np.exp(-xx**2 / (2. * sig1**2)), 'r-')
plt.xlim(lo, hi)
plt.title('Pixel histogram: %s' % im.name)
ps.savefig()
twcs = ConstantFitsWcs(wcs)
if x0 or y0:
twcs.setX0Y0(x0, y0)
info = im.get_image_info()
fullh, fullw = info['dims']
# read fit PsfEx model
psfex = PsfEx.fromFits(im.psffitfn)
print('Read', psfex)
# HACK -- highly approximate PSF here!
#psf_fwhm = imghdr['FWHM']
#psf_fwhm = im.seeing
psf_fwhm = im.seeing / (im.pixscale * 3600)
print('PSF FWHM', psf_fwhm, 'pixels')
psf_sigma = psf_fwhm / 2.35
psf = NCircularGaussianPSF([psf_sigma], [1.])
print('img type', img.dtype)
tim = Image(img,
invvar=invvar,
wcs=twcs,
psf=psf,
photocal=LinearPhotoCal(zpscale, band=band),
sky=ConstantSky(0.),
name=im.name + ' ' + band)
tim.zr = [-3. * sig1, 10. * sig1]
tim.sig1 = sig1
tim.band = band
tim.psf_fwhm = psf_fwhm
tim.psf_sigma = psf_sigma
tim.sip_wcs = wcs
tim.x0, tim.y0 = int(x0), int(y0)
tim.psfex = psfex
tim.imobj = im
mn, mx = tim.zr
tim.ima = dict(interpolation='nearest',
origin='lower',
cmap='gray',
vmin=mn,
vmax=mx)
tims.append(tim)
keepims.append(im)
ims = keepims
print('Computing resampling...')
# save resampling params
for tim in tims:
wcs = tim.sip_wcs
subh, subw = tim.shape
subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh)
tim.subwcs = subwcs
try:
Yo, Xo, Yi, Xi, rims = resample_with_wcs(targetwcs, subwcs, [], 2)
except OverlapError:
print('No overlap')
continue
if len(Yo) == 0:
continue
tim.resamp = (Yo, Xo, Yi, Xi)
print('Creating coadds...')
# Produce per-band coadds, for plots
coimgs = []
cons = []
for ib, band in enumerate(bands):
coimg = np.zeros((H, W), np.float32)
con = np.zeros((H, W), np.uint8)
for tim in tims:
if tim.band != band:
continue
(Yo, Xo, Yi, Xi) = tim.resamp
if len(Yo) == 0:
continue
nn = (tim.getInvvar()[Yi, Xi] > 0)
coimg[Yo, Xo] += tim.getImage()[Yi, Xi] * nn
con[Yo, Xo] += nn
# print
# print 'tim', tim.name
# print 'number of resampled pix:', len(Yo)
# reim = np.zeros_like(coimg)
# ren = np.zeros_like(coimg)
# reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn
# ren[Yo,Xo] = nn
# print 'number of resampled pix with positive invvar:', ren.sum()
# plt.clf()
# plt.subplot(2,2,1)
# mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]]
# print 'Percentiles:', mn,mx
# dimshow(reim, vmin=mn, vmax=mx)
# plt.colorbar()
# plt.subplot(2,2,2)
# dimshow(con)
# plt.colorbar()
# plt.subplot(2,2,3)
# dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1])
# plt.colorbar()
# plt.subplot(2,2,4)
# plt.hist(reim.ravel(), 100, histtype='step', color='b')
# plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r')
# plt.suptitle('%s: %s' % (band, tim.name))
# ps.savefig()
coimg /= np.maximum(con, 1)
coimgs.append(coimg)
cons.append(con)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ps.savefig()
plt.clf()
for i, b in enumerate(bands):
plt.subplot(2, 2, i + 1)
dimshow(cons[i], ticks=False)
plt.title('%s band' % b)
plt.colorbar()
plt.suptitle('Number of exposures')
ps.savefig()
print('Grabbing SDSS sources...')
bandlist = [b for b in bands]
cat, T = get_sdss_sources(bandlist, targetwcs)
# record coordinates in target brick image
ok, T.tx, T.ty = targetwcs.radec2pixelxy(T.ra, T.dec)
T.tx -= 1
T.ty -= 1
T.itx = np.clip(np.round(T.tx).astype(int), 0, W - 1)
T.ity = np.clip(np.round(T.ty).astype(int), 0, H - 1)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5)
plt.axis(ax)
plt.title('SDSS sources')
ps.savefig()
print('Detmaps...')
# Render the detection maps
detmaps = dict([(b, np.zeros((H, W), np.float32)) for b in bands])
detivs = dict([(b, np.zeros((H, W), np.float32)) for b in bands])
for tim in tims:
iv = tim.getInvvar()
psfnorm = 1. / (2. * np.sqrt(np.pi) * tim.psf_sigma)
detim = tim.getImage().copy()
detim[iv == 0] = 0.
detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2
detsig1 = tim.sig1 / psfnorm
subh, subw = tim.shape
detiv = np.zeros((subh, subw), np.float32) + (1. / detsig1**2)
detiv[iv == 0] = 0.
(Yo, Xo, Yi, Xi) = tim.resamp
detmaps[tim.band][Yo, Xo] += detiv[Yi, Xi] * detim[Yi, Xi]
detivs[tim.band][Yo, Xo] += detiv[Yi, Xi]
rtn = dict()
for k in [
'T', 'coimgs', 'cons', 'detmaps', 'detivs', 'targetrd', 'pixscale',
'targetwcs', 'W', 'H', 'bands', 'tims', 'ps', 'brick', 'cat'
]:
rtn[k] = locals()[k]
return rtn
def main():
# ps = PlotSequence('pro')
# star_profiles(ps)
# sys.exit(0)
#survey_dir = '/project/projectdirs/desiproc/dr3'
#survey = LegacySurveyData(survey_dir=survey_dir)
survey = LegacySurveyData()
ralo,rahi = 240,245
declo,dechi = 5, 12
ps = PlotSequence('comp')
bands = 'grz'
ccdfn = 'ccds-forced.fits'
if not os.path.exists(ccdfn):
ccds = survey.get_annotated_ccds()
ccds.cut((ccds.ra > ralo) * (ccds.ra < rahi) *
(ccds.dec > declo) * (ccds.dec < dechi))
print(len(ccds), 'CCDs')
ccds.path = np.array([os.path.join(#'dr3',
'forced', ('%08i' % e)[:5], '%08i' % e, 'decam-%08i-%s-forced.fits' % (e, n.strip()))
for e,n in zip(ccds.expnum, ccds.ccdname)])
I, = np.nonzero([os.path.exists(fn) for fn in ccds.path])
print(len(I), 'CCDs with forced photometry')
ccds.cut(I)
#ccds = ccds[:500]
#e,I = np.unique(ccds.expnum, return_index=True)
#print(len(I), 'unique exposures')
#ccds.cut(I)
FF = read_forcedphot_ccds(ccds, survey)
FF.writeto('forced-all-matches.fits')
# - z band -- no trend w/ PS1 mag (brighter-fatter)
ccds.writeto(ccdfn)
ccdfn2 = 'ccds-forced-2.fits'
if not os.path.exists(ccdfn2):
ccds = fits_table(ccdfn)
# Split into brighter/fainter halves
FF = fits_table('forced-all-matches.fits')
print(len(FF), 'forced measurements')
FF.cut(FF.masked == False)
print(len(FF), 'forced measurements not masked')
ccds.brightest_mdiff = np.zeros(len(ccds))
ccds.brightest_mscatter = np.zeros(len(ccds))
ccds.bright_mdiff = np.zeros(len(ccds))
ccds.bright_mscatter = np.zeros(len(ccds))
ccds.faint_mdiff = np.zeros(len(ccds))
ccds.faint_mscatter = np.zeros(len(ccds))
for iccd in range(len(ccds)):
I = np.flatnonzero(FF.iforced == iccd)
if len(I) == 0:
continue
if len(I) < 10:
continue
F = FF[I]
b = np.percentile(F.psmag, 10)
m = np.median(F.psmag)
print(len(F), 'matches for CCD', iccd, 'median mag', m, '10th pct', b)
J = np.flatnonzero(F.psmag < b)
diff = F.mag[J] - F.psmag[J]
ccds.brightest_mdiff[iccd] = np.median(diff)
ccds.brightest_mscatter[iccd] = (np.percentile(diff, 84) -
np.percentile(diff, 16))/2.
J = np.flatnonzero(F.psmag < m)
diff = F.mag[J] - F.psmag[J]
ccds.bright_mdiff[iccd] = np.median(diff)
ccds.bright_mscatter[iccd] = (np.percentile(diff, 84) -
np.percentile(diff, 16))/2.
J = np.flatnonzero(F.psmag > m)
diff = F.mag[J] - F.psmag[J]
ccds.faint_mdiff[iccd] = np.median(diff)
ccds.faint_mscatter[iccd] = (np.percentile(diff, 84) -
np.percentile(diff, 16))/2.
ccds.writeto(ccdfn2)
ccds = fits_table(ccdfn2)
plt.clf()
plt.hist(ccds.nforced, bins=100)
plt.title('nforced')
ps.savefig()
plt.clf()
plt.hist(ccds.nmatched, bins=100)
plt.title('nmatched')
ps.savefig()
#ccds.cut(ccds.nmatched >= 150)
ccds.cut(ccds.nmatched >= 50)
print('Cut to', len(ccds), 'with >50 matched')
ccds.cut(ccds.photometric)
print('Cut to', len(ccds), 'photometric')
neff = 1. / ccds.psfnorm_mean**2
# Narcsec is in arcsec**2
narcsec = neff * ccds.pixscale_mean**2
# to arcsec
narcsec = np.sqrt(narcsec)
# Correction factor to get back to equivalent of Gaussian sigma
narcsec /= (2. * np.sqrt(np.pi))
# Conversion factor to FWHM (2.35)
narcsec *= 2. * np.sqrt(2. * np.log(2.))
ccds.psfsize = narcsec
for band in bands:
I = np.flatnonzero((ccds.filter == band)
* (ccds.photometric) * (ccds.blacklist_ok))
mlo,mhi = -0.01, 0.05
plt.clf()
plt.plot(ccds.ccdzpt[I],
ccds.exptime[I], 'k.', alpha=0.1)
J = np.flatnonzero((ccds.filter == band) * (ccds.photometric == False))
plt.plot(ccds.ccdzpt[J],
ccds.exptime[J], 'r.', alpha=0.1)
plt.xlabel('Zeropoint (mag)')
plt.ylabel('exptime')
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
plt.clf()
plt.plot(ccds.ccdzpt[I],
np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
plt.xlabel('Zeropoint (mag)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
#plt.axis([0, mxsee, mlo,mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
plt.clf()
plt.plot(ccds.ccdzpt[I], ccds.psfsize[I], 'k.', alpha=0.1)
plt.xlabel('Zeropoint (mag)')
plt.ylabel('PSF size (arcsec)')
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
# plt.clf()
# plt.plot(ccds.avsky[I],
# np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
# plt.xlabel('avsky')
# plt.ylabel('DECaLS PSF - PS1 (mag)')
# plt.axhline(0, color='k', alpha=0.2)
# plt.title('DR3: EDR region, Forced phot: %s band' % band)
# ps.savefig()
#
# plt.clf()
# plt.plot(ccds.meansky[I],
# np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
# plt.xlabel('meansky')
# plt.ylabel('DECaLS PSF - PS1 (mag)')
# plt.axhline(0, color='k', alpha=0.2)
# plt.title('DR3: EDR region, Forced phot: %s band' % band)
# ps.savefig()
plt.clf()
lo,hi = (-0.02, 0.05)
ha = dict(bins=50, histtype='step', range=(lo,hi))
n,b,p1 = plt.hist(ccds.brightest_mdiff[I], color='r', **ha)
n,b,p2 = plt.hist(ccds.bright_mdiff[I], color='g', **ha)
n,b,p3 = plt.hist(ccds.faint_mdiff[I], color='b', **ha)
plt.legend((p1[0],p2[0],p3[0]), ('Brightest 10%', 'Brightest 50%',
'Faintest 50%'))
plt.xlabel('DECaLS PSF - PS1 (mag)')
plt.ylabel('Number of CCDs')
plt.title('DR3: EDR region, Forced phot: %s band' % band)
plt.xlim(lo,hi)
ps.savefig()
for band in bands:
I = np.flatnonzero(ccds.filter == band)
mxsee = 4.
mlo,mhi = -0.01, 0.05
plt.clf()
plt.plot(np.clip(ccds.psfsize[I], 0, mxsee),
np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
# for p in [1,2,3]:
# J = np.flatnonzero(ccds.tilepass[I] == p)
# if len(J):
# plt.plot(np.clip(ccds.psfsize[I[J]], 0, mxsee),
# np.clip(ccds.mdiff[I[J]], mlo,mhi), '.', color='rgb'[p-1], alpha=0.2)
#plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.')
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
plt.axis([0, mxsee, mlo,mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
# Group by exposure
for band in bands:
I = np.flatnonzero((ccds.filter == band)
* (ccds.photometric) * (ccds.blacklist_ok))
E,J = np.unique(ccds.expnum[I], return_index=True)
print(len(E), 'unique exposures in', band)
exps = ccds[I[J]]
print(len(exps), 'unique exposures in', band)
assert(len(np.unique(exps.expnum)) == len(exps))
exps.ddiff = np.zeros(len(exps))
exps.dsize = np.zeros(len(exps))
exps.nccds = np.zeros(len(exps), int)
exps.brightest_ddiff = np.zeros(len(exps))
exps.bright_ddiff = np.zeros(len(exps))
exps.faint_ddiff = np.zeros(len(exps))
for iexp,exp in enumerate(exps):
J = np.flatnonzero(ccds.expnum[I] == exp.expnum)
J = I[J]
print(len(J), 'CCDs in exposure', exp.expnum)
exps.brightest_mdiff[iexp] = np.median(ccds.brightest_mdiff[J])
exps.bright_mdiff[iexp] = np.median(ccds.bright_mdiff[J])
exps.faint_mdiff[iexp] = np.median(ccds.faint_mdiff[J])
exps.brightest_ddiff[iexp] = (
np.percentile(ccds.brightest_mdiff[J], 84) -
np.percentile(ccds.brightest_mdiff[J], 16))/2.
exps.bright_ddiff[iexp] = (
np.percentile(ccds.bright_mdiff[J], 84) -
np.percentile(ccds.bright_mdiff[J], 16))/2.
exps.faint_ddiff[iexp] = (
np.percentile(ccds.faint_mdiff[J], 84) -
np.percentile(ccds.faint_mdiff[J], 16))/2.
exps.mdiff[iexp] = np.median(ccds.mdiff[J])
exps.ddiff[iexp] = (np.percentile(ccds.mdiff[J], 84) - np.percentile(ccds.mdiff[J], 16))/2.
exps.psfsize[iexp] = np.median(ccds.psfsize[J])
exps.dsize[iexp] = (np.percentile(ccds.psfsize[J], 84) - np.percentile(ccds.psfsize[J], 16))/2.
exps.nccds[iexp] = len(J)
mxsee = 4.
mlo,mhi = -0.01, 0.05
exps.cut(exps.nccds >= 10)
plt.clf()
plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.mdiff, mlo,mhi), yerr=exps.ddiff,
#xerr=exps.dsize,
fmt='.', color='k')
# plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.brightest_mdiff, mlo,mhi),
# yerr=exps.brightest_ddiff, fmt='r.')
# plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.bright_mdiff, mlo,mhi),
# yerr=exps.bright_ddiff, fmt='g.')
# plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.faint_mdiff, mlo,mhi),
# yerr=exps.faint_ddiff, fmt='b.')
# plt.plot(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.brightest_mdiff, mlo,mhi), 'r.')
# plt.plot(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.bright_mdiff, mlo,mhi), 'g.')
# plt.plot(np.clip(exps.psfsize, 0, mxsee),
# np.clip(exps.faint_mdiff, mlo,mhi), 'b.')
#plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.')
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
plt.axis([0, mxsee, mlo,mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
plt.clf()
p1 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.brightest_mdiff, mlo,mhi), 'r.', alpha=0.5)
p2 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.bright_mdiff, mlo,mhi), 'g.', alpha=0.5)
p3 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
np.clip(exps.faint_mdiff, mlo,mhi), 'b.', alpha=0.5)
plt.legend((p1[0],p2[0],p3[0]), ('Brightest 10%', 'Brightest 50%',
'Faintest 50%'))
plt.xlabel('PSF size (arcsec)')
plt.ylabel('DECaLS PSF - PS1 (mag)')
plt.axhline(0, color='k', alpha=0.2)
plt.axis([0, mxsee, mlo,mhi])
plt.title('DR3: EDR region, Forced phot: %s band' % band)
ps.savefig()
J = np.argsort(-exps.mdiff)
for j in J:
print(' Photometric diff', exps.mdiff[j], 'PSF size', exps.psfsize[j], 'expnum', exps.expnum[j])
sys.exit(0)
def showSipSolutions(srcs, wcs0, andDir, x0, y0, W, H, filterName, plotPrefix):
'''
srcs: afw Catalog of sources
wcs0: original WCS
andDir: astrometry_net_data directory
'''
imargs = dict(imageSize=(W, H), filterName=filterName, x0=x0, y0=y0)
# Set up astrometry_net_data
os.environ['ASTROMETRY_NET_DATA_DIR'] = andDir
andConfig = measAstrom.AstrometryNetDataConfig()
fn = os.path.join(andDir, 'andConfig.py')
andConfig.load(fn)
# Set up meas_astrom
conf = measAstrom.ANetBasicAstrometryConfig(sipOrder=4)
ast = measAstrom.ANetBasicAstrometryTask(conf,
andConfig,
logLevel=Log.DEBUG)
# What reference sources are in the original WCS
refs = ast.getReferenceSourcesForWcs(wcs0, **imargs)
print('Got', len(refs), 'reference objects for initial WCS')
# How does a straight TAN solution look?
conf2 = measAstrom.ANetBasicAstrometryConfig(sipOrder=4,
calculateSip=False)
ast2 = measAstrom.ANetBasicAstrometryTask(conf2,
andConfig,
logLevel=Log.DEBUG)
solve = ast2.determineWcs2(srcs, **imargs)
tanwcs = solve.tanWcs
# How about if we fit a SIP WCS using the *original* WCS?
wcs2 = ast.getSipWcsFromWcs(wcs0, (W, H), x0=x0, y0=y0)
# (We determineWcs() for a SIP solution below...)
# Make some plots in pixel space by pushing ref sources through WCSes
rx0, ry0 = [], []
rx2, ry2 = [], []
rx3, ry3 = [], []
for src in refs:
xy = wcs0.skyToPixel(src.getCoord())
rx0.append(xy[0])
ry0.append(xy[1])
xy = tanwcs.skyToPixel(src.getCoord())
rx2.append(xy[0])
ry2.append(xy[1])
xy = wcs2.skyToPixel(src.getCoord())
rx3.append(xy[0])
ry3.append(xy[1])
rx0 = np.array(rx0)
ry0 = np.array(ry0)
rx2 = np.array(rx2)
ry2 = np.array(ry2)
rx3 = np.array(rx3)
ry3 = np.array(ry3)
x = np.array([src.getX() for src in srcs])
y = np.array([src.getY() for src in srcs])
from astrometry.libkd.spherematch import match
from astrometry.util.plotutils import plothist, PlotSequence
ps = PlotSequence(plotPrefix)
# Match up various sources...
R = 2.
II, d = match(np.vstack((x, y)).T, np.vstack((rx0, ry0)).T, R)
I = II[:, 0]
J = II[:, 1]
pa = dict(range=((-R, R), (-R, R)))
plt.clf()
plothist(x[I] - rx0[J], y[I] - ry0[J], 200, **pa)
plt.title('Source positions - Reference positions (initial WCS)')
plt.xlabel('delta-X (pixels)')
plt.ylabel('delta-Y (pixels)')
ps.savefig()
II, d = match(np.vstack((x, y)).T, np.vstack((rx2, ry2)).T, R)
I = II[:, 0]
J = II[:, 1]
plt.clf()
plothist(x[I] - rx2[J], y[I] - ry2[J], 200, **pa)
plt.title('Source positions - Reference positions (TAN WCS)')
plt.xlabel('delta-X (pixels)')
plt.ylabel('delta-Y (pixels)')
ps.savefig()
II, d = match(np.vstack((x, y)).T, np.vstack((rx3, ry3)).T, R)
I = II[:, 0]
J = II[:, 1]
plt.clf()
plothist(x[I] - rx3[J], y[I] - ry3[J], 200, **pa)
plt.title('Source positions - Reference positions (SIP WCS #2)')
plt.xlabel('delta-X (pixels)')
plt.ylabel('delta-Y (pixels)')
ps.savefig()
II, d = match(np.vstack((rx0, ry0)).T, np.vstack((rx3, ry3)).T, R)
I = II[:, 0]
J = II[:, 1]
plt.clf()
plothist(rx0[I] - rx3[J], ry0[I] - ry3[J], 200, **pa)
plt.title(
'Reference positions (Original WCS) - Reference positions (SIP WCS #2)'
)
plt.xlabel('delta-X (pixels)')
plt.ylabel('delta-Y (pixels)')
ps.savefig()
matches = solve.tanMatches
msx, msy = [], []
mrx, mry = [], []
for m in matches:
ref, src = m.first, m.second
xy = tanwcs.skyToPixel(ref.getCoord())
mrx.append(xy[0])
mry.append(xy[1])
msx.append(src.getX())
msy.append(src.getY())
plt.clf()
plt.plot(x, y, 'r.')
plt.plot(msx, msy, 'o', mec='r')
plt.plot(rx0, ry0, 'g.')
plt.plot(mrx, mry, 'gx')
plt.title('TAN matches')
ps.savefig()
# Get SIP solution (4th order)
solve = ast.determineWcs2(srcs, **imargs)
wcs1 = solve.sipWcs
matches = solve.sipMatches
msx, msy = [], []
mrx, mry = [], []
for m in matches:
ref, src = m.first, m.second
xy = tanwcs.skyToPixel(ref.getCoord())
mrx.append(xy[0])
mry.append(xy[1])
msx.append(src.getX())
msy.append(src.getY())
plt.clf()
plt.plot(x, y, 'r.')
plt.plot(msx, msy, 'o', mec='r')
plt.plot(rx0, ry0, 'g.')
plt.plot(mrx, mry, 'gx')
plt.title('SIP matches')
ps.savefig()
rx1, ry1 = [], []
for src in refs:
xy = wcs1.skyToPixel(src.getCoord())
rx1.append(xy[0])
ry1.append(xy[1])
rx1 = np.array(rx1)
ry1 = np.array(ry1)
plt.clf()
plt.plot(x, y, 'o', mec='r', mfc='none')
plt.plot(rx0, ry0, 'bx')
plt.plot(rx1, ry1, 'g+')
plt.plot(rx2, ry2, 'mx')
plt.plot(rx3, ry3, 'r+')
ps.savefig()
plt.axis([x0, x0 + 500, y0, y0 + 500])
ps.savefig()
II, d = match(np.vstack((x, y)).T, np.vstack((rx1, ry1)).T, R)
I = II[:, 0]
J = II[:, 1]
plt.clf()
plothist(x[I] - rx1[J], y[I] - ry1[J], 200, **pa)
plt.title('Source positions - Reference positions (SIP WCS)')
plt.xlabel('delta-X (pixels)')
plt.ylabel('delta-Y (pixels)')
ps.savefig()
def main():
import optparse
from astrometry.util.plotutils import PlotSequence
from astrometry.util.util import Tan
parser = optparse.OptionParser(usage='%prog [options] incat.fits out.fits')
parser.add_option('-r', '--ralo', dest='ralo', type=float,
help='Minimum RA')
parser.add_option('-R', '--rahi', dest='rahi', type=float,
help='Maximum RA')
parser.add_option('-d', '--declo', dest='declo', type=float,
help='Minimum Dec')
parser.add_option('-D', '--dechi', dest='dechi', type=float,
help='Maximum Dec')
parser.add_option('-b', '--band', dest='bands', action='append', type=int,
default=[], help='WISE band to photometer (default: 1,2)')
parser.add_option('-u', '--unwise', dest='unwise_dir',
default='unwise-coadds',
help='Directory containing unWISE coadds')
parser.add_option('--no-ceres', dest='ceres', action='store_false',
default=True,
help='Use scipy lsqr rather than Ceres Solver?')
parser.add_option('--ceres-block', '-B', dest='ceresblock', type=int,
default=8,
help='Ceres image block size (default: %default)')
parser.add_option('--plots', dest='plots',
default=False, action='store_true')
parser.add_option('--save-fits', dest='save_fits',
default=False, action='store_true')
# parser.add_option('--ellipses', action='store_true',
# help='Assume catalog shapes are ellipse descriptions (not r,ab,phi)')
# parser.add_option('--ra', help='Center RA')
# parser.add_option('--dec', help='Center Dec')
# parser.add_option('--width', help='Degrees width (in RA*cos(Dec))')
# parser.add_option('--height', help='Degrees height (Dec)')
opt, args = parser.parse_args()
if len(args) != 2:
parser.print_help()
sys.exit(-1)
if len(opt.bands) == 0:
opt.bands = [1, 2]
# Allow specifying bands like "123"
bb = []
for band in opt.bands:
for s in str(band):
bb.append(int(s))
opt.bands = bb
print('Bands', opt.bands)
ps = None
if opt.plots:
ps = PlotSequence('unwise')
infn, outfn = args
T = fits_table(infn)
print('Read', len(T), 'sources from', infn)
if opt.declo is not None:
T.cut(T.dec >= opt.declo)
if opt.dechi is not None:
T.cut(T.dec <= opt.dechi)
# Let's be a bit smart about RA wrap-around. Compute the 'center'
# of the RA points, use the cross product against that to define
# inequality (clockwise-of).
r = np.deg2rad(T.ra)
x = np.mean(np.cos(r))
y = np.mean(np.sin(r))
rr = np.hypot(x, y)
x /= rr
y /= rr
midra = np.rad2deg(np.arctan2(y, x))
midra += 360. * (midra < 0)
xx = np.cos(r)
yy = np.sin(r)
T.cross = x * yy - y * xx
minra = T.ra[np.argmin(T.cross)]
maxra = T.ra[np.argmax(T.cross)]
if opt.ralo is not None:
r = np.deg2rad(opt.ralo)
xx = np.cos(r)
yy = np.sin(r)
crosscut = x * yy - y * xx
T.cut(T.cross >= crosscut)
print('Cut to', len(T), 'with RA >', opt.ralo)
if opt.rahi is not None:
r = np.deg2rad(opt.rahi)
xx = np.cos(r)
yy = np.sin(r)
crosscut = x * yy - y * xx
T.cut(T.cross <= crosscut)
print('Cut to', len(T), 'with RA <', opt.rahi)
if opt.declo is None:
opt.declo = T.dec.min()
if opt.dechi is None:
opt.dechi = T.dec.max()
if opt.ralo is None:
opt.ralo = T.ra[np.argmin(T.cross)]
if opt.rahi is None:
opt.rahi = T.ra[np.argmax(T.cross)]
T.delete_column('cross')
print('RA range:', opt.ralo, opt.rahi)
print('Dec range:', opt.declo, opt.dechi)
x = np.mean([np.cos(np.deg2rad(r)) for r in (opt.ralo, opt.rahi)])
y = np.mean([np.sin(np.deg2rad(r)) for r in (opt.ralo, opt.rahi)])
midra = np.rad2deg(np.arctan2(y, x))
midra += 360. * (midra < 0)
middec = (opt.declo + opt.dechi) / 2.
print('RA,Dec center:', midra, middec)
pixscale = 2.75 / 3600.
H = (opt.dechi - opt.declo) / pixscale
dra = 2. * min(np.abs(midra - opt.ralo), np.abs(midra - opt.rahi))
W = dra * np.cos(np.deg2rad(middec)) / pixscale
margin = 5
W = int(W) + margin * 2
H = int(H) + margin * 2
print('W,H', W, H)
targetwcs = Tan(midra, middec, (W + 1) / 2., (H + 1) / 2.,
-pixscale, 0., 0., pixscale, float(W), float(H))
#print('Target WCS:', targetwcs)
ra0, dec0 = targetwcs.pixelxy2radec(0.5, 0.5)
ra1, dec1 = targetwcs.pixelxy2radec(W + 0.5, H + 0.5)
roiradecbox = [ra0, ra1, dec0, dec1]
#print('ROI RA,Dec box', roiradecbox)
tiles = unwise_tiles_touching_wcs(targetwcs)
print('Cut to', len(tiles), 'unWISE tiles')
disable_galaxy_cache()
cols = T.get_columns()
all_ptsrcs = not('type' in cols)
if not all_ptsrcs:
assert('shapeexp' in cols)
assert('shapedev' in cols)
assert('fracdev' in cols)
wanyband = 'w'
print('Creating Tractor catalog...')
cat = []
for i, t in enumerate(T):
pos = RaDecPos(t.ra, t.dec)
flux = NanoMaggies(**{wanyband: 1.})
if all_ptsrcs:
cat.append(PointSource(pos, flux))
continue
tt = t.type.strip()
if tt in ['PTSRC', 'STAR', 'S']:
cat.append(PointSource(pos, flux))
elif tt in ['EXP', 'E']:
shape = EllipseE(*t.shapeexp)
cat.append(ExpGalaxy(pos, flux, shape))
elif tt in ['DEV', 'D']:
shape = EllipseE(*t.shapedev)
cat.append(DevGalaxy(pos, flux, shape))
elif tt in ['COMP', 'C']:
eshape = EllipseE(*t.shapeexp)
dshape = EllipseE(*t.shapedev)
cat.append(FixedCompositeGalaxy(pos, flux, t.fracdev,
eshape, dshape))
else:
print('Did not understand row', i, 'of input catalog:')
t.about()
assert(False)
W = unwise_forcedphot(cat, tiles, roiradecbox=roiradecbox,
bands=opt.bands, unwise_dir=opt.unwise_dir,
use_ceres=opt.ceres, ceres_block=opt.ceresblock,
save_fits=opt.save_fits, ps=ps)
W.writeto(outfn)
print s
print s.getProfile()
s.sersicindex.setValue(4.0)
print s.getProfile()
d = DevGalaxy(s.pos, s.brightness, s.shape)
print d
print d.getProfile()
# Extrapolation!
# s.sersicindex.setValue(0.5)
# print s.getProfile()
ps = PlotSequence('ser')
# example PSF (from WISE W1 fit)
w = np.array([ 0.77953706, 0.16022146, 0.06024237])
mu = np.array([[-0.01826623, -0.01823262],
[-0.21878855, -0.0432496 ],
[-0.83365747, -0.13039277]])
sigma = np.array([[[ 7.72925584e-01, 5.23305564e-02],
[ 5.23305564e-02, 8.89078473e-01]],
[[ 9.84585869e+00, 7.79378820e-01],
[ 7.79378820e-01, 8.84764455e+00]],
[[ 2.02664489e+02, -8.16667434e-01],
[ -8.16667434e-01, 1.87881670e+02]]])
psf = GaussianMixturePSF(w, mu, sigma)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--plots', action='store_true')
parser.add_argument('--brick', help='Brick name to run')
parser.add_argument(
'--input-dir',
default='/global/projecta/projectdirs/cosmo/work/legacysurvey/dr7')
#/global/cscratch1/sd/desiproc/dr7out')
parser.add_argument('--survey-dir',
default='/global/cscratch1/sd/dstn/dr7-depthcut')
parser.add_argument('--output-dir',
default='/global/cscratch1/sd/dstn/bright')
opt = parser.parse_args()
plots = opt.plots
ps = PlotSequence('bright')
brickname = opt.brick
insurvey = LegacySurveyData(opt.input_dir, cache_dir=opt.survey_dir)
outsurvey = LegacySurveyData(opt.output_dir, output_dir=opt.output_dir)
bfn = insurvey.find_file('blobmap', brick=brickname)
print('Found blob map', bfn)
blobs = fitsio.read(bfn)
h, w = blobs.shape
brick = insurvey.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brick)
radius = np.sqrt(2.) * 0.25 * 1.01
neighbors = insurvey.get_bricks_near(brick.ra, brick.dec, radius)
print(len(neighbors), 'bricks nearby')
def showbool(X):
d = downsample_max(X, 8)
h, w = X.shape
plt.imshow(d,
interpolation='nearest',
origin='lower',
vmin=0,
vmax=1,
extent=[0, w, 0, h],
cmap='gray')
brightblobs = set()
for nb in neighbors:
if nb.brickname == brickname:
# ignore myself!
continue
print('Neighbor:', nb.brickname)
mfn = insurvey.find_file('maskbits', brick=nb.brickname)
if not os.path.exists(mfn):
print('No maskbits file:', mfn)
continue
maskbits = fitsio.read(mfn)
bright = ((maskbits & MASKBITS['BRIGHT']) > 0)
print(np.sum(bright > 0), 'BRIGHT pixels set')
primary = (maskbits & MASKBITS['NPRIMARY'] == 0)
print(np.sum(primary), 'PRIMARY pixels set')
edge = binary_dilation(primary, structure=np.ones((3, 3), bool))
edge = edge * np.logical_not(primary)
brightedge = edge & bright
if plots:
plt.clf()
showbool(bright)
plt.title('bright: brick %s' % nb.brickname)
ps.savefig()
# plt.clf()
# showbool(primary)
# plt.title('PRIMARY, brick %s' % nb.brickname)
# ps.savefig()
#
# plt.clf()
# showbool(edge)
# plt.title('boundary, brick %s' % nb.brickname)
# ps.savefig()
plt.clf()
showbool(brightedge)
plt.title('bright at edge, brick %s' % nb.brickname)
ps.savefig()
nwcs = wcs_for_brick(nb)
yy, xx = np.nonzero(brightedge)
print(len(yy), 'bright edge pixels')
if len(yy) == 0:
continue
rr, dd = nwcs.pixelxy2radec(xx + 1, yy + 1)
print('RA range', rr.min(), rr.max(), 'vs brick', brick.ra1, brick.ra2)
print('Dec range', dd.min(), dd.max(), 'vs brick', brick.dec1,
brick.dec2)
# Find pixels that are within this brick's unique area
I, = np.nonzero((rr >= brick.ra1) * (rr <= brick.ra2) *
(dd >= brick.dec1) * (dd <= brick.dec2))
if plots:
plt.clf()
plt.plot(
[brick.ra1, brick.ra1, brick.ra2, brick.ra2, brick.ra1],
[brick.dec1, brick.dec2, brick.dec2, brick.dec1, brick.dec1],
'b-')
plt.plot(rr, dd, 'k.')
plt.plot(rr[I], dd[I], 'r.')
plt.title('Bright pixels from %s' % nb.brickname)
ps.savefig()
if len(I) == 0:
print('No edge pixels touch')
#plt.plot(br,bd, 'b-')
continue
#print('Edge pixels touch!')
#plt.plot(br,bd, 'r-', zorder=20)
ok, x, y = brickwcs.radec2pixelxy(rr[I], dd[I])
x = np.round(x).astype(int) - 1
y = np.round(y).astype(int) - 1
print('Pixel ranges X', x.min(), x.max(), 'Y', y.min(), y.max())
assert (np.all((x >= 0) * (x < w) * (y >= 0) * (y < h)))
print('Adding blobs:', np.unique(blobs[y, x]))
brightblobs.update(blobs[y, x])
print('Blobs touching bright pixels:', brightblobs)
print()
brightblobs.discard(-1)
if len(brightblobs) == 0:
print('No neighboring bright blobs to update!')
return
print('Updating', len(brightblobs), 'blobs:', brightblobs)
tmap = np.zeros(blobs.max() + 2, bool)
for b in brightblobs:
tmap[b + 1] = True
touching = tmap[blobs + 1]
if plots:
plt.clf()
showbool(touching)
plt.title('Blobs touching bright, brick %s' % brickname)
ps.savefig()
mfn = insurvey.find_file('maskbits', brick=brickname)
maskbits, hdr = fitsio.read(mfn, header=True)
updated = maskbits | (MASKBITS['BRIGHT'] * touching)
if np.all(maskbits == updated):
print('No bits updated! (Bright stars were already masked)')
return
maskbits = updated
if plots:
plt.clf()
showbool((maskbits & MASKBITS['BRIGHT']) > 0)
plt.title('New maskbits map for BRIGHT, brick %s' % brickname)
ps.savefig()
with outsurvey.write_output('maskbits', brick=brickname) as out:
out.fits.write(maskbits, hdr=hdr)
tfn = insurvey.find_file('tractor', brick=brickname)
phdr = fitsio.read_header(tfn, ext=0)
hdr = fitsio.read_header(tfn, ext=1)
T = fits_table(tfn)
print('Read', len(T), 'sources')
print('Bright:', Counter(T.brightstarinblob))
iby = np.clip(np.round(T.by), 0, h - 1).astype(int)
ibx = np.clip(np.round(T.bx), 0, w - 1).astype(int)
if plots:
before = np.flatnonzero(T.brightstarinblob)
T.brightstarinblob |= touching[iby, ibx]
print('Bright:', Counter(T.brightstarinblob))
# yuck -- copy the TUNIT headers from input to output.
units = [
hdr.get('TUNIT%i' % (i + 1), '') for i in range(len(T.get_columns()))
]
if plots:
plt.clf()
showbool((maskbits & MASKBITS['BRIGHT']) > 0)
ax = plt.axis()
after = np.flatnonzero(T.brightstarinblob)
plt.plot(T.bx[before], T.by[before], 'gx')
plt.plot(T.bx[after], T.by[after], 'r.')
plt.axis(ax)
plt.title('sources with brightstarinblob, brick %s' % brickname)
ps.savefig()
with outsurvey.write_output('tractor', brick=brickname) as out:
T.writeto(None,
fits_object=out.fits,
primheader=phdr,
header=hdr,
units=units)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--name1', help='Name for first data set')
parser.add_argument('--name2', help='Name for second data set')
parser.add_argument('--plot-prefix', default='compare',
help='Prefix for plot filenames; default "%default"')
parser.add_argument('--match', default=1.0,
help='Astrometric cross-match distance in arcsec')
parser.add_argument('dir1', help='First directory to compare')
parser.add_argument('dir2', help='Second directory to compare')
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = 'Comparing %s to %s' % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile('tractor-.*.fits')
cat1,cat2 = [],[]
for basedir,cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath,dirnames,filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print('Skipping', fn, 'due to filename')
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), 'from', fn)
cat.append(t)
cat1 = merge_tables(cat1, columns='fillzero')
cat2 = merge_tables(cat2, columns='fillzero')
print('Total of', len(cat1), 'from', name1)
print('Total of', len(cat2), 'from', name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)
cat1.cut((cat1.decam_anymask[:,1] == 0) *
(cat1.decam_anymask[:,2] == 0) *
(cat1.decam_anymask[:,4] == 0))
cat2.cut((cat2.decam_anymask[:,1] == 0) *
(cat2.decam_anymask[:,2] == 0) *
(cat2.decam_anymask[:,4] == 0))
print('Total of', len(cat1), 'unmasked from', name1)
print('Total of', len(cat2), 'unmasked from', name2)
I,J,d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match/3600.,
nearest=True)
print(len(I), 'matched')
plt.clf()
plt.hist(d * 3600., 100)
plt.xlabel('Match distance (arcsec)')
plt.title(tt)
ps.savefig()
matched1 = cat1[I]
matched2 = cat2[J]
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
K = np.flatnonzero((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
print('Median mw_trans', band, 'is',
np.median(matched1.decam_mw_transmission[:,iband]))
plt.clf()
plt.errorbar(matched1.decam_flux[K,iband],
matched2.decam_flux[K,iband],
fmt='.', color=cc,
xerr=1./np.sqrt(matched1.decam_flux_ivar[K,iband]),
yerr=1./np.sqrt(matched2.decam_flux_ivar[K,iband]),
alpha=0.1,
)
plt.xlabel('%s flux: %s' % (name1, band))
plt.ylabel('%s flux: %s' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.)
plt.axis([-100, 1000, -100, 1000])
plt.title(tt)
ps.savefig()
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
K = np.flatnonzero(good)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
P = np.flatnonzero(good * psf1 * psf2)
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1./iv1 + 1./iv2)
plt.clf()
plt.plot(mag1[K],
(matched2.decam_flux[K,iband] - matched1.decam_flux[K,iband]) / std[K],
'.', alpha=0.1, color=cc)
plt.plot(mag1[P],
(matched2.decam_flux[P,iband] - matched1.decam_flux[P,iband]) / std[P],
'.', alpha=0.1, color='k')
plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' % (name2, name1, band))
plt.xlabel('%s mag: %s' % (name1, band))
plt.axhline(0, color='k', alpha=0.5)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
plt.clf()
lp,lt = [],[]
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
#good = True
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1./iv1 + 1./iv2)
#std = np.hypot(std, 0.01)
G = np.flatnonzero(good * psf1 * psf2 *
np.isfinite(mag1) *
(mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band]))
n,b,p = plt.hist((matched2.decam_flux[G,iband] -
matched1.decam_flux[G,iband]) / std[G],
range=(-4, 4), bins=50, histtype='step', color=cc,
normed=True)
sig = (matched2.decam_flux[G,iband] -
matched1.decam_flux[G,iband]) / std[G]
print('Raw mean and std of points:', np.mean(sig), np.std(sig))
med = np.median(sig)
rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
print('Median and percentile-based sigma:', med, rsigma)
lp.append(p[0])
lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))
bins = []
gaussint = []
for blo,bhi in zip(b, b[1:]):
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
#bins.extend([blo,bhi])
#gaussint.extend([c,c])
bins.append((blo+bhi)/2.)
gaussint.append(c)
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.title(tt)
plt.xlabel('Flux difference / error (sigma)')
plt.axvline(0, color='k', alpha=0.1)
plt.ylim(0, 0.45)
plt.legend(lp, lt, loc='upper right')
ps.savefig()
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
plt.clf()
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
matched2.decam_flux[:,iband], matched2.decam_flux_ivar[:,iband])
meanmag = NanoMaggies.nanomaggiesToMag((
matched1.decam_flux[:,iband] + matched2.decam_flux[:,iband]) / 2.)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0) *
np.isfinite(mag1) * np.isfinite(mag2))
K = np.flatnonzero(good)
P = np.flatnonzero(good * psf1 * psf2)
plt.errorbar(mag1[K], mag2[K], fmt='.', color=cc,
xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s (mag)' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.)
plt.axis([24, 16, 24, 16])
plt.title(tt)
ps.savefig()
plt.clf()
plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt='.', color=cc,
xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -1, 1])
plt.title(tt)
ps.savefig()
magbins = np.arange(16, 24.001, 0.5)
plt.clf()
plt.plot(mag1[K], (mag2[K]-mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
'.', color=cc, alpha=0.1)
plt.plot(mag1[P], (mag2[P]-mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
plt.clf()
plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)
midmag = []
vals = np.zeros((len(magbins)-1, 5))
median_err1 = []
iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)
# FIXME -- should we do some stats after taking off the mean difference?
for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
midmag.append((mlo+mhi)/2.)
median_err1.append(np.median(magerr1[I]))
if len(I) == 0:
continue
# median and +- 1 sigma quantiles
ybin = y[I]
vals[bini,0] = np.percentile(ybin, 16)
vals[bini,1] = np.median(ybin)
vals[bini,2] = np.percentile(ybin, 84)
# +- 2 sigma quantiles
vals[bini,3] = np.percentile(ybin, 2.3)
vals[bini,4] = np.percentile(ybin, 97.7)
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
'vs expected for Gaussian;', len(ybin), 'points')
# if iqd > iqd_gauss:
# # What error adding in quadrature would you need to make the IQD match?
# err = median_err1[-1]
# target_err = err * (iqd / iqd_gauss)
# sys_err = np.sqrt(target_err**2 - err**2)
# print('--> add systematic error', sys_err)
# ~ Johan's cuts
mlo = 21.
mhi = dict(g=24., r=23.5, z=22.5)[band]
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
ybin = y[I]
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', mlo, mhi, 'band', band, ': IQD is factor',
iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points')
if iqd > iqd_gauss:
# What error adding in quadrature would you need to make
# the IQD match?
err = np.median(np.hypot(magerr1[I], magerr2[I]))
print('Median error (hypot):', err)
target_err = err * (iqd / iqd_gauss)
print('Target:', target_err)
sys_err = np.sqrt((target_err**2 - err**2) / 2.)
print('--> add systematic error', sys_err)
# check...
err_sys = np.hypot(np.hypot(magerr1, sys_err),
np.hypot(magerr2, sys_err))
ysys = (mag2 - mag1) / err_sys
ysys = ysys[I]
print('Resulting median error:', np.median(err_sys[I]))
iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
# Hmmm, this doesn't work... totally overshoots.
plt.errorbar(midmag, vals[:,1], fmt='o', color='b',
yerr=(vals[:,1]-vals[:,0], vals[:,2]-vals[:,1]),
capthick=3, zorder=20)
plt.errorbar(midmag, vals[:,1], fmt='o', color='b',
yerr=(vals[:,1]-vals[:,3], vals[:,4]-vals[:,1]),
capthick=2, zorder=20)
plt.axhline( 1., color='b', alpha=0.2)
plt.axhline(-1., color='b', alpha=0.2)
plt.axhline( 2., color='b', alpha=0.2)
plt.axhline(-2., color='b', alpha=0.2)
for mag,err,y in zip(midmag, median_err1, vals[:,3]):
if not np.isfinite(err):
continue
if y < -6:
continue
plt.text(mag, y-0.1, '%.3f' % err, va='top', ha='center', color='k',
fontsize=10)
plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axvline(21, color='k', alpha=0.3)
plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)
plt.axis([24.1, 16, -6, 6])
plt.title(tt)
ps.savefig()
#magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
if band == 'g':
magbins = [20, 24]
elif band == 'r':
magbins = [20, 23.5]
elif band == 'z':
magbins = [20, 22.5]
slo,shi = -5,5
plt.clf()
ha = dict(bins=25, range=(slo,shi), histtype='step', normed=True)
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
midmag = []
nn = []
rgbs = []
lt,lp = [],[]
for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
if len(I) == 0:
continue
ybin = y[I]
rgb = [0.,0.,0.]
rgb[0] = float(bini) / (len(magbins)-1)
rgb[2] = 1. - rgb[0]
n,b,p = plt.hist(ybin, color=rgb, **ha)
lt.append('mag %g to %g' % (mlo,mhi))
lp.append(p[0])
midmag.append((mlo+mhi)/2.)
nn.append(n)
rgbs.append(rgb)
bins = []
gaussint = []
for blo,bhi in zip(b, b[1:]):
#midbin.append((blo+bhi)/2.)
#gaussint.append(scipy.stats.norm.cdf(bhi) -
# scipy.stats.norm.cdf(blo))
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
bins.extend([blo,bhi])
gaussint.extend([c,c])
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo,shi)
ps.savefig()
bincenters = b[:-1] + (b[1]-b[0])/2.
plt.clf()
lp = []
for n,rgb,mlo,mhi in zip(nn, rgbs, magbins, magbins[1:]):
p = plt.plot(bincenters, n, '-', color=rgb)
lp.append(p[0])
plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo,shi)
ps.savefig()
def stage0(**kwargs):
ps = PlotSequence('cfht')
decals = CfhtDecals()
B = decals.get_bricks()
print 'Bricks:'
B.about()
ra,dec = 190.0, 11.0
#bands = 'ugri'
bands = 'gri'
B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec)))
print 'Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5]
brick = B[0]
pixscale = 0.186
#W,H = 1024,1024
#W,H = 2048,2048
#W,H = 3600,3600
W,H = 4800,4800
targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H)
ccdfn = 'cfht-ccds.fits'
if os.path.exists(ccdfn):
T = fits_table(ccdfn)
else:
T = get_ccd_list()
T.writeto(ccdfn)
print len(T), 'CCDs'
T.cut(ccds_touching_wcs(targetwcs, T))
print len(T), 'CCDs touching brick'
T.cut(np.array([b in bands for b in T.filter]))
print len(T), 'in bands', bands
ims = []
for t in T:
im = CfhtImage(t)
# magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME'])
# fwhm = t.seeing / (pixscale * 3600)
# print '-> FWHM', fwhm, 'pix'
im.seeing = t.seeing
im.pixscale = t.pixscale
print 'seeing', t.seeing
print 'pixscale', im.pixscale*3600, 'arcsec/pix'
im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height)
ims.append(im)
# Read images, clip to ROI
targetrd = np.array([targetwcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
keepims = []
tims = []
for im in ims:
print
print 'Reading expnum', im.expnum, 'name', im.extname, 'band', im.band, 'exptime', im.exptime
band = im.band
wcs = im.read_wcs()
imh,imw = wcs.imageh,wcs.imagew
imgpoly = [(1,1),(1,imh),(imw,imh),(imw,1)]
ok,tx,ty = wcs.radec2pixelxy(targetrd[:-1,0], targetrd[:-1,1])
tpoly = zip(tx,ty)
clip = clip_polygon(imgpoly, tpoly)
clip = np.array(clip)
#print 'Clip', clip
if len(clip) == 0:
continue
x0,y0 = np.floor(clip.min(axis=0)).astype(int)
x1,y1 = np.ceil (clip.max(axis=0)).astype(int)
slc = slice(y0,y1+1), slice(x0,x1+1)
## FIXME -- it seems I got lucky and the cross product is
## negative == clockwise, as required by clip_polygon. One
## could check this and reverse the polygon vertex order.
# dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0]
# dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1]
# cross = dx0*dy1 - dx1*dy0
# print 'Cross:', cross
print 'Image slice: x [%i,%i], y [%i,%i]' % (x0,x1, y0,y1)
print 'Reading image from', im.imgfn, 'HDU', im.hdu
img,imghdr = im.read_image(header=True, slice=slc)
goodpix = (img != 0)
print 'Number of pixels == 0:', np.sum(img == 0)
print 'Number of pixels != 0:', np.sum(goodpix)
if np.sum(goodpix) == 0:
continue
# print 'Image shape', img.shape
print 'Image range', img.min(), img.max()
print 'Goodpix image range:', (img[goodpix]).min(), (img[goodpix]).max()
if img[goodpix].min() == img[goodpix].max():
print 'No dynamic range in image'
continue
# print 'Reading invvar from', im.wtfn, 'HDU', im.hdu
# invvar = im.read_invvar(slice=slc)
# # print 'Invvar shape', invvar.shape
# # print 'Invvar range:', invvar.min(), invvar.max()
# invvar[goodpix == 0] = 0.
# if np.all(invvar == 0.):
# print 'Skipping zero-invvar image'
# continue
# assert(np.all(np.isfinite(img)))
# assert(np.all(np.isfinite(invvar)))
# assert(not(np.all(invvar == 0.)))
# # Estimate per-pixel noise via Blanton's 5-pixel MAD
# slice1 = (slice(0,-5,10),slice(0,-5,10))
# slice2 = (slice(5,None,10),slice(5,None,10))
# # print 'sliced shapes:', img[slice1].shape, img[slice2].shape
# # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape
# # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2])
# # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape
# mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel())
# sig1 = 1.4826 * mad / np.sqrt(2.)
# print 'MAD sig1:', sig1
# # invvar was 1 or 0
# invvar *= (1./(sig1**2))
# medsky = np.median(img[goodpix])
# Read full image for sig1 and sky estimate
fullimg = im.read_image()
fullgood = (fullimg != 0)
# Estimate per-pixel noise via Blanton's 5-pixel MAD
slice1 = (slice(0,-5,10),slice(0,-5,10))
slice2 = (slice(5,None,10),slice(5,None,10))
mad = np.median(np.abs(fullimg[slice1] - fullimg[slice2])[fullgood[slice1] * fullgood[slice2]].ravel())
sig1 = 1.4826 * mad / np.sqrt(2.)
print 'MAD sig1:', sig1
medsky = np.median(fullimg[fullgood])
invvar = np.zeros_like(img)
invvar[goodpix] = 1./sig1**2
# Median-smooth sky subtraction
plt.clf()
dimshow(np.round((img-medsky) / sig1), vmin=-3, vmax=5)
plt.title('Scalar median: %s' % im.name)
ps.savefig()
# medsky = np.zeros_like(img)
# # astrometry.util.util
# median_smooth(img, np.logical_not(goodpix), 256, medsky)
fullmed = np.zeros_like(fullimg)
median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed)
fullmed += medsky
medimg = fullmed[slc]
plt.clf()
dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5)
plt.title('Median filtered: %s' % im.name)
ps.savefig()
#print 'Subtracting median:', medsky
#img -= medsky
img -= medimg
primhdr = im.read_image_primary_header()
magzp = decals.get_zeropoint_for(im)
print 'magzp', magzp
zpscale = NanoMaggies.zeropointToScale(magzp)
print 'zpscale', zpscale
# Scale images to Nanomaggies
img /= zpscale
sig1 /= zpscale
invvar *= zpscale**2
orig_zpscale = zpscale
zpscale = 1.
assert(np.sum(invvar > 0) > 0)
print 'After scaling:'
print 'sig1', sig1
print 'invvar range', invvar.min(), invvar.max()
print 'image range', img.min(), img.max()
assert(np.all(np.isfinite(img)))
assert(np.all(np.isfinite(invvar)))
assert(np.isfinite(sig1))
plt.clf()
lo,hi = -5*sig1, 10*sig1
n,b,p = plt.hist(img[goodpix].ravel(), 100, range=(lo,hi), histtype='step', color='k')
xx = np.linspace(lo, hi, 200)
plt.plot(xx, max(n)*np.exp(-xx**2 / (2.*sig1**2)), 'r-')
plt.xlim(lo,hi)
plt.title('Pixel histogram: %s' % im.name)
ps.savefig()
twcs = ConstantFitsWcs(wcs)
if x0 or y0:
twcs.setX0Y0(x0,y0)
info = im.get_image_info()
fullh,fullw = info['dims']
# read fit PsfEx model
psfex = PsfEx.fromFits(im.psffitfn)
print 'Read', psfex
# HACK -- highly approximate PSF here!
#psf_fwhm = imghdr['FWHM']
#psf_fwhm = im.seeing
psf_fwhm = im.seeing / (im.pixscale * 3600)
print 'PSF FWHM', psf_fwhm, 'pixels'
psf_sigma = psf_fwhm / 2.35
psf = NCircularGaussianPSF([psf_sigma],[1.])
print 'img type', img.dtype
tim = Image(img, invvar=invvar, wcs=twcs, psf=psf,
photocal=LinearPhotoCal(zpscale, band=band),
sky=ConstantSky(0.), name=im.name + ' ' + band)
tim.zr = [-3. * sig1, 10. * sig1]
tim.sig1 = sig1
tim.band = band
tim.psf_fwhm = psf_fwhm
tim.psf_sigma = psf_sigma
tim.sip_wcs = wcs
tim.x0,tim.y0 = int(x0),int(y0)
tim.psfex = psfex
tim.imobj = im
mn,mx = tim.zr
tim.ima = dict(interpolation='nearest', origin='lower', cmap='gray',
vmin=mn, vmax=mx)
tims.append(tim)
keepims.append(im)
ims = keepims
print 'Computing resampling...'
# save resampling params
for tim in tims:
wcs = tim.sip_wcs
subh,subw = tim.shape
subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh)
tim.subwcs = subwcs
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(targetwcs, subwcs, [], 2)
except OverlapError:
print 'No overlap'
continue
if len(Yo) == 0:
continue
tim.resamp = (Yo,Xo,Yi,Xi)
print 'Creating coadds...'
# Produce per-band coadds, for plots
coimgs = []
cons = []
for ib,band in enumerate(bands):
coimg = np.zeros((H,W), np.float32)
con = np.zeros((H,W), np.uint8)
for tim in tims:
if tim.band != band:
continue
(Yo,Xo,Yi,Xi) = tim.resamp
if len(Yo) == 0:
continue
nn = (tim.getInvvar()[Yi,Xi] > 0)
coimg[Yo,Xo] += tim.getImage ()[Yi,Xi] * nn
con [Yo,Xo] += nn
# print
# print 'tim', tim.name
# print 'number of resampled pix:', len(Yo)
# reim = np.zeros_like(coimg)
# ren = np.zeros_like(coimg)
# reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn
# ren[Yo,Xo] = nn
# print 'number of resampled pix with positive invvar:', ren.sum()
# plt.clf()
# plt.subplot(2,2,1)
# mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]]
# print 'Percentiles:', mn,mx
# dimshow(reim, vmin=mn, vmax=mx)
# plt.colorbar()
# plt.subplot(2,2,2)
# dimshow(con)
# plt.colorbar()
# plt.subplot(2,2,3)
# dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1])
# plt.colorbar()
# plt.subplot(2,2,4)
# plt.hist(reim.ravel(), 100, histtype='step', color='b')
# plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r')
# plt.suptitle('%s: %s' % (band, tim.name))
# ps.savefig()
coimg /= np.maximum(con,1)
coimgs.append(coimg)
cons .append(con)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ps.savefig()
plt.clf()
for i,b in enumerate(bands):
plt.subplot(2,2,i+1)
dimshow(cons[i], ticks=False)
plt.title('%s band' % b)
plt.colorbar()
plt.suptitle('Number of exposures')
ps.savefig()
print 'Grabbing SDSS sources...'
bandlist = [b for b in bands]
cat,T = get_sdss_sources(bandlist, targetwcs)
# record coordinates in target brick image
ok,T.tx,T.ty = targetwcs.radec2pixelxy(T.ra, T.dec)
T.tx -= 1
T.ty -= 1
T.itx = np.clip(np.round(T.tx).astype(int), 0, W-1)
T.ity = np.clip(np.round(T.ty).astype(int), 0, H-1)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5)
plt.axis(ax)
plt.title('SDSS sources')
ps.savefig()
print 'Detmaps...'
# Render the detection maps
detmaps = dict([(b, np.zeros((H,W), np.float32)) for b in bands])
detivs = dict([(b, np.zeros((H,W), np.float32)) for b in bands])
for tim in tims:
iv = tim.getInvvar()
psfnorm = 1./(2. * np.sqrt(np.pi) * tim.psf_sigma)
detim = tim.getImage().copy()
detim[iv == 0] = 0.
detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2
detsig1 = tim.sig1 / psfnorm
subh,subw = tim.shape
detiv = np.zeros((subh,subw), np.float32) + (1. / detsig1**2)
detiv[iv == 0] = 0.
(Yo,Xo,Yi,Xi) = tim.resamp
detmaps[tim.band][Yo,Xo] += detiv[Yi,Xi] * detim[Yi,Xi]
detivs [tim.band][Yo,Xo] += detiv[Yi,Xi]
rtn = dict()
for k in ['T', 'coimgs', 'cons', 'detmaps', 'detivs',
'targetrd', 'pixscale', 'targetwcs', 'W','H',
'bands', 'tims', 'ps', 'brick', 'cat']:
rtn[k] = locals()[k]
return rtn