def ipe_errors():
#T = fits_table('ipe1_dstn.fit')
#T.cut((T.ra < 249.85) * (T.dec < 17.65))
#print 'Cut to', len(T)
ps = PlotSequence('ipe')
#T = fits_table('ipe2_dstn_1.fit')
T = fits_table('ipe3_dstn_2.fit')
print len(T), 'objects'
print 'Runs', np.unique(T.run)
print 'Camcols', np.unique(T.camcol)
print 'Fields', np.unique(T.field)
T1 = T[T.run == 5183]
T2 = T[T.run == 5224]
plt.clf()
plt.plot(T1.ra, T1.dec, 'r.', alpha=0.1)
plt.plot(T2.ra, T2.dec, 'bx', alpha=0.1)
ps.savefig()
for T in [T1, T2]:
# self-matches:
print 'T:', len(T)
R = 0.5 / 3600.
I, J, d = match_radec(T.ra, T.dec, T.ra, T.dec, R, notself=True)
print len(I), 'matches'
K = (I < J)
I = I[K]
J = J[K]
print len(I), 'symmetric'
print sum(T.field[I] == T.field[J]), 'are in the same field'
#plt.clf()
#plt.plot(T.rowc[I], T.colc[I], 'r.')
#plt.plot(T.rowc[J], T.colc[J], 'b.')
#plt.savefig('ipe2.png')
keep = np.ones(len(T), bool)
keep[I[T.field[I] != T.field[J]]] = False
T.cut(keep)
print 'Cut to', len(T), 'with no matches in other fields'
R = 1. / 3600.
I, J, d = match_radec(T1.ra, T1.dec, T2.ra, T2.dec, R)
print len(I), 'matches'
dra = (T1.ra[I] - T2.ra[J]) * np.cos(np.deg2rad(T1.dec[I])) * 3600.
ddec = (T1.dec[I] - T2.dec[J]) * 3600.
#plt.clf()
#loghist(dra, ddec, 100, range=((-1,1),(-1,1)))
#ps.savefig()
print 'Range of Decs:', T1.dec.min(), T1.dec.max(), T2.dec.min(
), T2.dec.max()
ras = np.cos(np.deg2rad(np.append(T1.dec, T2.dec)))
print 'Range of RAscales:', ras.min(), ras.max()
rascale = np.mean(ras)
X1 = np.vstack((T1.ra * rascale, T1.dec)).T
X2 = np.vstack((T2.ra * rascale, T2.dec)).T
inds, d = nearest(X1, X2, R)
J = np.flatnonzero(inds > -1)
I = inds[J]
print 'Nearest-neighbour matches:', len(I)
d = np.sqrt(d[J])
print 'd', d.shape
print 'I,J', len(I), len(J)
print 'd max', np.max(d), 'min', np.min(d)
print 'R', R
assert (np.all(d <= R))
assert (np.all(J >= 0))
assert (np.all(I >= 0))
dx = X1[I] - X2[J]
print 'dx', dx.shape
dr = np.hypot(dx[:, 0], dx[:, 1])
print 'dr', dr.shape
assert (np.all(dr <= R))
dra = (T1.ra[I] - T2.ra[J]) * rascale * 3600.
ddec = (T1.dec[I] - T2.dec[J]) * 3600.
plt.clf()
loghist(dra, ddec, 100, range=((-1, 1), (-1, 1)))
ps.savefig()
M1 = T1[I]
M2 = T2[J]
#print 'All T1', T1.rowc.min(), T1.rowc.max(), T1.colc.min(), T1.colc.max()
#print 'Matched T1', T1.rowc[I].min(), T1.rowc[I].max(), T1.colc[I].min(), T1.colc[I].max()
#print 'All T2', T2.rowc.min(), T2.rowc.max(), T2.colc.min(), T2.colc.max()
#print 'Matched T2', T2.rowc[J].min(), T2.rowc[J].max(), T2.colc[J].min(), T2.colc[J].max()
# Errors are in arcsec.
rerr1 = M1.raerr
derr1 = M1.decerr
rerr2 = M2.raerr
derr2 = M2.decerr
hi = 6.
dscale = 1. / np.sqrt(2.)
plt.clf()
n, b, p = plt.hist(dscale * np.hypot(dra, ddec) / np.hypot(rerr1, derr1),
100,
range=(0, hi),
histtype='step',
color='r')
plt.hist(dscale * np.hypot(dra, ddec) / np.hypot(rerr2, derr2),
100,
range=(0, hi),
histtype='step',
color='b')
xx = np.linspace(0, hi, 500)
from scipy.stats import chi
yy = chi.pdf(xx, 2)
plt.plot(xx, yy * len(dra) * (b[1] - b[0]), 'k-')
plt.xlim(0, hi)
plt.xlabel('N sigma of RA,Dec repeat observations')
plt.ylabel('Number of sources')
ps.savefig()
#loghist(np.hypot(dra, ddec), np.sqrt(np.hypot(rerr1, derr1) * np.hypot(rerr2, derr2)), 100,
# clamp=((0,1),(0,1)))
loghist(np.hypot(dra, ddec),
(np.hypot(rerr1, derr1) + np.hypot(rerr2, derr2)) / 2.,
100,
range=((0, 1), (0, 1)),
clamp=True)
plt.xlabel('Inter-ipe difference: RA,Dec (arcsec)')
plt.ylabel('Photo errors: RA,Dec (arcsec)')
ps.savefig()
loghist(np.log10(np.hypot(dra, ddec)),
np.log10((np.hypot(rerr1, derr1) + np.hypot(rerr2, derr2)) / 2.),
100,
range=((-3, 0), (-3, 0)),
clamp=True)
plt.xlabel('Inter-ipe difference: log RA,Dec (arcsec)')
plt.ylabel('Photo errors: log RA,Dec (arcsec)')
ps.savefig()
plt.clf()
n, b, p = plt.hist(dscale * np.abs(M1.psfmag_r - M2.psfmag_r) /
M1.psfmagerr_r,
100,
range=(0, hi),
histtype='step',
color='r')
plt.xlabel('N sigma of psfmag_r')
xx = np.linspace(0, hi, 500)
yy = 2. / np.sqrt(2. * np.pi) * np.exp(-0.5 * xx**2)
print 'yy', sum(yy)
plt.plot(xx, yy * len(M1) * (b[1] - b[0]), 'k-')
ps.savefig()
# Galaxy-star matches
K1 = (M1.type == 3) * (M2.type == 6)
K2 = (M1.type == 6) * (M2.type == 3)
G = merge_tables((M1[K1], M2[K2]))
S = merge_tables((M2[K1], M1[K2]))
print 'G types:', np.unique(G.type)
print 'S types:', np.unique(S.type)
mhi, mlo = 24, 10
K = ((G.modelmag_r < mhi) * (S.psfmag_r < mhi) * (G.modelmag_r > mlo) *
(S.psfmag_r > mlo))
print 'Star/gal mismatches with good mags:', np.sum(K)
# gm = G.modelmag_r.copy()
# gm[np.logical_or(gm > mhi, gm < mlo)] = 25.
# sm = S.psfmag_r.copy()
# sm[np.logical_or(sm > mhi, sm < mlo)] = 25.
#
# #loghist(G.modelmag_r[K], S.psfmag_r[K], 100)
# loghist(gm, sm, 100)
loghist(G.modelmag_r,
S.psfmag_r,
clamp=((mlo, mhi), (mlo, mhi)),
clamp_to=((mlo - 1, mhi + 1), (mlo - 1, mhi + 1)))
ax = plt.axis()
plt.axhline(mhi, color='b')
plt.axvline(mhi, color='b')
plt.plot(*([[min(ax[0], ax[2]), max(ax[1], ax[3])]] * 2) + [
'b-',
])
plt.axis(ax)
plt.xlabel('Galaxy modelmag_r')
plt.ylabel('Star psfmag_r')
plt.title('Star/Galaxy ipe mismatches')
ps.savefig()
K = ((G.modelmag_r < mhi) * (G.modelmag_r > mlo))
plt.clf()
KK = (G.fracdev_r < 0.5)
kwargs = dict(bins=100,
range=(np.log10(0.01), np.log10(30.)),
histtype='step')
plt.hist(np.log10(G.exprad_r[K * KK]), color='r', **kwargs)
KK = (G.fracdev_r >= 0.5)
plt.hist(np.log10(G.devrad_r[K * KK]), color='b', **kwargs)
plt.xlabel('*rad_r (arcsec)')
loc, lab = plt.xticks()
plt.xticks(loc, ['%g' % (10.**x) for x in loc])
plt.title('Star/Galaxy ipe mismatches')
ps.savefig()
# Pairs where both are galaxies
K = ((M1.type == 3) * (M2.type == 3))
G1 = M1[K]
G2 = M2[K]
print len(G1), 'pairs where both are galaxies'
#for
plt.clf()
c, cerr = 'modelmag_r', 'modelmagerr_r'
n, b, p = plt.hist(dscale * np.abs(G1.get(c) - G2.get(c)) / G1.get(cerr),
100,
range=(0, hi),
histtype='step',
color='r')
plt.xlabel('N sigma of ' + c)
yy = np.exp(-0.5 * b**2)
yy *= sum(n) / np.sum(yy)
plt.plot(b, yy, 'k-')
ps.savefig()
loghist(np.abs(G1.get(c) - G2.get(c)),
G1.get(cerr),
100,
range=((0, 2), (0, 2)),
clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
loghist(np.log10(np.abs(G1.get(c) - G2.get(c))),
np.log10(G1.get(cerr)),
100,
range=((-3, 1), (-3, 1)),
clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
plt.clf()
loghist(G1.fracdev_r,
G2.fracdev_r,
100,
range=((0, 1), (0, 1)),
clamp=True)
plt.xlabel('G1 fracdev_r')
plt.ylabel('G2 fracdev_r')
ps.savefig()
dscale = 1.
I = (G1.fracdev_r < 0.5) * (G2.fracdev_r < 0.5)
print sum(I), 'of', len(G1), 'both have fracdev_r < 0.5'
E1 = G1[I]
E2 = G2[I]
I = (G1.fracdev_r >= 0.5) * (G2.fracdev_r >= 0.5)
print sum(I), 'of', len(G1), 'both have fracdev_r >= 0.5'
D1 = G1[I]
D2 = G2[I]
for t, H1, H2 in [('exp', E1, E2), ('dev', D1, D2)]:
c, cerr = '%smag_r' % t, '%smagerr_r' % t
dval = np.abs(H1.get(c) - H2.get(c))
derr = H1.get(cerr)
rng = ((0, 1), (0, 1))
loghist(dval, derr, 100, range=rng, clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
loghist(np.log10(dval),
np.log10(derr),
100,
range=((-3, 0), (-3, 0)),
clamp=True)
plt.xlabel('Inter-ipe difference: log ' + c)
plt.ylabel('Photo error: log ' + cerr)
ps.savefig()
c, cerr = '%sab_r' % t, '%saberr_r' % t
dval = np.abs(H1.get(c) - H2.get(c))
derr = H1.get(cerr)
rng = ((0, 1), (0, 1))
loghist(dval, derr, 100, range=rng, clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
loghist(np.log10(dval),
np.log10(derr),
100,
range=((-3, 0), (-3, 0)),
clamp=True)
plt.xlabel('Inter-ipe difference: log ' + c)
plt.ylabel('Photo error: log ' + cerr)
ps.savefig()
c, cerr = '%srad_r' % t, '%sraderr_r' % t
dval = np.abs(H1.get(c) - H2.get(c))
derr = H1.get(cerr)
rng = ((0, 30), (0, 30))
loghist(dval, derr, 100, range=rng, clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
loghist(np.log10(dval),
np.log10(derr),
100,
range=((-2, 2), (-2, 2)),
clamp=True)
plt.xlabel('Inter-ipe difference: log ' + c)
plt.ylabel('Photo error: log ' + cerr)
ps.savefig()
return
I, J, d = match_radec(T.ra, T.dec, T.ra, T.dec, 0.5 / 3600., notself=True)
print len(I), 'matches'
plt.clf()
loghist((T.ra[I] - T.ra[J]) * np.cos(np.deg2rad(T.dec[I])) * 3600.,
(T.dec[I] - T.dec[J]) * 3600.,
100,
range=((-1, 1), (-1, 1)))
plt.savefig('ipe4.png')
K = (I < J)
I = I[K]
J = J[K]
d = d[K]
print 'Cut to', len(I), 'symmetric'
plt.clf()
plt.plot(T.ra, T.dec, 'r.')
plt.plot(T.ra[I], T.dec[I], 'bo', mec='b', mfc='none')
plt.savefig('ipe2.png')
dra, ddec = [], []
raerr, decerr = [], []
RC = T.run * 10 + T.camcol
RCF = T.run * 10 * 1000 + T.camcol * 1000 + T.field
for i in np.unique(I):
K = (I == i)
JJ = J[K]
print
print 'Source', i, 'has', len(JJ), 'matches'
print ' ', np.sum(RC[JJ] == RC[i]), 'in the same run/camcol'
print ' ', np.sum(RCF[JJ] == RCF[i]), 'in the same run/camcol/field'
orc = (RC[JJ] != RC[i])
print ' ', np.sum(orc), 'are in other run/camcols'
print ' ', len(np.unique(RC[JJ][orc])), 'unique other run/camcols'
print ' ', len(np.unique(
RCF[JJ][orc])), 'unique other run/camcols/fields'
print ' other sources:', JJ
dra.extend((T.ra[JJ] - T.ra[i]) * np.cos(np.deg2rad(T.dec[i])))
ddec.extend(T.dec[JJ] - T.dec[i])
raerr.extend([T.raerr[i]] * len(JJ))
decerr.extend([T.decerr[i]] * len(JJ))
dra, ddec = np.array(dra), np.array(ddec)
raerr, decerr = np.array(raerr), np.array(decerr)
plt.clf()
plt.hist(np.hypot(dra, ddec) / np.hypot(raerr, decerr), 100)
plt.savefig('ipe3.png')
def my_ipe_errors():
ps = PlotSequence('myipe')
T = fits_table('my-ipes.fits')
print 'Got', len(T), 'galaxies'
# Photo errors are in arcsec.
T.raerr /= 3600.
T.decerr /= 3600.
# The galaxies here are in consecutive pairs.
T1 = T[::2]
T2 = T[1::2]
print len(T1), 'pairs'
plt.clf()
plt.plot(T1.raerr * 3600., np.abs((T1.ra - T2.ra) / T1.raerr), 'r.')
plt.plot(T2.raerr * 3600., np.abs((T1.ra - T2.ra) / T2.raerr), 'm.')
plt.plot(T1.my_ra_err * 3600.,
np.abs(T1.my_ra - T2.my_ra) / T1.my_ra_err, 'b.')
plt.plot(T2.my_ra_err * 3600.,
np.abs(T1.my_ra - T2.my_ra) / T2.my_ra_err, 'c.')
plt.gca().set_xscale('log')
plt.gca().set_yscale('log')
ps.savefig()
plt.clf()
#mn = np.min(T1.raerr.min(), T1.my_ra_err.min())*3600.
#mx = np.max(T1.raerr.max(), T1.my_ra_err.max())*3600.
plt.subplot(2, 1, 1)
plt.loglog(T1.raerr * 3600.,
np.abs((T1.ra - T2.ra) / T1.raerr),
'r.',
alpha=0.5)
plt.plot(T2.raerr * 3600.,
np.abs((T1.ra - T2.ra) / T2.raerr),
'r.',
alpha=0.5)
ax1 = plt.axis()
x = np.append(T1.raerr, T2.raerr) * 3600.
y = np.append(
np.abs(T1.ra - T2.ra) / T1.raerr,
np.abs(T1.ra - T2.ra) / T2.raerr)
x1, y1 = x, y
I = np.argsort(x)
S = np.linspace(0, len(x), 11).astype(int)
xm, ym, ys = [], [], []
for ilo, ihi in zip(S[:-1], S[1:]):
J = I[ilo:ihi]
xm.append(np.mean(x[J]))
ym.append(np.median(y[J]))
ys.append(np.abs(np.percentile(y[J], 75) - np.percentile(y[J], 25)))
p, c, b = plt.errorbar(xm, ym, yerr=ys, color='k', fmt='o')
for pp in [p] + list(c) + list(b):
pp.set_zorder(20)
#for ci in c: ci.set_zorder(20)
#for bi in b: bi.set_zorder(20)
xm1, ym1, ys1 = xm, ym, ys
#plt.xlim(mn,mx)
plt.subplot(2, 1, 2)
plt.loglog(T1.my_ra_err * 3600.,
np.abs(T1.my_ra - T2.my_ra) / T1.my_ra_err,
'b.',
alpha=0.5)
plt.plot(T2.my_ra_err * 3600.,
np.abs(T1.my_ra - T2.my_ra) / T2.my_ra_err,
'b.',
alpha=0.5)
x = np.append(T1.my_ra_err, T2.my_ra_err) * 3600.
y = np.append(
np.abs(T1.my_ra - T2.my_ra) / T1.my_ra_err,
np.abs(T1.my_ra - T2.my_ra) / T2.my_ra_err)
x2, y2 = x, y
I = np.argsort(x)
S = np.linspace(0, len(x), 11).astype(int)
xm, ym, ys = [], [], []
for ilo, ihi in zip(S[:-1], S[1:]):
J = I[ilo:ihi]
xm.append(np.mean(x[J]))
ym.append(np.median(y[J]))
ys.append(np.abs(np.percentile(y[J], 75) - np.percentile(y[J], 25)))
p, c, b = plt.errorbar(xm, ym, yerr=ys, color='k', fmt='o')
for pp in [p] + list(c) + list(b):
pp.set_zorder(20)
ax2 = plt.axis()
ax = [
min(ax1[0], ax2[0]),
max(ax1[1], ax2[1]),
min(ax1[2], ax2[2]),
max(ax1[3], ax2[3])
]
plt.axis(ax)
plt.gca().set_yscale('symlog', linthreshy=1e-3)
plt.axhline(1, color='k', alpha=0.5, lw=2)
plt.subplot(2, 1, 1)
plt.axis(ax)
plt.gca().set_yscale('symlog', linthreshy=1e-3)
plt.axhline(1, color='k', alpha=0.5, lw=2)
ps.savefig()
x1 = np.sqrt(T1.raerr * T2.raerr)
y1 = (T1.ra - T2.ra) / x1
x1 *= 3600.
x2 = np.sqrt(T1.my_ra_err * T2.my_ra_err)
y2 = (T1.my_ra - T2.my_ra) / x2
x2 *= 3600.
ipe_err_plots(x1, y1, x2, y2, 'RA error (arcsec)', (5e-3, 2.), ps)
x1 = np.sqrt(T1.decerr * T2.decerr)
y1 = (T1.dec - T2.dec) / x1
x1 *= 3600.
x2 = np.sqrt(T1.my_dec_err * T2.my_dec_err)
y2 = (T1.my_dec - T2.my_dec) / x2
x2 *= 3600.
ipe_err_plots(x1, y1, x2, y2, 'Dec error (arcsec)', (5e-3, 2.), ps)
for p in ['dev', 'exp']:
for c1, e1, c2, e2, tt, xlim in [
('%smag_r', '%smagerr_r', 'my_%smag_r', 'my_%smag_r_err',
'%s r mag error' % p, (5e-3, 2)),
('%srad_r', '%sraderr_r', 'my_%srad_r', 'my_%srad_r_err',
'%s radius error (arcsec)' % p, (5e-3, 10)),
('%sab_r', '%saberr_r', 'my_%sab_r', 'my_%sab_r_err',
'%s A/B error' % p, (9e-3, 5.)),
#('%sphi_r', '%sphierr_r', 'my_%sphi_r', 'my_%sphi_r_err', '%s phi error'%p, (1e-3, 1e2)),
]:
c1 = c1 % p
c2 = c2 % p
e1 = e1 % p
e2 = e2 % p
x1 = np.sqrt(T1.get(e1) * T2.get(e1))
y1 = (T1.get(c1) - T2.get(c1)) / x1
I = (np.isfinite(x1) * np.isfinite(y1))
x1, y1 = x1[I], y1[I]
x2 = np.sqrt(T1.get(e2) * T2.get(e2))
y2 = (T1.get(c2) - T2.get(c2)) / x2
I = (np.isfinite(x2) * np.isfinite(y2))
x2, y2 = x2[I], y2[I]
ipe_err_plots(x1, y1, x2, y2, tt, xlim, ps)
