def plot_A(self, histtype="stepfilled"):
if histtype=="step":
ALPHAG=0.5
else:
ALPHAG=ALPHA
plot_hist(plt, self.gA, clr=self.clrs[0], alp=ALPHAG, ht=histtype)
sG=np.sqrt(np.var(self.gAi))
mG=np.mean(self.gAi)
amin, amax = plt.xlim()
alist=np.arange(0.0, np.max(self.fgNLA), 0.001)
theoryGdist=chi.pdf(alist, 3,scale=sG)
plt.plot(alist, theoryGdist,self.clrs[0], linestyle=self.ls[0], linewidth=LW, label=self.TYPELABEL+r"$=0$")
for i in range(len(self.fgnls)):
if (self.theoryplot==False):
lbl=self.TYPELABEL+"="+NtoSTR(self.fgnls[i])
else:
lbl=None
plot_hist(plt, self.fgNLA[i], clr=self.clrs[i+1], alp=ALPHA, labl=lbl)
if (self.theoryplot):
theorynGdist=chi.pdf(alist, 3, scale=np.sqrt((self.A1const*self.fgnls[i])**2.0+sG**2.0))
plt.plot(alist, theorynGdist, self.clrs[i+1], linestyle=self.ls[i+1], linewidth=LW, label=self.TYPELABEL+"="+NtoSTR(self.fgnls[i]))
# theory plots
#plt.xlim(0.0, 0.2)
#plt.xticks([0.02, 0.04, 0.06, 0.08, 0.10])
plt.xlabel(r'$A$')
plt.ylabel(r'$p(A)$')
plt.legend(fontsize=20)
def plot_A(self, histtype="stepfilled"):
if histtype=="step":
ALPHAG=0.5
else:
ALPHAG=ALPHA
plot_hist(plt, self.gA, clr=self.clrs[0], alp=ALPHAG, ht=histtype)
sG=np.sqrt(np.var(self.gAi))
mG=np.mean(self.gAi)
amin, amax = plt.xlim()
alist=np.arange(0.0, np.max(self.fgNLA), 0.001)
theoryGdist=chi.pdf(alist, 3,scale=sG)
plt.plot(alist, theoryGdist,self.clrs[0], linestyle=self.ls[0], linewidth=LW, label=self.TYPELABEL+r"$=0$")
for i in range(len(self.fgnls)):
if (self.theoryplot==False):
lbl=self.TYPELABEL+"="+NtoSTR(self.fgnls[i])
else:
lbl=None
plot_hist(plt, self.fgNLA[i], clr=self.clrs[i+1], alp=ALPHA, labl=lbl)
if (self.theoryplot):
theorynGdist=chi.pdf(alist, 3, scale=np.sqrt((self.A1const*self.fgnls[i])**2.0+sG**2.0))
plt.plot(alist, theorynGdist, self.clrs[i+1], linestyle=self.ls[i+1], linewidth=LW, label=self.TYPELABEL+"="+NtoSTR(self.fgnls[i]))
# theory plots
#plt.xlim(0.0, 0.2)
#plt.xticks([0.02, 0.04, 0.06, 0.08, 0.10])
plt.xlabel(r'$A$')
plt.ylabel(r'$p(A)$')
plt.legend(fontsize=20)
def error_multi_output_regressor(actual, prediction_gbr, prediction_mlp,
prediction_rfr):
distance_error_gbr = []
distance_error_mlp = []
distance_error_rfr = []
for i in range(len(prediction_gbr)):
error_gbr = pd.distance_formula(actual[i, 0], actual[i, 1],
prediction_gbr[i, 0],
prediction_gbr[i, 1])
distance_error_gbr.append(error_gbr)
for i in range(len(prediction_mlp)):
error_mlp = pd.distance_formula(actual[i, 0], actual[i, 1],
prediction_mlp[i, 0],
prediction_mlp[i, 1])
distance_error_mlp.append(error_mlp)
for i in range(len(prediction_rfr)):
error_rfr = pd.distance_formula(actual[i, 0], actual[i, 1],
prediction_rfr[i, 0],
prediction_rfr[i, 1])
distance_error_rfr.append(error_rfr)
plt.figure(figsize=(10, 7))
n0, bins0, patches0 = plt.hist(distance_error_gbr,
10,
normed=True,
facecolor='green',
edgecolor='black',
alpha=0.4,
label='GBR')
n1, bins1, patches1 = plt.hist(distance_error_mlp,
10,
normed=True,
facecolor='blue',
edgecolor='black',
alpha=0.4,
label='MLP')
n2, bin2, patches2 = plt.hist(distance_error_rfr,
10,
normed=True,
facecolor='red',
edgecolor='black',
alpha=0.4,
label='RFR')
y0 = chi.pdf(bins0, 2)
y1 = chi.pdf(bins1, 2)
y2 = chi.pdf(bin2, 2)
plt.plot(bins0, y0, 'g--', linewidth=4)
plt.plot(bins1, y1, 'b-.', linewidth=4)
plt.plot(bin2, y2, 'r:', linewidth=4)
plt.legend(loc='best')
plt.xlabel(r'distance: $\sqrt{(x_p-x)^2 + (y_p-y})^2}$')
plt.title("Prediction Error")
plt.show()
def art_qi2(img, airmask, ncoils=12, erodemask=True):
"""
Calculates **qi2**, the distance between the distribution
of noise voxel (non-artifact background voxels) intensities, and a
centered Chi distribution.
:param numpy.ndarray img: input data
:param numpy.ndarray airmask: input air mask without artifacts
"""
from matplotlib import rc
import seaborn as sn
import matplotlib.pyplot as plt
rc('font',**{'family':'sans-serif','sans-serif':['Helvetica']})
# rc('text', usetex=True)
if erodemask:
struc = nd.generate_binary_structure(3, 2)
# Perform an opening operation on the background data.
airmask = nd.binary_erosion(airmask, structure=struc).astype(np.uint8)
# Artifact-free air region
data = img[airmask > 0]
data = data[data < np.percentile(data, 99.5)]
maxvalue = int(data.max())
nbins = maxvalue if maxvalue < 100 else 100
# Estimate data pdf
hist, bin_edges = np.histogram(data, density=True, bins=nbins)
bin_centers = [np.mean(bin_edges[i:i+1]) for i in range(len(bin_edges)-1)]
max_pos = np.argmax(hist)
# Fit central chi distribution
param = chi.fit(data, 2*ncoils, loc=bin_centers[max_pos])
pdf_fitted = chi.pdf(bin_centers, *param[:-2], loc=param[-2], scale=param[-1])
# Write out figure of the fitting
out_file = op.abspath('background_fit.png')
fig = plt.figure()
ax1 = fig.add_subplot(111)
sn.distplot(data, bins=nbins, norm_hist=True, kde=False, ax=ax1)
#_, bins, _ = ax1.hist(data, nbins, normed=True, color='gray', linewidth=0)
ax1.plot(bin_centers, pdf_fitted, 'k--', linewidth=1.2)
fig.suptitle('Noise distribution on the air mask, and fitted chi distribution')
ax1.set_xlabel('Intensity')
ax1.set_ylabel('Frequency')
fig.savefig(out_file, format='png', dpi=300)
plt.close()
# Find t2 (intensity at half width, right side)
ihw = 0.5 * hist[max_pos]
t2idx = 0
for i in range(max_pos + 1, len(bin_centers)):
if hist[i] < ihw:
t2idx = i
break
# Compute goodness-of-fit (gof)
return (float(np.abs(hist[t2idx:] - pdf_fitted[t2idx:]).sum() /
len(pdf_fitted[t2idx:])), out_file)
def pdf(self, A, fNL):
"""
for power asymmetry amplitude A as the variable, and fNL as the parameter
the pdf is a Maxwell distribution
"""
if (fNL>=0.0):
return chi.pdf(A, 3, scale=np.sqrt((self.A1const*fNL)**2.0+self.sigmaG**2.0))
else:
return 0.0
def spectra_error(param, args=()):
#args = (F_model,F_noise,k,F_obs)
args_theo = args[:-1]
F_obs = args[-1]
k_1 = args[-2]
param_chi = param[-1]
param_theo = param[:-2]
F_theo = spectra_theo(param_theo, args=args_theo)
#Chi-squared error
F_obs_f = F_obs * k_1**param[-2]
error = param_chi * F_obs_f / F_theo
return chi.pdf(error, param_chi)
def pdf(self, A, fNL):
"""
for power asymmetry amplitude A as the variable, and fNL as the parameter
the pdf is a Maxwell distribution
"""
if (fNL >= 0.0):
return chi.pdf(A,
3,
scale=np.sqrt((self.A1const * fNL)**2.0 +
self.sigmaG**2.0))
else:
return 0.0
def test_chi(self):
from scipy.stats import chi
import matplotlib.pyplot as plt
fig, ax = plt.subplots(1, 1)
df = 78
mean, var, skew, kurt = chi.stats(df, moments='mvsk')
x = np.linspace(chi.ppf(0.01, df), chi.ppf(0.99, df), 100)
ax.plot(x, chi.pdf(x, df), 'r-', lw=5, alpha=0.6, label='chi pdf')
rv = chi(df)
ax.plot(x, rv.pdf(x), 'k-', lw=2, label='frozen pdf')
vals = chi.ppf([0.001, 0.5, 0.999], df)
np.allclose([0.001, 0.5, 0.999], chi.cdf(vals, df))
r = chi.rvs(df, size=1000)
ax.hist(r, density=True, histtype='stepfilled', alpha=0.2)
ax.legend(loc='best', frameon=False)
self.assertEqual(str(ax), "AxesSubplot(0.125,0.11;0.775x0.77)")
def art_qi2(img, airmask, artmask, ncoils=1):
"""
Calculates **qi2**, the distance between the distribution
of noise voxel (non-artifact background voxels) intensities, and a
centered Chi distribution.
:param numpy.ndarray img: input data
:param numpy.ndarray airmask: input air mask without artifacts
"""
# Artifact-free air region
data = img[airmask > 0]
# Estimate data pdf
hist, bin_edges = np.histogram(data, density=True, bins=128)
bin_centers = [
np.mean(bin_edges[i:i + 1]) for i in range(len(bin_edges) - 1)
]
max_pos = np.argmax(hist)
# Fit central chi distribution
param = chi.fit(data, 2 * ncoils, loc=bin_centers[max_pos])
pdf_fitted = chi.pdf(bin_centers,
*param[:-2],
loc=param[-2],
scale=param[-1])
# Find t2 (intensity at half width, right side)
ihw = 0.5 * hist[max_pos]
t2idx = 0
for i in range(max_pos + 1, len(bin_centers)):
if hist[i] < ihw:
t2idx = i
break
# Compute goodness-of-fit (gof)
gof = np.abs(hist[t2idx:] - pdf_fitted[t2idx:]).sum() / airmask.sum()
return float(art_qi1(airmask, artmask) + gof)
def _pdf_notTruncated(self, z, dps):
scale = self._scale
k = self._k
dps = self._dps
return chi.pdf(z/scale, k)
def art_qi2(img,
airmask,
ncoils=12,
erodemask=True,
out_file='qi2_fitting.txt',
min_voxels=1e3):
r"""
Calculates :math:`\text{QI}_2`, based on the goodness-of-fit of a centered
:math:`\chi^2` distribution onto the intensity distribution of
non-artifactual background (within the "hat" mask):
.. math ::
\chi^2_n = \frac{2}{(\sigma \sqrt{2})^{2n} \, (n - 1)!}x^{2n - 1}\, e^{-\frac{x}{2}}
where :math:`n` is the number of coil elements.
:param numpy.ndarray img: input data
:param numpy.ndarray airmask: input air mask without artifacts
"""
out_file = op.abspath(out_file)
open(out_file, 'a').close()
if erodemask:
struc = nd.generate_binary_structure(3, 2)
# Perform an opening operation on the background data.
airmask = nd.binary_erosion(airmask, structure=struc).astype(np.uint8)
# Artifact-free air region
data = img[airmask > 0]
# Background can only be fit if we have a min number of voxels
if len(data[data > 0]) < min_voxels:
return 0.0, out_file
# Estimate data pdf
dmax = np.percentile(data[data > 0], 99.9)
hist, bin_edges = np.histogram(data[data > 0],
density=True,
range=(0.0, dmax),
bins='doane')
bin_centers = [
float(np.mean(bin_edges[i:i + 1])) for i in range(len(bin_edges) - 1)
]
max_pos = np.argmax(hist)
json_out = {'x': bin_centers, 'y': [float(v) for v in hist]}
# Fit central chi distribution
param = chi.fit(data[data > 0], 2 * ncoils, loc=bin_centers[max_pos])
pdf_fitted = chi.pdf(bin_centers,
*param[:-2],
loc=param[-2],
scale=param[-1])
json_out['y_hat'] = [float(v) for v in pdf_fitted]
# Find t2 (intensity at half width, right side)
ihw = 0.5 * hist[max_pos]
t2idx = 0
for i in range(max_pos + 1, len(bin_centers)):
if hist[i] < ihw:
t2idx = i
break
json_out['x_cutoff'] = float(bin_centers[t2idx])
# Compute goodness-of-fit (gof)
gof = float(
np.abs(hist[t2idx:] - pdf_fitted[t2idx:]).sum() /
len(pdf_fitted[t2idx:]))
# Clip values for sanity
gof = 1.0 if gof > 1.0 else gof
gof = 0.0 if gof < 0.0 else gof
json_out['gof'] = gof
with open(out_file, 'w' if PY3 else 'wb') as ofd:
json.dump(json_out, ofd)
return gof, out_file
def ipe_errors():
#T = fits_table('ipe1_dstn.fit')
#T.cut((T.ra < 249.85) * (T.dec < 17.65))
#print 'Cut to', len(T)
ps = PlotSequence('ipe')
#T = fits_table('ipe2_dstn_1.fit')
T = fits_table('ipe3_dstn_2.fit')
print len(T), 'objects'
print 'Runs', np.unique(T.run)
print 'Camcols', np.unique(T.camcol)
print 'Fields', np.unique(T.field)
T1 = T[T.run == 5183]
T2 = T[T.run == 5224]
plt.clf()
plt.plot(T1.ra, T1.dec, 'r.', alpha=0.1)
plt.plot(T2.ra, T2.dec, 'bx', alpha=0.1)
ps.savefig()
for T in [T1,T2]:
# self-matches:
print 'T:', len(T)
R = 0.5 / 3600.
I,J,d = match_radec(T.ra, T.dec, T.ra, T.dec, R, notself=True)
print len(I), 'matches'
K = (I < J)
I = I[K]
J = J[K]
print len(I), 'symmetric'
print sum(T.field[I] == T.field[J]), 'are in the same field'
#plt.clf()
#plt.plot(T.rowc[I], T.colc[I], 'r.')
#plt.plot(T.rowc[J], T.colc[J], 'b.')
#plt.savefig('ipe2.png')
keep = np.ones(len(T), bool)
keep[I[T.field[I] != T.field[J]]] = False
T.cut(keep)
print 'Cut to', len(T), 'with no matches in other fields'
R = 1./3600.
I,J,d = match_radec(T1.ra, T1.dec, T2.ra, T2.dec, R)
print len(I), 'matches'
dra = (T1.ra[I] - T2.ra[J])*np.cos(np.deg2rad(T1.dec[I])) * 3600.
ddec = (T1.dec[I] - T2.dec[J]) * 3600.
#plt.clf()
#loghist(dra, ddec, 100, range=((-1,1),(-1,1)))
#ps.savefig()
print 'Range of Decs:', T1.dec.min(), T1.dec.max(), T2.dec.min(), T2.dec.max()
ras = np.cos(np.deg2rad(np.append(T1.dec, T2.dec)))
print 'Range of RAscales:', ras.min(), ras.max()
rascale = np.mean(ras)
X1 = np.vstack((T1.ra * rascale, T1.dec)).T
X2 = np.vstack((T2.ra * rascale, T2.dec)).T
inds,d = nearest(X1, X2, R)
J = np.flatnonzero(inds > -1)
I = inds[J]
print 'Nearest-neighbour matches:', len(I)
d = np.sqrt(d[J])
print 'd', d.shape
print 'I,J', len(I), len(J)
print 'd max', np.max(d), 'min', np.min(d)
print 'R', R
assert(np.all(d <= R))
assert(np.all(J >= 0))
assert(np.all(I >= 0))
dx = X1[I] - X2[J]
print 'dx', dx.shape
dr = np.hypot(dx[:,0], dx[:,1])
print 'dr', dr.shape
assert(np.all(dr <= R))
dra = (T1.ra [I] - T2.ra [J]) * rascale * 3600.
ddec = (T1.dec[I] - T2.dec[J]) * 3600.
plt.clf()
loghist(dra, ddec, 100, range=((-1,1),(-1,1)))
ps.savefig()
M1 = T1[I]
M2 = T2[J]
#print 'All T1', T1.rowc.min(), T1.rowc.max(), T1.colc.min(), T1.colc.max()
#print 'Matched T1', T1.rowc[I].min(), T1.rowc[I].max(), T1.colc[I].min(), T1.colc[I].max()
#print 'All T2', T2.rowc.min(), T2.rowc.max(), T2.colc.min(), T2.colc.max()
#print 'Matched T2', T2.rowc[J].min(), T2.rowc[J].max(), T2.colc[J].min(), T2.colc[J].max()
# Errors are in arcsec.
rerr1 = M1.raerr
derr1 = M1.decerr
rerr2 = M2.raerr
derr2 = M2.decerr
hi = 6.
dscale = 1./np.sqrt(2.)
plt.clf()
n,b,p = plt.hist(dscale * np.hypot(dra, ddec) / np.hypot(rerr1, derr1), 100,
range=(0, hi), histtype='step', color='r')
plt.hist(dscale * np.hypot(dra, ddec) / np.hypot(rerr2, derr2), 100,
range=(0, hi), histtype='step', color='b')
xx = np.linspace(0, hi, 500)
from scipy.stats import chi
yy = chi.pdf(xx, 2)
plt.plot(xx, yy * len(dra) * (b[1]-b[0]), 'k-')
plt.xlim(0,hi)
plt.xlabel('N sigma of RA,Dec repeat observations')
plt.ylabel('Number of sources')
ps.savefig()
#loghist(np.hypot(dra, ddec), np.sqrt(np.hypot(rerr1, derr1) * np.hypot(rerr2, derr2)), 100,
# clamp=((0,1),(0,1)))
loghist(np.hypot(dra, ddec), (np.hypot(rerr1, derr1) + np.hypot(rerr2, derr2)) / 2., 100, range=((0,1),(0,1)), clamp=True)
plt.xlabel('Inter-ipe difference: RA,Dec (arcsec)')
plt.ylabel('Photo errors: RA,Dec (arcsec)')
ps.savefig()
loghist(np.log10(np.hypot(dra, ddec)), np.log10((np.hypot(rerr1, derr1) + np.hypot(rerr2, derr2)) / 2.),
100, range=((-3,0),(-3,0)), clamp=True)
plt.xlabel('Inter-ipe difference: log RA,Dec (arcsec)')
plt.ylabel('Photo errors: log RA,Dec (arcsec)')
ps.savefig()
plt.clf()
n,b,p = plt.hist(dscale * np.abs(M1.psfmag_r - M2.psfmag_r) / M1.psfmagerr_r, 100, range=(0,hi), histtype='step', color='r')
plt.xlabel('N sigma of psfmag_r')
xx = np.linspace(0, hi, 500)
yy = 2./np.sqrt(2.*np.pi)*np.exp(-0.5 * xx**2)
print 'yy', sum(yy)
plt.plot(xx, yy * len(M1) * (b[1]-b[0]), 'k-')
ps.savefig()
# Galaxy-star matches
K1 = (M1.type == 3) * (M2.type == 6)
K2 = (M1.type == 6) * (M2.type == 3)
G = merge_tables((M1[K1], M2[K2]))
S = merge_tables((M2[K1], M1[K2]))
print 'G types:', np.unique(G.type)
print 'S types:', np.unique(S.type)
mhi,mlo = 24,10
K = ((G.modelmag_r < mhi) * (S.psfmag_r < mhi) *
(G.modelmag_r > mlo) * (S.psfmag_r > mlo))
print 'Star/gal mismatches with good mags:', np.sum(K)
# gm = G.modelmag_r.copy()
# gm[np.logical_or(gm > mhi, gm < mlo)] = 25.
# sm = S.psfmag_r.copy()
# sm[np.logical_or(sm > mhi, sm < mlo)] = 25.
#
# #loghist(G.modelmag_r[K], S.psfmag_r[K], 100)
# loghist(gm, sm, 100)
loghist(G.modelmag_r, S.psfmag_r, clamp=((mlo,mhi),(mlo,mhi)),
clamp_to=((mlo-1,mhi+1),(mlo-1,mhi+1)))
ax = plt.axis()
plt.axhline(mhi, color='b')
plt.axvline(mhi, color='b')
plt.plot(*([ [min(ax[0],ax[2]), max(ax[1],ax[3])] ]*2) + ['b-',])
plt.axis(ax)
plt.xlabel('Galaxy modelmag_r')
plt.ylabel('Star psfmag_r')
plt.title('Star/Galaxy ipe mismatches')
ps.savefig()
K = ((G.modelmag_r < mhi) * (G.modelmag_r > mlo))
plt.clf()
KK = (G.fracdev_r < 0.5)
kwargs = dict(bins=100, range=(np.log10(0.01), np.log10(30.)), histtype='step')
plt.hist(np.log10(G.exprad_r[K * KK]), color='r', **kwargs)
KK = (G.fracdev_r >= 0.5)
plt.hist(np.log10(G.devrad_r[K * KK]), color='b', **kwargs)
plt.xlabel('*rad_r (arcsec)')
loc,lab = plt.xticks()
plt.xticks(loc, ['%g' % (10.**x) for x in loc])
plt.title('Star/Galaxy ipe mismatches')
ps.savefig()
# Pairs where both are galaxies
K = ((M1.type == 3) * (M2.type == 3))
G1 = M1[K]
G2 = M2[K]
print len(G1), 'pairs where both are galaxies'
#for
plt.clf()
c,cerr = 'modelmag_r', 'modelmagerr_r'
n,b,p = plt.hist(dscale * np.abs(G1.get(c) - G2.get(c)) / G1.get(cerr), 100, range=(0,hi),
histtype='step', color='r')
plt.xlabel('N sigma of ' + c)
yy = np.exp(-0.5 * b**2)
yy *= sum(n) / np.sum(yy)
plt.plot(b, yy, 'k-')
ps.savefig()
loghist(np.abs(G1.get(c) - G2.get(c)), G1.get(cerr), 100, range=((0,2),(0,2)), clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
loghist(np.log10(np.abs(G1.get(c) - G2.get(c))), np.log10(G1.get(cerr)), 100, range=((-3,1),(-3,1)), clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
plt.clf()
loghist(G1.fracdev_r, G2.fracdev_r, 100, range=((0,1),(0,1)), clamp=True)
plt.xlabel('G1 fracdev_r')
plt.ylabel('G2 fracdev_r')
ps.savefig()
dscale = 1.
I = (G1.fracdev_r < 0.5) * (G2.fracdev_r < 0.5)
print sum(I), 'of', len(G1), 'both have fracdev_r < 0.5'
E1 = G1[I]
E2 = G2[I]
I = (G1.fracdev_r >= 0.5) * (G2.fracdev_r >= 0.5)
print sum(I), 'of', len(G1), 'both have fracdev_r >= 0.5'
D1 = G1[I]
D2 = G2[I]
for t,H1,H2 in [('exp',E1,E2),('dev',D1,D2)]:
c,cerr = '%smag_r'%t, '%smagerr_r'%t
dval = np.abs(H1.get(c) - H2.get(c))
derr = H1.get(cerr)
rng = ((0,1),(0,1))
loghist(dval, derr, 100, range=rng, clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
loghist(np.log10(dval), np.log10(derr), 100, range=((-3,0),(-3,0)), clamp=True)
plt.xlabel('Inter-ipe difference: log ' + c)
plt.ylabel('Photo error: log ' + cerr)
ps.savefig()
c,cerr = '%sab_r'%t, '%saberr_r'%t
dval = np.abs(H1.get(c) - H2.get(c))
derr = H1.get(cerr)
rng = ((0,1),(0,1))
loghist(dval, derr, 100, range=rng, clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
loghist(np.log10(dval), np.log10(derr), 100, range=((-3,0),(-3,0)), clamp=True)
plt.xlabel('Inter-ipe difference: log ' + c)
plt.ylabel('Photo error: log ' + cerr)
ps.savefig()
c,cerr = '%srad_r'%t, '%sraderr_r'%t
dval = np.abs(H1.get(c) - H2.get(c))
derr = H1.get(cerr)
rng = ((0,30),(0,30))
loghist(dval, derr, 100, range=rng, clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
loghist(np.log10(dval), np.log10(derr), 100, range=((-2,2),(-2,2)), clamp=True)
plt.xlabel('Inter-ipe difference: log ' + c)
plt.ylabel('Photo error: log ' + cerr)
ps.savefig()
return
I,J,d = match_radec(T.ra, T.dec, T.ra, T.dec, 0.5/3600., notself=True)
print len(I), 'matches'
plt.clf()
loghist((T.ra[I] - T.ra[J])*np.cos(np.deg2rad(T.dec[I])) * 3600.,
(T.dec[I] - T.dec[J])*3600., 100, range=((-1,1),(-1,1)))
plt.savefig('ipe4.png')
K = (I < J)
I = I[K]
J = J[K]
d = d[K]
print 'Cut to', len(I), 'symmetric'
plt.clf()
plt.plot(T.ra, T.dec, 'r.')
plt.plot(T.ra[I], T.dec[I], 'bo', mec='b', mfc='none')
plt.savefig('ipe2.png')
dra,ddec = [],[]
raerr,decerr = [],[]
RC = T.run * 10 + T.camcol
RCF = T.run * 10 * 1000 + T.camcol * 1000 + T.field
for i in np.unique(I):
K = (I == i)
JJ = J[K]
print
print 'Source', i, 'has', len(JJ), 'matches'
print ' ', np.sum(RC[JJ] == RC[i]), 'in the same run/camcol'
print ' ', np.sum(RCF[JJ] == RCF[i]), 'in the same run/camcol/field'
orc = (RC[JJ] != RC[i])
print ' ', np.sum(orc), 'are in other run/camcols'
print ' ', len(np.unique(RC[JJ][orc])), 'unique other run/camcols'
print ' ', len(np.unique(RCF[JJ][orc])), 'unique other run/camcols/fields'
print ' other sources:', JJ
dra.extend((T.ra[JJ] - T.ra[i]) * np.cos(np.deg2rad(T.dec[i])))
ddec.extend(T.dec[JJ] - T.dec[i])
raerr.extend ([T.raerr [i]] * len(JJ))
decerr.extend([T.decerr[i]] * len(JJ))
dra,ddec = np.array(dra), np.array(ddec)
raerr,decerr = np.array(raerr), np.array(decerr)
plt.clf()
plt.hist(np.hypot(dra,ddec) / np.hypot(raerr,decerr), 100)
plt.savefig('ipe3.png')
cdf = chi.cdf(r_grid, df=k)
bin_prob = np.diff(cdf)
return bin_prob
n = 10000
eps_mass = 0.001
k1 = 2
k2 = 100
r_grid1 = get_r_grid(k1, n, get_r_max(k1, eps_mass))
r_grid2 = get_r_grid(k2, n, get_r_max(k2, eps_mass))
bin_prob = get_bin_prob(k2, r_grid2)
chi_pdf = chi.pdf(r_grid2[:-1], df=k2)
r_grid2B = r_grid1**(float(k1) / k2)
c = r_grid2[-1] / r_grid2B[-1]
import matplotlib.pyplot as plt # noqa: E402, mpl gives no other choice :(
plt.plot(r_grid2[:-1], bin_prob / np.diff(r_grid2), 'b.-')
plt.plot(r_grid2[:-1], chi_pdf, 'r.-')
plt.figure()
plt.plot(r_grid1[:-1], bin_prob / np.diff(r_grid1), 'b.-')
chi_pdf = chi.pdf(c * r_grid1[:-1]**(float(k1) / k2), df=k2)
plt.plot(r_grid1[:-1], chi_pdf * (np.diff(r_grid2) / np.diff(r_grid1)), 'r.-')
#x = np.arange(0,len(r_grid2))
from scipy.stats import chi import matplotlib.pyplot as plt fig, ax = plt.subplots(1, 1) # Calculate a few first moments: df = 78 mean, var, skew, kurt = chi.stats(df, moments='mvsk') # Display the probability density function (``pdf``): x = np.linspace(chi.ppf(0.01, df), chi.ppf(0.99, df), 100) ax.plot(x, chi.pdf(x, df), 'r-', lw=5, alpha=0.6, label='chi pdf') # Alternatively, the distribution object can be called (as a function) # to fix the shape, location and scale parameters. This returns a "frozen" # RV object holding the given parameters fixed. # Freeze the distribution and display the frozen ``pdf``: rv = chi(df) ax.plot(x, rv.pdf(x), 'k-', lw=2, label='frozen pdf') # Check accuracy of ``cdf`` and ``ppf``: vals = chi.ppf([0.001, 0.5, 0.999], df) np.allclose([0.001, 0.5, 0.999], chi.cdf(vals, df)) # True # Generate random numbers:
def art_qi2(img, airmask, ncoils=12, erodemask=True,
out_file='qi2_fitting.txt', min_voxels=1e3):
r"""
Calculates :math:`\text{QI}_2`, based on the goodness-of-fit of a centered
:math:`\chi^2` distribution onto the intensity distribution of
non-artifactual background (within the "hat" mask):
.. math ::
\chi^2_n = \frac{2}{(\sigma \sqrt{2})^{2n} \, (n - 1)!}x^{2n - 1}\, e^{-\frac{x}{2}}
where :math:`n` is the number of coil elements.
:param numpy.ndarray img: input data
:param numpy.ndarray airmask: input air mask without artifacts
"""
out_file = op.abspath(out_file)
open(out_file, 'a').close()
if erodemask:
struc = nd.generate_binary_structure(3, 2)
# Perform an opening operation on the background data.
airmask = nd.binary_erosion(airmask, structure=struc).astype(np.uint8)
# Artifact-free air region
data = img[airmask > 0]
# Background can only be fit if we have a min number of voxels
if len(data[data > 0]) < min_voxels:
return 0.0, out_file
# Estimate data pdf
dmax = np.percentile(data[data > 0], 99.9)
hist, bin_edges = np.histogram(data[data > 0], density=True,
range=(0.0, dmax), bins='doane')
bin_centers = [float(np.mean(bin_edges[i:i+1])) for i in range(len(bin_edges)-1)]
max_pos = np.argmax(hist)
json_out = {
'x': bin_centers,
'y': [float(v) for v in hist]
}
# Fit central chi distribution
param = chi.fit(data[data > 0], 2*ncoils, loc=bin_centers[max_pos])
pdf_fitted = chi.pdf(bin_centers, *param[:-2], loc=param[-2], scale=param[-1])
json_out['y_hat'] = [float(v) for v in pdf_fitted]
# Find t2 (intensity at half width, right side)
ihw = 0.5 * hist[max_pos]
t2idx = 0
for i in range(max_pos + 1, len(bin_centers)):
if hist[i] < ihw:
t2idx = i
break
json_out['x_cutoff'] = float(bin_centers[t2idx])
# Compute goodness-of-fit (gof)
gof = float(np.abs(hist[t2idx:] - pdf_fitted[t2idx:]).sum() / len(pdf_fitted[t2idx:]))
# Clip values for sanity
gof = 1.0 if gof > 1.0 else gof
gof = 0.0 if gof < 0.0 else gof
json_out['gof'] = gof
with open(out_file, 'w' if PY3 else 'wb') as ofd:
json.dump(json_out, ofd)
return gof, out_file
def ipe_errors():
#T = fits_table('ipe1_dstn.fit')
#T.cut((T.ra < 249.85) * (T.dec < 17.65))
#print 'Cut to', len(T)
ps = PlotSequence('ipe')
#T = fits_table('ipe2_dstn_1.fit')
T = fits_table('ipe3_dstn_2.fit')
print len(T), 'objects'
print 'Runs', np.unique(T.run)
print 'Camcols', np.unique(T.camcol)
print 'Fields', np.unique(T.field)
T1 = T[T.run == 5183]
T2 = T[T.run == 5224]
plt.clf()
plt.plot(T1.ra, T1.dec, 'r.', alpha=0.1)
plt.plot(T2.ra, T2.dec, 'bx', alpha=0.1)
ps.savefig()
for T in [T1, T2]:
# self-matches:
print 'T:', len(T)
R = 0.5 / 3600.
I, J, d = match_radec(T.ra, T.dec, T.ra, T.dec, R, notself=True)
print len(I), 'matches'
K = (I < J)
I = I[K]
J = J[K]
print len(I), 'symmetric'
print sum(T.field[I] == T.field[J]), 'are in the same field'
#plt.clf()
#plt.plot(T.rowc[I], T.colc[I], 'r.')
#plt.plot(T.rowc[J], T.colc[J], 'b.')
#plt.savefig('ipe2.png')
keep = np.ones(len(T), bool)
keep[I[T.field[I] != T.field[J]]] = False
T.cut(keep)
print 'Cut to', len(T), 'with no matches in other fields'
R = 1. / 3600.
I, J, d = match_radec(T1.ra, T1.dec, T2.ra, T2.dec, R)
print len(I), 'matches'
dra = (T1.ra[I] - T2.ra[J]) * np.cos(np.deg2rad(T1.dec[I])) * 3600.
ddec = (T1.dec[I] - T2.dec[J]) * 3600.
#plt.clf()
#loghist(dra, ddec, 100, range=((-1,1),(-1,1)))
#ps.savefig()
print 'Range of Decs:', T1.dec.min(), T1.dec.max(), T2.dec.min(
), T2.dec.max()
ras = np.cos(np.deg2rad(np.append(T1.dec, T2.dec)))
print 'Range of RAscales:', ras.min(), ras.max()
rascale = np.mean(ras)
X1 = np.vstack((T1.ra * rascale, T1.dec)).T
X2 = np.vstack((T2.ra * rascale, T2.dec)).T
inds, d = nearest(X1, X2, R)
J = np.flatnonzero(inds > -1)
I = inds[J]
print 'Nearest-neighbour matches:', len(I)
d = np.sqrt(d[J])
print 'd', d.shape
print 'I,J', len(I), len(J)
print 'd max', np.max(d), 'min', np.min(d)
print 'R', R
assert (np.all(d <= R))
assert (np.all(J >= 0))
assert (np.all(I >= 0))
dx = X1[I] - X2[J]
print 'dx', dx.shape
dr = np.hypot(dx[:, 0], dx[:, 1])
print 'dr', dr.shape
assert (np.all(dr <= R))
dra = (T1.ra[I] - T2.ra[J]) * rascale * 3600.
ddec = (T1.dec[I] - T2.dec[J]) * 3600.
plt.clf()
loghist(dra, ddec, 100, range=((-1, 1), (-1, 1)))
ps.savefig()
M1 = T1[I]
M2 = T2[J]
#print 'All T1', T1.rowc.min(), T1.rowc.max(), T1.colc.min(), T1.colc.max()
#print 'Matched T1', T1.rowc[I].min(), T1.rowc[I].max(), T1.colc[I].min(), T1.colc[I].max()
#print 'All T2', T2.rowc.min(), T2.rowc.max(), T2.colc.min(), T2.colc.max()
#print 'Matched T2', T2.rowc[J].min(), T2.rowc[J].max(), T2.colc[J].min(), T2.colc[J].max()
# Errors are in arcsec.
rerr1 = M1.raerr
derr1 = M1.decerr
rerr2 = M2.raerr
derr2 = M2.decerr
hi = 6.
dscale = 1. / np.sqrt(2.)
plt.clf()
n, b, p = plt.hist(dscale * np.hypot(dra, ddec) / np.hypot(rerr1, derr1),
100,
range=(0, hi),
histtype='step',
color='r')
plt.hist(dscale * np.hypot(dra, ddec) / np.hypot(rerr2, derr2),
100,
range=(0, hi),
histtype='step',
color='b')
xx = np.linspace(0, hi, 500)
from scipy.stats import chi
yy = chi.pdf(xx, 2)
plt.plot(xx, yy * len(dra) * (b[1] - b[0]), 'k-')
plt.xlim(0, hi)
plt.xlabel('N sigma of RA,Dec repeat observations')
plt.ylabel('Number of sources')
ps.savefig()
#loghist(np.hypot(dra, ddec), np.sqrt(np.hypot(rerr1, derr1) * np.hypot(rerr2, derr2)), 100,
# clamp=((0,1),(0,1)))
loghist(np.hypot(dra, ddec),
(np.hypot(rerr1, derr1) + np.hypot(rerr2, derr2)) / 2.,
100,
range=((0, 1), (0, 1)),
clamp=True)
plt.xlabel('Inter-ipe difference: RA,Dec (arcsec)')
plt.ylabel('Photo errors: RA,Dec (arcsec)')
ps.savefig()
loghist(np.log10(np.hypot(dra, ddec)),
np.log10((np.hypot(rerr1, derr1) + np.hypot(rerr2, derr2)) / 2.),
100,
range=((-3, 0), (-3, 0)),
clamp=True)
plt.xlabel('Inter-ipe difference: log RA,Dec (arcsec)')
plt.ylabel('Photo errors: log RA,Dec (arcsec)')
ps.savefig()
plt.clf()
n, b, p = plt.hist(dscale * np.abs(M1.psfmag_r - M2.psfmag_r) /
M1.psfmagerr_r,
100,
range=(0, hi),
histtype='step',
color='r')
plt.xlabel('N sigma of psfmag_r')
xx = np.linspace(0, hi, 500)
yy = 2. / np.sqrt(2. * np.pi) * np.exp(-0.5 * xx**2)
print 'yy', sum(yy)
plt.plot(xx, yy * len(M1) * (b[1] - b[0]), 'k-')
ps.savefig()
# Galaxy-star matches
K1 = (M1.type == 3) * (M2.type == 6)
K2 = (M1.type == 6) * (M2.type == 3)
G = merge_tables((M1[K1], M2[K2]))
S = merge_tables((M2[K1], M1[K2]))
print 'G types:', np.unique(G.type)
print 'S types:', np.unique(S.type)
mhi, mlo = 24, 10
K = ((G.modelmag_r < mhi) * (S.psfmag_r < mhi) * (G.modelmag_r > mlo) *
(S.psfmag_r > mlo))
print 'Star/gal mismatches with good mags:', np.sum(K)
# gm = G.modelmag_r.copy()
# gm[np.logical_or(gm > mhi, gm < mlo)] = 25.
# sm = S.psfmag_r.copy()
# sm[np.logical_or(sm > mhi, sm < mlo)] = 25.
#
# #loghist(G.modelmag_r[K], S.psfmag_r[K], 100)
# loghist(gm, sm, 100)
loghist(G.modelmag_r,
S.psfmag_r,
clamp=((mlo, mhi), (mlo, mhi)),
clamp_to=((mlo - 1, mhi + 1), (mlo - 1, mhi + 1)))
ax = plt.axis()
plt.axhline(mhi, color='b')
plt.axvline(mhi, color='b')
plt.plot(*([[min(ax[0], ax[2]), max(ax[1], ax[3])]] * 2) + [
'b-',
])
plt.axis(ax)
plt.xlabel('Galaxy modelmag_r')
plt.ylabel('Star psfmag_r')
plt.title('Star/Galaxy ipe mismatches')
ps.savefig()
K = ((G.modelmag_r < mhi) * (G.modelmag_r > mlo))
plt.clf()
KK = (G.fracdev_r < 0.5)
kwargs = dict(bins=100,
range=(np.log10(0.01), np.log10(30.)),
histtype='step')
plt.hist(np.log10(G.exprad_r[K * KK]), color='r', **kwargs)
KK = (G.fracdev_r >= 0.5)
plt.hist(np.log10(G.devrad_r[K * KK]), color='b', **kwargs)
plt.xlabel('*rad_r (arcsec)')
loc, lab = plt.xticks()
plt.xticks(loc, ['%g' % (10.**x) for x in loc])
plt.title('Star/Galaxy ipe mismatches')
ps.savefig()
# Pairs where both are galaxies
K = ((M1.type == 3) * (M2.type == 3))
G1 = M1[K]
G2 = M2[K]
print len(G1), 'pairs where both are galaxies'
#for
plt.clf()
c, cerr = 'modelmag_r', 'modelmagerr_r'
n, b, p = plt.hist(dscale * np.abs(G1.get(c) - G2.get(c)) / G1.get(cerr),
100,
range=(0, hi),
histtype='step',
color='r')
plt.xlabel('N sigma of ' + c)
yy = np.exp(-0.5 * b**2)
yy *= sum(n) / np.sum(yy)
plt.plot(b, yy, 'k-')
ps.savefig()
loghist(np.abs(G1.get(c) - G2.get(c)),
G1.get(cerr),
100,
range=((0, 2), (0, 2)),
clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
loghist(np.log10(np.abs(G1.get(c) - G2.get(c))),
np.log10(G1.get(cerr)),
100,
range=((-3, 1), (-3, 1)),
clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
plt.clf()
loghist(G1.fracdev_r,
G2.fracdev_r,
100,
range=((0, 1), (0, 1)),
clamp=True)
plt.xlabel('G1 fracdev_r')
plt.ylabel('G2 fracdev_r')
ps.savefig()
dscale = 1.
I = (G1.fracdev_r < 0.5) * (G2.fracdev_r < 0.5)
print sum(I), 'of', len(G1), 'both have fracdev_r < 0.5'
E1 = G1[I]
E2 = G2[I]
I = (G1.fracdev_r >= 0.5) * (G2.fracdev_r >= 0.5)
print sum(I), 'of', len(G1), 'both have fracdev_r >= 0.5'
D1 = G1[I]
D2 = G2[I]
for t, H1, H2 in [('exp', E1, E2), ('dev', D1, D2)]:
c, cerr = '%smag_r' % t, '%smagerr_r' % t
dval = np.abs(H1.get(c) - H2.get(c))
derr = H1.get(cerr)
rng = ((0, 1), (0, 1))
loghist(dval, derr, 100, range=rng, clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
loghist(np.log10(dval),
np.log10(derr),
100,
range=((-3, 0), (-3, 0)),
clamp=True)
plt.xlabel('Inter-ipe difference: log ' + c)
plt.ylabel('Photo error: log ' + cerr)
ps.savefig()
c, cerr = '%sab_r' % t, '%saberr_r' % t
dval = np.abs(H1.get(c) - H2.get(c))
derr = H1.get(cerr)
rng = ((0, 1), (0, 1))
loghist(dval, derr, 100, range=rng, clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
loghist(np.log10(dval),
np.log10(derr),
100,
range=((-3, 0), (-3, 0)),
clamp=True)
plt.xlabel('Inter-ipe difference: log ' + c)
plt.ylabel('Photo error: log ' + cerr)
ps.savefig()
c, cerr = '%srad_r' % t, '%sraderr_r' % t
dval = np.abs(H1.get(c) - H2.get(c))
derr = H1.get(cerr)
rng = ((0, 30), (0, 30))
loghist(dval, derr, 100, range=rng, clamp=True)
plt.xlabel('Inter-ipe difference: ' + c)
plt.ylabel('Photo error: ' + cerr)
ps.savefig()
loghist(np.log10(dval),
np.log10(derr),
100,
range=((-2, 2), (-2, 2)),
clamp=True)
plt.xlabel('Inter-ipe difference: log ' + c)
plt.ylabel('Photo error: log ' + cerr)
ps.savefig()
return
I, J, d = match_radec(T.ra, T.dec, T.ra, T.dec, 0.5 / 3600., notself=True)
print len(I), 'matches'
plt.clf()
loghist((T.ra[I] - T.ra[J]) * np.cos(np.deg2rad(T.dec[I])) * 3600.,
(T.dec[I] - T.dec[J]) * 3600.,
100,
range=((-1, 1), (-1, 1)))
plt.savefig('ipe4.png')
K = (I < J)
I = I[K]
J = J[K]
d = d[K]
print 'Cut to', len(I), 'symmetric'
plt.clf()
plt.plot(T.ra, T.dec, 'r.')
plt.plot(T.ra[I], T.dec[I], 'bo', mec='b', mfc='none')
plt.savefig('ipe2.png')
dra, ddec = [], []
raerr, decerr = [], []
RC = T.run * 10 + T.camcol
RCF = T.run * 10 * 1000 + T.camcol * 1000 + T.field
for i in np.unique(I):
K = (I == i)
JJ = J[K]
print
print 'Source', i, 'has', len(JJ), 'matches'
print ' ', np.sum(RC[JJ] == RC[i]), 'in the same run/camcol'
print ' ', np.sum(RCF[JJ] == RCF[i]), 'in the same run/camcol/field'
orc = (RC[JJ] != RC[i])
print ' ', np.sum(orc), 'are in other run/camcols'
print ' ', len(np.unique(RC[JJ][orc])), 'unique other run/camcols'
print ' ', len(np.unique(
RCF[JJ][orc])), 'unique other run/camcols/fields'
print ' other sources:', JJ
dra.extend((T.ra[JJ] - T.ra[i]) * np.cos(np.deg2rad(T.dec[i])))
ddec.extend(T.dec[JJ] - T.dec[i])
raerr.extend([T.raerr[i]] * len(JJ))
decerr.extend([T.decerr[i]] * len(JJ))
dra, ddec = np.array(dra), np.array(ddec)
raerr, decerr = np.array(raerr), np.array(decerr)
plt.clf()
plt.hist(np.hypot(dra, ddec) / np.hypot(raerr, decerr), 100)
plt.savefig('ipe3.png')
def art_qi2(img, airmask, ncoils=12, erodemask=True, out_file='qi2_fitting.txt'):
"""
Calculates **qi2**, the distance between the distribution
of noise voxel (non-artifact background voxels) intensities, and a
centered Chi distribution.
:param numpy.ndarray img: input data
:param numpy.ndarray airmask: input air mask without artifacts
"""
out_file = op.abspath(out_file)
open(out_file, 'a').close()
if erodemask:
struc = nd.generate_binary_structure(3, 2)
# Perform an opening operation on the background data.
airmask = nd.binary_erosion(airmask, structure=struc).astype(np.uint8)
# Artifact-free air region
data = img[airmask > 0]
if np.all(data <= 0):
return 0.0, out_file
# Compute an upper bound threshold
thresh = np.percentile(data[data > 0], 99.5)
# If thresh is too low, for some reason there is no noise
# in the background image (image was preprocessed, etc)
if thresh < 1.0:
return 0.0, out_file
# Threshold image
data = data[data < thresh]
maxvalue = int(data.max())
nbins = maxvalue if maxvalue < 100 else 100
# Estimate data pdf
hist, bin_edges = np.histogram(data, density=True, bins=nbins)
bin_centers = [float(np.mean(bin_edges[i:i+1])) for i in range(len(bin_edges)-1)]
max_pos = np.argmax(hist)
json_out = {
'x': bin_centers,
'y': [float(v) for v in hist]
}
# Fit central chi distribution
param = chi.fit(data, 2*ncoils, loc=bin_centers[max_pos])
pdf_fitted = chi.pdf(bin_centers, *param[:-2], loc=param[-2], scale=param[-1])
json_out['y_hat'] = [float(v) for v in pdf_fitted]
# Find t2 (intensity at half width, right side)
ihw = 0.5 * hist[max_pos]
t2idx = 0
for i in range(max_pos + 1, len(bin_centers)):
if hist[i] < ihw:
t2idx = i
break
json_out['x_cutoff'] = float(bin_centers[t2idx])
# Compute goodness-of-fit (gof)
gof = float(np.abs(hist[t2idx:] - pdf_fitted[t2idx:]).sum() / len(pdf_fitted[t2idx:]))
# Clip values for sanity
gof = 1.0 if gof > 1.0 else gof
gof = 0.0 if gof < 0.0 else gof
json_out['gof'] = gof
with open(out_file, 'w' if PY3 else 'wb') as ofd:
json.dump(json_out, ofd)
return gof, out_file
def main():
"""
The main code - generates the training, validation and test samples
"""
# get the command line args
args = parser()
np.random.seed(args.seed)
# redefine things for conciseness
Tobs = args.Tobs # observation time
fs = args.fsample # sampling frequency
dets = args.detectors # detectors
isnr = args.isnr # integrated SNR
ndet = len(dets) # number of detectors
N = Tobs*fs # the total number of time samples
n = N // 2 + 1 # the number of frequency bins
# make the psds
psds = [gen_psd(fs,Tobs,op='AdvDesign',det=d) for d in args.detectors]
wpsds = (2.0/fs)*np.ones((ndet,n)) # define effective PSD for whited data
# loop over signals
peakSNRvec = [] # store individual detector peak SNRs
intSNRvec = [] # store indiviual detector integrated SNRs
optSNRm = np.zeros(args.Nsig) # store optimal SNR
testSNRm = np.zeros(args.Nsig) # store alternative optimal SNR
woptSNRm = np.zeros(args.Nsig) # store optimal SNR for whitened signal
wtestSNRm = np.zeros(args.Nsig) # store alternative optimal SNR for whitened data
filtSNRm = np.zeros(args.Nsig) # store exact filter measured SNR
nfiltSNRm = np.zeros(args.Nsig) # store exact filter noise only measured SNR
wfiltSNRm = np.zeros(args.Nsig) # store exact filter measured SNR for whitened data
wnfiltSNRm = np.zeros(args.Nsig) # store exact filter noise only measured SNR and whitened data
maxtsSNRm = np.zeros(args.Nsig) # store maximised (over time) measured SNR
nmaxtsSNRm = np.zeros(args.Nsig) # store maximised (over time) noise only measured SNR
wmaxtsSNRm = np.zeros(args.Nsig) # store maximised (over time) measured SNR and whitened data
wnmaxtsSNRm = np.zeros(args.Nsig) # store maximised (over time) noise only measured SNR and whitenee data
print '{}: starting to generate data'.format(time.asctime())
for i in xrange(args.Nsig):
# generate unwhitened time-series
sig,noise,par,hpc = gen_ts(fs,Tobs,isnr,dets)
data = sig + noise
# whiten data
wdata = np.array([whiten_data(s,psd.data.data,fs) for s,psd in zip(data,psds)]).reshape(ndet,-1)
print '{}: Whitened data variance -> {}'.format(time.asctime(),np.std(wdata[0,int(0.1*N):int(0.9*N)]))
# whiten signal and template (ndet signals and 2 templates for +,x)
wsig = np.array([whiten_data(s,psd.data.data,fs) for s,psd in zip(sig,psds)]).reshape(ndet,-1)
whpc = np.array([whiten_data(h,psd.data.data,fs) for h,psd in zip(hpc,psds)]).reshape(2,-1)
peakSNRvec.append(np.max(np.abs(wsig),axis=1))
# whiten noise
wnoise = np.array([whiten_data(s,psd.data.data,fs) for s,psd in zip(noise,psds)]).reshape(ndet,-1)
print '{}: Whitened noise variance -> {}'.format(time.asctime(),np.std(wnoise[0,int(0.1*N):int(0.9*N)]))
# compute optimal SNR in 2 different ways
optSNR = np.array([get_snr(s,Tobs,fs,p.data.data) for s,p in zip(sig,psds)])
testSNRsq = np.array([inner(s,s,Tobs,fs,p.data.data) for s,p in zip(sig,psds)])
optSNRm[i] = np.sqrt(np.sum(optSNR**2))
testSNRm[i] = np.sqrt(np.sum(testSNRsq))
intSNRvec.append(optSNR)
print '{}: optimal multidetector SNR = {}'.format(time.asctime(),optSNRm[i])
print '{}: optimal multidetector SNR (test) = {}'.format(time.asctime(),testSNRm[i])
# compute optimal SNR for whitened signal
woptSNR = np.array([get_snr(s,Tobs,fs,p) for s,p in zip(wsig,wpsds)])
wtestSNRsq = np.array([inner(s,s,Tobs,fs,p) for s,p in zip(wsig,wpsds)])
woptSNRm[i] = np.sqrt(np.sum(woptSNR**2))
wtestSNRm[i] = np.sqrt(np.sum(wtestSNRsq))
print '{}: optimal multidetector SNR (whited signal) = {}'.format(time.asctime(),woptSNRm[i])
print '{}: optimal multidetector SNR (whitened data test) = {}'.format(time.asctime(),wtestSNRm[i])
# compute measured SNR using the exact template
filtSNR = np.array([meas_snr(d,s,np.zeros(N),Tobs,fs,p.data.data) for d,s,p in zip(data,sig,psds)])
filtSNRm[i] = np.sqrt(np.sum(filtSNR**2))
print '{}: exact template multidetector SNR = {}'.format(time.asctime(),filtSNRm[i])
# compute measured SNR on whitened data using the exact template
wfiltSNR = np.array([meas_snr(d,s,np.zeros(N),Tobs,fs,p) for d,s,p in zip(wdata,wsig,wpsds)])
wfiltSNRm[i] = np.sqrt(np.sum(wfiltSNR**2))
print '{}: exact template multidetector SNR (whitened data) = {}'.format(time.asctime(),wfiltSNRm[i])
# compute measured SNR *on noise only* using the exact template
nfiltSNR = np.array([meas_snr(n,s,np.zeros(N),Tobs,fs,p.data.data) for n,s,p in zip(noise,sig,psds)])
nfiltSNRm[i] = np.sqrt(np.sum(nfiltSNR**2))
print '{}: exact template noise only multidetector SNR = {}'.format(time.asctime(),nfiltSNRm[i])
# compute measured SNR *on noise only* whitened data using the exact template
wnfiltSNR = np.array([meas_snr(n,s,np.zeros(N),Tobs,fs,p) for n,s,p in zip(wnoise,wsig,wpsds)])
wnfiltSNRm[i] = np.sqrt(np.sum(wnfiltSNR**2))
print '{}: exact template noise only multidetector SNR (whitened data) = {}'.format(time.asctime(),wnfiltSNRm[i])
# compute measured SNR as a function of time
# using LOSC convolution method
tsSNR = np.array([snr_ts(d,hpc[0],hpc[1],Tobs,fs,p.data.data) for d,p in zip(data,psds)])
maxtsSNR = np.max(tsSNR,axis=1)
maxtsSNRm[i] = np.sqrt(np.sum(maxtsSNR**2))
print '{}: maximised multidetector SNR = {}'.format(time.asctime(),maxtsSNRm[i])
# compute measured SNR *on noise only* as a function of time
# using LOSC convolution method
ntsSNR = np.array([snr_ts(n,hpc[0],hpc[1],Tobs,fs,p.data.data) for n,p in zip(noise,psds)])
nmaxtsSNR = np.max(ntsSNR,axis=1)
nmaxtsSNRm[i] = np.sqrt(np.sum(nmaxtsSNR**2))
print '{}: maximised noise only multidetector SNR = {}'.format(time.asctime(),nmaxtsSNRm[i])
# compute measured SNR as a function of time on whitened data
# using LOSC convolution method
wtsSNR = np.array([snr_ts(d,whpc[0],whpc[1],Tobs,fs,p) for d,p in zip(wdata,wpsds)])
wmaxtsSNR = np.max(wtsSNR,axis=1)
wmaxtsSNRm[i] = np.sqrt(np.sum(wmaxtsSNR**2))
print '{}: maximised multidetector SNR = {}'.format(time.asctime(),wmaxtsSNRm[i])
# compute measured SNR *on noise only* as a function of time on
# whitened data using LOSC convolution method
wntsSNR = np.array([snr_ts(n,whpc[0],whpc[1],Tobs,fs,p) for n,p in zip(wnoise,wpsds)])
wnmaxtsSNR = np.max(wntsSNR,axis=1)
wnmaxtsSNRm[i] = np.sqrt(np.sum(wnmaxtsSNR**2))
print '{}: maximised noise only multidetector SNR = {}'.format(time.asctime(),wnmaxtsSNRm[i])
# make distribution plots
nbins = int(np.sqrt(args.Nsig))
temp = np.linspace(0,isnr+5,1000)
plt.figure()
plt.hist(filtSNRm,nbins,normed=True,alpha=0.5)
plt.hist(maxtsSNRm,nbins,normed=True,alpha=0.5)
plt.hist(nfiltSNRm,nbins,normed=True,alpha=0.5)
plt.hist(nmaxtsSNRm,nbins,normed=True,alpha=0.5)
plt.plot(temp,norm.pdf(temp,loc=isnr),'k')
plt.plot(temp,chi.pdf(temp,2),'k')
plt.xlim([0,np.max(temp)])
plt.savefig('./verify.png')
# make distribution plots
plt.figure()
plt.hist(wfiltSNRm,nbins,normed=True,alpha=0.5)
plt.hist(wmaxtsSNRm,nbins,normed=True,alpha=0.5)
plt.hist(wnfiltSNRm,nbins,normed=True,alpha=0.5)
plt.hist(wnmaxtsSNRm,nbins,normed=True,alpha=0.5)
plt.plot(temp,norm.pdf(temp,loc=isnr),'k')
plt.plot(temp,chi.pdf(temp,2),'k')
plt.xlim([0,np.max(temp)])
plt.savefig('./verify_whitened.png')
# make peak vs int snr plots
peakSNRvec = np.array(peakSNRvec).flatten()
intSNRvec = np.array(intSNRvec).flatten()
plt.figure()
plt.plot(peakSNRvec,intSNRvec,'.')
plt.xlim([0,1.2*np.max(peakSNRvec)])
plt.ylim([0,1.2*np.max(intSNRvec)])
plt.savefig('./peakvsint.png')
plt.figure()
plt.hist(intSNRvec/peakSNRvec,nbins,normed=True,alpha=0.5)
plt.savefig('./peakintratio.png')
def _pdf_notTruncated(self, z, dps):
scale = self._scale
k = self._k
dps = self._dps
return chi.pdf(z/scale, k)
return y
def kdeplot(x, lower=-np.inf, upper=np.inf):
x_grid = np.linspace(max(lower, np.min(x)), min(upper, np.max(x)), 1000)
k = gaussian_kde(x)
dd = k(x_grid)
return x_grid, dd
# plot distn of r for diff D, also do same for uniform ball
x = np.linspace(0, 2.0, 1000)
plt.figure()
for dim, color in zip(D, colors):
pdf = chi.pdf(np.sqrt(dim) * x, df=dim)
plt.plot(x, pdf / np.max(pdf), color, label='$D=%d$' % dim)
plt.xlabel('$r$ (rescaled)')
plt.ylabel('$\chi$ pdf (rescaled)')
plt.legend()
plt.title('distributions on radius in gaussians')
plt.savefig('fig0' + ext, dpi=300)
plt.figure()
for dim, color in zip(D, colors):
pdf = powerlaw.pdf(np.sqrt(dim) * x, a=dim, loc=0, scale=np.sqrt(dim))
plt.plot(x, pdf / np.max(pdf), color, label='$D=%d$' % dim)
plt.xlabel('$r$ (rescaled)')
plt.ylabel('power law pdf (rescaled)')
plt.legend()
plt.title('distributions on radius in uniform ball')