def shift(yy1, yy2):
l = [] # array of the sum of squares
shift_factor = np.linspace(-5, 5, 1001)
for i in shift_factor:
nx = x + i
nyy2 = list(np.ones(len(yy2)))
squaresum = []
for j in range(len(yy2)):
ny2 = spl(nx, yy2[j]) # same shape
nyy2[j] = ny2(x) # new y-values
squaresum.append(sum(abs(yy1[j]-nyy2[j])))
l.append(sum(squaresum))
minindex = l.index(min(l))
sf = list(shift_factor)[minindex]
nx = x + sf
nyy2 = []
for i in range(len(yy2)):
ny2 = spl(nx, yy2[i])
nyy2.append(ny2(x))
return nyy2, sf
def interpolateCurve(self):
if self.data is None:
self.states()
values = np.array(self.data)
np.swapaxes(values, 1, 0)
s = []
lastPoint = values[:, 0]
lastS = 0
for i, point in enumerate(values[0]):
point = values[:, i]
lastS = math.sqrt(
np.dot(np.array(point - lastPoint),
np.array(point - lastPoint))) + lastS
s.append(lastS)
lastPoint = point
s2 = np.linspace(s[0], s[-1], len(s))
interpedData = []
for dim in range(0, len(values[:, 1])):
data = np.array(values[dim, :]).reshape(len(s), 1).squeeze()
dimData = []
s2Index = 0
lastIndex = 0
for i, value in enumerate(data):
if i % 5 == 0 and i + 5 <= len(s):
if i + 5 == len(s):
tck, u = spl(np.array(data[i - 2:i + 4]).reshape(1, 6),
u=np.array(s[i - 2:i + 4]).reshape(
6, 1).flatten(),
k=5)
newValues = splev(s2[lastIndex:-1], tck)
for xi in newValues:
for xij in xi:
dimData.append(xij)
else:
tck, u = spl(np.array(data[i:i + 6]).reshape(1, 6),
u=np.array(s[i:i + 6]).reshape(
6, 1).flatten(),
k=5)
while s2[s2Index] < s[i + 5]:
s2Index += 1
if s2Index != lastIndex:
newValues = splev(s2[lastIndex:s2Index], tck)
for xi in newValues:
for xij in xi:
dimData.append(xij)
lastIndex = s2Index
interpedData.append(dimData)
interpedData = np.array(interpedData)
return interpedData.transpose()
def _findBest(snType, wave, flux):
flux = flux[wave > 4000][wave[wave > 4000] < 7500]
wave = wave[wave > 4000][wave[wave > 4000] < 7500]
mySpline = spl(wave, flux, k=5, ext=1)
tempSpline = mySpline(wave)
splineRemoved = np.log10(flux / tempSpline)
sne, types = np.loadtxt(os.path.join(__dir__, 'data', 'spectra',
'snTypes.ref'),
dtype='str',
unpack=True)
mySpecs = sne[types == snType]
bestChi = np.inf
bestSpec = None
bestConst = None
for spec in mySpecs:
data = _readSpec(
os.path.join(__dir__, 'data', 'spectra', spec + '.lnw'))
x = [float(col[2:]) for col in data.colnames if col != 'wave']
if len(x) < 2:
continue
y = data['wave']
data.remove_column('wave')
func = interp2d(x, y, np.array([list(r) for r in np.array(data)]))
tempFlux = np.transpose(func(0, wave))[0]
res = minimize(_specChi, np.array([1]), args=(splineRemoved, tempFlux))
if res.fun < bestChi:
bestChi = res.fun
bestSpec = spec
bestConst = res.x
return (_readSpec(
os.path.join(__dir__, 'data', 'spectra',
bestSpec + '.lnw')), bestConst)
def create_score2prob_lin(p_cnv):
"""
create the function that will transform a cnv score in a p-value
of being a cnv
Just a linear approximation
"""
kpcnv = p_cnv.keys()
lin_funct = spl(kpcnv, p_cnv.values(), k=1)
#x = np.arange(0,50,1)
#plt.plot(x, lin_funct(x), '-', p_cnv.keys(), p_cnv.values(), '+')
(x1, x2) = (kpcnv[0], kpcnv[-1])
a = (lin_funct(x2) - lin_funct(x1)) / (x2 - x1)
b = lin_funct(x2) - a * x2
x_where_y_is_1 = (1. - b) / a
x_where_y_is_0 = (-b) / a
def sc2prob(x):
# x should not be a string
# assert not isinstance(x, six.string_types), "{} is str".format(x)
# but should be a number
assert isinstance(x, numbers.Real), \
"{} is not a number: type {}".format(x,type(x))
if x < x_where_y_is_0:
return 0.
elif x < x_where_y_is_1:
return float(lin_funct(x))
else:
return 1.
return sc2prob
def compress_pdf(self, data):
""" Compress the PDF
"""
simple_spline = np.array([], dtype=np.float16)
index_array = np.array([], dtype=np.uint8)
for i in xrange(1, data.shape[1]):
spline = spl(data[:, 0], data[:, i], ext=1)
int_zw = spline(self.zlim)
index_over = np.where(int_zw > self.thresh)[0]
if len(index_over) < 2:
print len(index_over)
raise ValueError('The grid spacing is too big. This can happen for instance if the PDF resembles a delta function.')
try:
index_array = np.append(index_array,
np.array([index_over[0], index_over[-1]], dtype=np.float16))
except IndexError as e:
print "Decrease the grid spacing! This can happen if the PDFs are a bit undersmoothed."
print e.message
#which points are above the threshold?
data_over = np.array(int_zw[index_over[0]:index_over[-1]], dtype=np.float16)
simple_spline = np.append(simple_spline, data_over)
return simple_spline, index_array
def _fit_cloud(self):
from scipy.interpolate import InterpolatedUnivariateSpline as spl
def _line(n, m, i):
x = np.mean(m[i]) if m[i] else None
k_p = n - 1
while k_p > 0 and k_p not in self:
k_p -= 1
x_up = self[k_p][not i](0) if k_p >= 0 else x
if x is None or x > x_up:
x = x_up
return spl([0, 1], [x] * 2, k=1)
self.clear()
self.update(copy.deepcopy(self.cloud))
for k, v in sorted(self.items()):
v[0] = _line(k, v, 0)
if len(v[1][0]) > 2:
v[1] = _gspv_interpolate_cloud(*v[1])
elif v[1][1]:
v[1] = spl([0, 1], [np.mean(v[1][1])] * 2, k=1)
else:
v[1] = self[k - 1][0]
def perform_compression(self, data):
""" Compress the PDF
"""
simple_spline = np.array([])
index_array = np.array([])
for i in xrange(1, data.shape[1]):
spline = spl(data[:, 0], data[:, i], ext=1)
int_zw = spline(zlim)
index_over = np.where(int_zw > self.thresh)[0]
if len(index_over) < 2:
print len(index_over)
#raise ValueError('The grid spacing is too big. This can happen for instance if the PDF resembles a delta function.')
continue
try:
index_array = np.append(index_array,
np.array([index_over[0], index_over[-1]]))
except IndexError as e:
print "Decrease the grid spacing! This can happen if the PDFs are a bit undersmoothed."
print e.message
data_over = np.array(int_zw[index_over[0]:index_over[-1]])
simple_spline = np.append(simple_spline, data_over)
simple_spline = np.array(simple_spline, dtype='float16')
index_array = np.array(index_array, dtype='uint8')
return simple_spline, index_array
def _line(n, m, i):
x = np.mean(m[i]) if m[i] else None
k_p = n - 1
while k_p > 0 and k_p not in self:
k_p -= 1
x_up = self[k_p][not i](0) if k_p >= 0 else x
if x is None or x > x_up:
x = x_up
return spl([0, 1], [x] * 2, k=1)
def get_probs(z, pdf, z1, z2):
pdf = pdf / numpy.sum(pdf)
PP = spl(z, pdf, bounds_error=False, fill_value=0.0)
dzo = z[1] - z[0]
dz = 0.001
Ndz = int((z2 - z1) / dz)
A = 0
for i in xrange(Ndz):
A += dz * PP((z1) + dz / 2. + dz * i)
return A / dzo
def get_probs(z, pdf, z1, z2):
pdf = pdf / np.sum(pdf)
PP = spl(z, pdf, bounds_error=False, fill_value=0.0)
dzo = z[1] - z[0]
dz = 0.001
Ndz = int((z2 - z1) / dz)
A = 0
for i in range(Ndz):
A += dz * PP((z1) + dz / 2. + dz * i)
return A / dzo
def Full_Fisher(N_bins, bin_dists, f_sky, type_sam):
"""
Calculates the Full Fisher matrix for N bins
--------------------------------------------
Inputs:
--------------------------------------------
Outputs:
"""
nz = 10000 #number of steps to use for the radial/redshift integration
zarray = np.linspace(0, 4.0, nz)
z = zarray[1:-1]
# Dimension of the Fisher matrix
dim = N_bins + 2
# The total Fihser matrix has dimensions (N_bins + 2)*(N_bins + 2)
Full_Fish = np.zeros([dim, dim])
for i in range(N_bins):
i_bin = i + 1 #the bin number - starting from one
# Redshift distribution in the bin
bin_z = bin_dists[i]
mean_z = np.mean(bin_z)
bias = 1.0 + mean_z
if (type_sam == 0):
N_gal = 32.0 * np.float(len(bin_z))
else:
N_gal = np.float(len(bin_z))
# ==========================================================
# ==========================================================
# take a histogram for the redshift distribution
y_like, x_like = np.histogram(bin_z, bins=200, density=True)
x_like = x_like[:-1]
bin_sp = spl(x_like, y_like, s=0.0, ext=1) #Interpolate
Dz = 0.0 # This can change
n_z = bin_sp(z -
Dz) # this is the normalized n(z) in the range (0.0,4.0)
# Calculate the derivative now
der_spl = bin_sp.derivative()
dn_dDz = -der_spl(z)
Full_Fish += Fish_single_bin(N_bins, i_bin, mean_z, bias, n_z, dn_dDz,
f_sky, N_gal)
return Full_Fish
def get_prob_Nz(z, pdf, zbins):
pdf = pdf / np.sum(pdf)
PP = spl(z, pdf, bounds_error=False, fill_value=0.0)
dzo = z[1] - z[0]
dz = 0.001
Ndz = int((zbins[1] - zbins[0]) / dz)
Nzt = np.zeros(len(zbins) - 1)
for j in range(len(Nzt)):
for i in range(Ndz):
Nzt[j] += dz * PP((zbins[j]) + dz / 2. + dz * i)
return Nzt / dzo
def get_prob_Nz(z, pdf, zbins):
pdf = pdf / numpy.sum(pdf)
PP = spl(z, pdf, bounds_error=False, fill_value=0.0)
dzo = z[1] - z[0]
dz = 0.001
Ndz = int((zbins[1] - zbins[0]) / dz)
Nzt = numpy.zeros(len(zbins) - 1)
for j in xrange(len(Nzt)):
for i in xrange(Ndz):
Nzt[j] += dz * PP((zbins[j]) + dz / 2. + dz * i)
return Nzt / dzo
def stack_pdf(self, zvalues, compressed_pdfs, zrange=None, weights=None):
"""Stack the PDFs with weights
zlims: integrate PDF in the ranges
weights: additional weights
"""
uncompr = self.uncompress(compressed_pdfs[0],
compressed_pdfs[1])
if zrange is not None:
zrange_weights = self.get_zrange_weights(uncompr, zrange)
if weights is None:
weights = zrange_weights
else:
weights *= zrange_weights
data_stack = np.average(uncompr, axis=1, weights=weights)
model = spl(self.zlim, data_stack, ext=1)
model = spl(self.zlim, data_stack/model.integral(self.zmin, self.zmax), ext=1)
outputpdf = model(zvalues)
return np.column_stack((zvalues, outputpdf))
def get_zrange_weights(self, uncompress, zrange, z=None):
"""Calculate the weights by integrating in
the predefined zrange
"""
if z is None:
z = self.zlim
zrange_weights = np.ones((uncompress.shape[1],))
for i in xrange(len(zrange_weights)):
model = spl(z, uncompress[:, i], ext=1)
zrange_weights[i] = model.integral(zrange[0], zrange[1])
return zrange_weights
def interpolate(self, crossingPoints):
points = crossingPoints.transpose()
tck,u = spl(points,k=5)
curve = splev(np.linspace(0,1,1000),tck)
curve = np.array(curve).transpose()
min = self.disPlan(curve[0])
minPoint = 0
for point in curve:
tmp = self.disPlan(point)
if(min>tmp):
min = tmp
minPoint = point
return minPoint
def plot_hdr1d(data, prob, bins=20, smooth=True, **kwargs):
if np.isscalar(bins):
bins=np.linspace(min(data), max(data), bins)
elif type(bins) is dict:
bins=bins[data.name]
xp=bins[:-1] + np.diff(bins)/2.
yp=np.zeros(len(xp))
for i, (l, u) in enumerate(zip(bins[:-1], bins[1:])):
p=prob[(data >= l) & (data < u)]
yp[i]=max(p) if len(p) != 0 else 0.
x=np.linspace(xp[0], xp[-1], 100)
if smooth:
plt.plot(x, spl(xp, yp)(x), **kwargs)
else:
plt.plot(xp, yp, **kwargs)
def get_area(z, pdf, z1, z2):
"""
Compute area under photo-z Pdf between z1 and z2, PDF must add to 1
:param float z: redshift
:param float pdf: photo-z PDF
:param float z1: Lower boundary
:param float z2: Upper boundary
:return: area between z1 and z2
:rtype: float
"""
PP = spl(z, pdf, bounds_error=False, fill_value=0.0)
area = rom(PP, z1, z2, tol=1.0e-05, rtol=1.0e-05)
dz = z[1] - z[0]
return area / dz
def get_area(z, pdf, z1, z2):
"""
Compute area under photo-z Pdf between z1 and z2, PDF must add to 1
:param float z: redshift
:param float pdf: photo-z PDF
:param float z1: Lower boundary
:param float z2: Upper boundary
:return: area between z1 and z2
:rtype: float
"""
PP = spl(z, pdf, bounds_error=False, fill_value=0.0)
area = rom(PP, z1, z2, tol=1.0e-05, rtol=1.0e-05)
dz = z[1] - z[0]
return area / dz
def spline_fit(x, y, xmin, xmax, n):
# Slice out region
xc = x[np.nonzero(x == xmin)[0][0]:np.nonzero(x == xmax)[0][0]]
yc = y[np.nonzero(x == xmin)[0][0]:np.nonzero(x == xmax)[0][0]]
# Compute spline fit
if (xmax + n) > len(x):
print('\n')
print('NUMBER OF POINTS EXCEEDS INPUT SIZE')
print('\n')
else:
# Initialize output variables
xcur = np.zeros(n)
ycur = np.zeros(n)
yspl = []
# Compute fit
for i in np.arange(0, len(xc) + n, n):
# Slice out n points per step, compute third order cubic spline
xcur = x[np.nonzero(x == xmin)[0][0] +
i:np.nonzero(x == xmin)[0][0] + i + n]
ycur = y[np.nonzero(x == xmin)[0][0] +
i:np.nonzero(x == xmin)[0][0] + i + n]
ys = spl(xcur, ycur, k=3)
if i == len(xc) - 1:
yspl.extend(ys(xcur)[0])
break
else:
yspl.extend(ys(xcur))
# Store data
yspl = np.float64(yspl[:len(xc)])
# Return output
return xc, yc, yspl
def compute_error(z, pdf, zv):
"""
Computes the error in the PDF calculation using a reference values from PDF
it computes the 68% percentile limit around this value
:param float z: redshift
:param float pdf: photo-z PDF
:param float zv: Reference value from PDF (can be mean, mode, median, etc.)
:return: error associated to reference value
:rtype: float
"""
res = 0.001
PP = spl(z, pdf, bounds_error=False, fill_value=0.0)
dz = z[1] - z[0]
j = 0
area = 0
while area <= 0.68:
j += 1
za = zv - res * j
zb = zv + res * j
area = rom(PP, za, zb, tol=1.0e-04, rtol=1.0e-04) / dz
return j * res
def compute_error2(z, pdf, zv):
L1 = 0.0001
L2 = (max(z) - min(z)) / 2.
PP = spl(z, pdf, bounds_error=False, fill_value=0.0)
dz = z[1] - z[0]
eps = 0.05
za1 = zv - L1
zb1 = zv + L1
area = 0
LM = L2
while abs(area - 0.68) > eps:
za2 = zv - LM
zb2 = zv + LM
area = rom(PP, za2, zb2, tol=1.0e-04, rtol=1.0e-04) / dz
Lreturn = LM
if area > 0.68:
L2 = LM
LM = (L1 + L2) / 2.
else:
L1 = LM
LM = (L1 + L2) / 2.
return Lreturn
def compute_error2(z, pdf, zv):
L1 = 0.0001
L2 = (max(z) - min(z)) / 2.
PP = spl(z, pdf, bounds_error=False, fill_value=0.0)
dz = z[1] - z[0]
eps = 0.05
za1 = zv - L1
zb1 = zv + L1
area = 0
LM = L2
while abs(area - 0.68) > eps:
za2 = zv - LM
zb2 = zv + LM
area = rom(PP, za2, zb2, tol=1.0e-04, rtol=1.0e-04) / dz
Lreturn = LM
if area > 0.68:
L2 = LM
LM = (L1 + L2) / 2.
else:
L1 = LM
LM = (L1 + L2) / 2.
return Lreturn
def compute_error(z, pdf, zv):
"""
Computes the error in the PDF calculation using a reference values from PDF
it computes the 68% percentile limit around this value
:param float z: redshift
:param float pdf: photo-z PDF
:param float zv: Reference value from PDF (can be mean, mode, median, etc.)
:return: error associated to reference value
:rtype: float
"""
res = 0.001
PP = spl(z, pdf, bounds_error=False, fill_value=0.0)
dz = z[1] - z[0]
j = 0
area = 0
while area <= 0.68:
j += 1
za = zv - res * j
zb = zv + res * j
area = rom(PP, za, zb, tol=1.0e-04, rtol=1.0e-04) / dz
return j * res
def create_score2prob_lin(p_cnv):
"""
create the function that will transform a cnv score in a p-value
of being a cnv
Just a linear approximation
"""
kpcnv = p_cnv.keys()
lin_funct = spl(kpcnv, p_cnv.values(), k=1)
#x = np.arange(0,50,1)
#plt.plot(x, lin_funct(x), '-', p_cnv.keys(), p_cnv.values(), '+')
(x1, x2) = (kpcnv[0], kpcnv[-1])
a = (lin_funct(x2) - lin_funct(x1))/(x2 - x1)
b = lin_funct(x2) - a*x2
x_where_y_is_1 = (1. - b)/a
x_where_y_is_0 = (-b)/a
def sc2prob(x):
# x should not be a string
# assert not isinstance(x, six.string_types), "{} is str".format(x)
# but should be a number
assert isinstance(x, numbers.Real), \
"{} is not a number: type {}".format(x,type(x))
if x < x_where_y_is_0:
return 0.
elif x < x_where_y_is_1:
return float(lin_funct(x))
else:
return 1.
return sc2prob
def estimate_continuum(wave,
flux,
mask=True,
minmask=None,
maxmask=None,
deg=1,
flat=True,
spline=False,
nknots=None):
"""
perform polynomial fit for local continuum subtraction
"""
if flat:
continuum = np.min(flux) + np.zeros(wave.shape[0])
return continuum
else:
ww = wave
ff = flux
if mask is True:
if minmask is not None and maxmask is not None:
kk = np.where((wave > minmask) & (wave < maxmask))
ww = np.delete(wave, kk)
ff = np.delete(flux, kk)
print "No of points for estimating continuum", ww.shape[
0], 'of', wave.shape[0]
if spline:
from scipy.interpolate import UnivariateSpline as spl
if nknots is None:
nknots = len(ww)
spfit = spl(ww, ff, s=nknots) #knots downsample by a factor of 10
continuum = spfit(wave)
else: #- poly fit
coeff = np.polyfit(ww, ff, deg) #- first order, deg=1
continuum = np.polyval(coeff, wave)
return continuum
def get_N0_iter(qe_key: str,
nlev_t: float,
nlev_p: float,
beam_fwhm: float,
cls_unl_fid: dict,
lmin_ivf,
lmax_ivf,
itermax,
cls_unl_dat=None,
lmax_qlm=None,
ret_delcls=False,
datnoise_cls: dict or None = None,
unlQE=False,
version='1'):
"""Iterative lensing-N0 estimate
Calculates iteratively partially lensed spectra and lensing noise levels.
This uses the python camb package to get the partially lensed spectra.
This makes no assumption on response = 1 / noise hence is about twice as slow as it could be in standard cases.
Args:
qe_key: QE estimator key
nlev_t: temperature noise level (in :math:`\mu `K-arcmin)
nlev_p: polarisation noise level (in :math:`\mu `K-arcmin)
beam_fwhm: Gaussian beam full width half maximum in arcmin
cls_unl_fid(dict): unlensed CMB power spectra
lmin_ivf: minimal CMB multipole used in the QE
lmax_ivf: maximal CMB multipole used in the QE
itermax: number of iterations to perform
lmax_qlm(optional): maximum lensing multipole to consider. Defaults to :math:`2 lmax_ivf`
ret_delcls(optional): returns the partially delensed CMB cls as well if set
datnoise_cls(optional): feeds in custom noise spectra to the data. The nlevs and beam only apply to the filtering in this case
Returns
Array of shape (itermax + 1, lmax_qlm + 1) with all iterated N0s. First entry is standard N0.
Note: This assumes the unlensed spectra are known
#FIXME: this is requiring the full camb python package for the lensed spectra calc.
"""
assert qe_key in ['p_p', 'p', 'ptt'], qe_key
try:
from camb.correlations import lensed_cls
except ImportError:
assert 0, "could not import camb.correlations.lensed_cls"
if lmax_qlm is None:
lmax_qlm = 2 * lmax_ivf
lmax_qlm = min(lmax_qlm, 2 * lmax_ivf)
lmin_ivf = max(lmin_ivf, 1)
transfi2 = utils.cli(
hp.gauss_beam(beam_fwhm / 180. / 60. * np.pi, lmax=lmax_ivf))**2
llp2 = np.arange(lmax_qlm + 1, dtype=float)**2 * np.arange(
1, lmax_qlm + 2, dtype=float)**2 / (2. * np.pi)
if datnoise_cls is None:
datnoise_cls = dict()
if qe_key in ['ptt', 'p']:
datnoise_cls['tt'] = (nlev_t * np.pi / 180. / 60.)**2 * transfi2
if qe_key in ['p_p', 'p']:
datnoise_cls['ee'] = (nlev_p * np.pi / 180. / 60.)**2 * transfi2
datnoise_cls['bb'] = (nlev_p * np.pi / 180. / 60.)**2 * transfi2
N0s_biased = []
N0s_unbiased = []
N1s_biased = []
N1s_unbiased = []
delcls_fid = []
delcls_true = []
N0_unbiased = np.inf
N1_unbiased = np.inf
dls_unl_fid, cldd_fid = cls2dls(cls_unl_fid)
cls_len_fid = dls2cls(lensed_cls(dls_unl_fid, cldd_fid))
if cls_unl_dat is None:
cls_unl_dat = cls_unl_fid
cls_len_true = cls_len_fid
else:
dls_unl_true, cldd_true = cls2dls(cls_unl_dat)
cls_len_true = dls2cls(lensed_cls(dls_unl_true, cldd_true))
cls_plen_true = cls_len_true
for irr, it in utils.enumerate_progress(range(itermax + 1)):
dls_unl_true, cldd_true = cls2dls(cls_unl_dat)
dls_unl_fid, cldd_fid = cls2dls(cls_unl_fid)
if it == 0:
rho_sqd_phi = 0.
else:
# The cross-correlation coefficient is identical for the Rfid-biased QE or the rescaled one
rho_sqd_phi = np.zeros(len(cldd_true))
rho_sqd_phi[:lmax_qlm + 1] = cldd_true[:lmax_qlm + 1] * utils.cli(
cldd_true[:lmax_qlm + 1] + llp2 *
(N0_unbiased[:lmax_qlm + 1] + N1_unbiased[:lmax_qlm + 1]))
if 'wE' in version:
assert qe_key in ['p_p']
if it == 0:
print('including imperfect knowledge of E in iterations')
slic = slice(lmin_ivf, lmax_ivf + 1)
rho_sqd_E = np.zeros(len(dls_unl_true[:, 1]))
rho_sqd_E[slic] = cls_unl_dat['ee'][slic] * utils.cli(
cls_plen_true['ee'][slic] + datnoise_cls['ee'][slic])
dls_unl_fid[:, 1] *= rho_sqd_E
dls_unl_true[:, 1] *= rho_sqd_E
cldd_fid *= rho_sqd_phi
cldd_true *= rho_sqd_phi
cls_plen_fid_resolved = dls2cls(lensed_cls(dls_unl_fid, cldd_fid))
cls_plen_true_resolved = dls2cls(
lensed_cls(dls_unl_true, cldd_true))
cls_plen_fid = {
ck: cls_len_fid[ck] - (cls_plen_fid_resolved[ck] -
cls_unl_fid[ck][:len(cls_len_fid[ck])])
for ck in cls_len_fid.keys()
}
cls_plen_true = {
ck:
cls_len_true[ck] - (cls_plen_true_resolved[ck] -
cls_unl_dat[ck][:len(cls_len_true[ck])])
for ck in cls_len_true.keys()
}
else:
cldd_true *= (1. - rho_sqd_phi) # The true residual lensing spec.
cldd_fid *= (1. - rho_sqd_phi
) # What I think the residual lensing spec is
cls_plen_fid = dls2cls(lensed_cls(dls_unl_fid, cldd_fid))
cls_plen_true = dls2cls(lensed_cls(dls_unl_true, cldd_true))
cls_filt = cls_plen_fid if not unlQE else cls_unl_fid
cls_w = cls_plen_fid if not unlQE else cls_unl_fid
cls_f = cls_plen_true
fal = {}
dat_delcls = {}
if qe_key in ['ptt', 'p']:
fal['tt'] = cls_filt['tt'][:lmax_ivf + 1] + (
nlev_t * np.pi / 180. / 60.)**2 * transfi2
dat_delcls['tt'] = cls_plen_true['tt'][:lmax_ivf +
1] + datnoise_cls['tt']
if qe_key in ['p_p', 'p']:
fal['ee'] = cls_filt['ee'][:lmax_ivf + 1] + (
nlev_p * np.pi / 180. / 60.)**2 * transfi2
fal['bb'] = cls_filt['bb'][:lmax_ivf + 1] + (
nlev_p * np.pi / 180. / 60.)**2 * transfi2
dat_delcls['ee'] = cls_plen_true['ee'][:lmax_ivf +
1] + datnoise_cls['ee']
dat_delcls['bb'] = cls_plen_true['bb'][:lmax_ivf +
1] + datnoise_cls['bb']
if qe_key in ['p']:
fal['te'] = np.copy(cls_filt['te'][:lmax_ivf + 1])
dat_delcls['te'] = np.copy(cls_plen_true['te'][:lmax_ivf + 1])
fal = utils.cl_inverse(fal)
for cl in fal.values():
cl[:lmin_ivf] *= 0.
for cl in dat_delcls.values():
cl[:lmin_ivf] *= 0.
cls_ivfs_arr = utils.cls_dot([fal, dat_delcls, fal])
cls_ivfs = dict()
for i, a in enumerate(['t', 'e', 'b']):
for j, b in enumerate(['t', 'e', 'b'][i:]):
if np.any(cls_ivfs_arr[i, j + i]):
cls_ivfs[a + b] = cls_ivfs_arr[i, j + i]
n_gg = get_nhl(qe_key,
qe_key,
cls_w,
cls_ivfs,
lmax_ivf,
lmax_ivf,
lmax_out=lmax_qlm)[0]
r_gg_true = qresp.get_response(qe_key,
lmax_ivf,
'p',
cls_w,
cls_f,
fal,
lmax_qlm=lmax_qlm)[0]
r_gg_fid = qresp.get_response(
qe_key, lmax_ivf, 'p', cls_w, cls_w, fal,
lmax_qlm=lmax_qlm)[0] if cls_f is not cls_w else r_gg_true
N0_biased = n_gg * utils.cli(
r_gg_fid**
2) # N0 of possibly biased (by Rtrue / Rfid) QE estimator
N0_unbiased = n_gg * utils.cli(
r_gg_true**2
) # N0 of QE estimator after rescaling by Rfid / Rtrue to make it unbiased
N0s_biased.append(N0_biased)
N0s_unbiased.append(N0_unbiased)
cls_plen_true['pp'] = cldd_true * utils.cli(
np.arange(len(cldd_true))**2 *
np.arange(1, len(cldd_true) + 1, dtype=float)**2 / (2. * np.pi))
cls_plen_fid['pp'] = cldd_fid * utils.cli(
np.arange(len(cldd_fid))**2 *
np.arange(1, len(cldd_fid) + 1, dtype=float)**2 / (2. * np.pi))
if 'wN1' in version:
if it == 0: print('Adding n1 in iterations')
from lensitbiases import n1_fft
from scipy.interpolate import UnivariateSpline as spl
lib = n1_fft.n1_fft(fal,
cls_w,
cls_f,
np.copy(cls_plen_true['pp']),
lminbox=50,
lmaxbox=5000,
k2l=None)
n1_Ls = np.arange(50, lmax_qlm + 1, 50)
if lmax_qlm not in n1_Ls: n1_Ls = np.append(n1_Ls, lmax_qlm)
n1 = np.array(
[lib.get_n1(qe_key, L, do_n1mat=False) for L in n1_Ls])
N1_biased = spl(n1_Ls,
n1_Ls**2 * (n1_Ls * 1. + 1)**2 * n1 /
r_gg_fid[n1_Ls]**2,
k=2,
s=0,
ext='zeros')(np.arange(len(N0_unbiased)))
N1_biased *= utils.cli(
np.arange(lmax_qlm + 1)**2 *
np.arange(1, lmax_qlm + 2, dtype=float)**2)
N1_unbiased = N1_biased * (r_gg_fid * utils.cli(r_gg_true))**2
else:
N1_biased = np.zeros(lmax_qlm + 1, dtype=float)
N1_unbiased = np.zeros(lmax_qlm + 1, dtype=float)
delcls_fid.append(cls_plen_fid)
delcls_true.append(cls_plen_true)
N1s_biased.append(N1_biased)
N1s_unbiased.append(N1_unbiased)
return (np.array(N0s_biased),
np.array(N0s_unbiased)) if not ret_delcls else (
(np.array(N0s_biased), np.array(N0s_unbiased), delcls_fid,
delcls_true))
def main(model_name, output_dir=OUTPATH):
# Load SNEMO model
snemo = sncosmo.get_source(model_name)
WAVES = snemo._wave
PHASES = snemo._phase
fluxes = np.array([flux(PHASES, WAVES) for flux in snemo._model_fluxes]).T
# Load Hsiao template
hsiao = sncosmo.Model('hsiao')
# Define extended wavelengths
EXTWAVE = np.concatenate([
np.arange(1000, WAVES[0], 2), WAVES,
np.arange(np.around(WAVES[-1], -1), 20010, 2)
])
# Extend mean spectral timeseries (SNEMO C0)
uv_waves = EXTWAVE[EXTWAVE < WAVES[0]]
ir_waves = EXTWAVE[EXTWAVE > WAVES[-1]]
opt_waves = EXTWAVE[(EXTWAVE >= WAVES[0]) & (EXTWAVE <= WAVES[-1])]
uv_join_wave = WAVES[0]
ir_join_wave = WAVES[-1]
uv_scale = fluxes[0, :, 0] / (hsiao.flux(time=PHASES, wave=uv_join_wave).T)
ir_scale = fluxes[-1, :, 0] / (hsiao.flux(time=PHASES,
wave=ir_join_wave).T)
ext_flux0 = np.concatenate([
uv_scale * hsiao.flux(time=PHASES, wave=uv_waves).T, fluxes[:, :, 0],
ir_scale * hsiao.flux(time=PHASES, wave=ir_waves).T
])
# Apodize additional components
uv_join_end = 3500 # location (in Angstroms) to stop smoothing in the UV
ir_join_start = 8400 # location (in Angstroms) to start smoothing in the IR
def apodize(x):
if x < uv_join_end:
return 1 / (uv_join_end - uv_join_wave) * (x - uv_join_wave)
elif x > ir_join_start:
return 1 - 1 / (ir_join_wave - ir_join_start) * (x - ir_join_start)
else:
return 1
window = np.array(list(map(apodize, WAVES)))
apodized_comps = (window * fluxes[:, :, 1:].T).T
ext_flux1 = np.concatenate([
np.zeros((len(uv_waves), *apodized_comps.shape[1:])), apodized_comps,
np.zeros((len(ir_waves), *apodized_comps.shape[1:]))
])
# Reshape
ext_flux0 = ext_flux0.reshape((*ext_flux0.shape, 1))
ext_flux = np.concatenate([ext_flux0, ext_flux1], axis=-1)
# Rebin
minwave = 1000
maxwave = 20000
velocity = 1000
nbins = int(
np.log10(maxwave / minwave) / np.log10(1 + velocity / 3.e5) + 1)
velspace_EXTWAVE = np.geomspace(1000, 20000, nbins)
flux_spls = [spl(PHASES, EXTWAVE, f) for f in ext_flux.T]
velspace_flux = np.array([s(PHASES, velspace_EXTWAVE)
for s in flux_spls]).T
# Write to file
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
out = []
for i, phase in enumerate(PHASES):
for j, wave in enumerate(velspace_EXTWAVE):
out.append([phase, wave, *velspace_flux[j, i]])
np.savetxt(os.path.join(output_dir, 'ext_{}.dat'.format(snemo.name)), out)
dims = 2048 # dimensions of density array
verbose = False
DEBUG = 2
# Calculate z for range of comoving distances
z = np.zeros(3000)
d_c = np.zeros(3000)
H = np.zeros(3000)
for d in range(1, 3000):
d_c[d] = d
z[d] = z_at_value(cosmo.comoving_distance, d * uni.Mpc)
H[d] = cosmo.H(z[d]).value
# create spine for quick lookup
d2z = spl(d_c, z)
z2H = spl(z, H, s=0)
def read_header(snap):
snapshot_fname = '/cosma6/data/dp004/dc-smit4/Daemmerung/Planck2013-Npart_2048_Box_3000-Fiducial/run1/snapdir_{0:03d}/Planck2013-L3000-N2048-Fiducial_{0:03d}.0'.format(
snap)
return readgadget.header(snapshot_fname)
# read headers for all snapshots to get redshifts
snaps = 24
zz = np.zeros(snaps)
for snap in range(snaps):
head = read_header(63 - snap)
zz[snap] = head.redshift
# Empty arrays to be filled with the splines (y1, y2), and the interpolated y-values (yy1, yy2)
y1 = [] # master instrument splines
y2 = []
yy1 = [] # master instrument interpolated y-values
yy2 = []
# for broader peaks a range(40, 105) covers 400 nm to 700 nm
# for narrower peaks a range of (50, 90) covers 500 nm to 640 nm
x = [first_x[i] for i in range(40, 105)] # Here, the range of y-values are chosen
for i in rcal_i:
first_row.append(fill_list(sheet1, i, 7))
first_row2.append(fill_list(sheet2, i, 7))
for i in range(len(rcal_i)):
y1.append(spl(first_x, first_row[i]))
y2.append(spl(first_x2, first_row2[i]))
yy1.append(y1[i](x))
yy2.append(y2[i](x))
# -------------------- FUNCTIONS -----------------------------------
### Mean subtraction
mean_yy1 = [np.mean(yy1[i]) for i in range(len(yy1))]
mean_yy2 = [np.mean(yy2[i]) for i in range(len(yy2))]
sm_yy1 = []
sm_yy2 = []
for i in range(len(yy1)):
sm_yy1.append([yy1[i][j] - mean_yy1[i] for j in range(len(yy1[i]))])
sm_yy2.append([yy2[i][j] - mean_yy2[i] for j in range(len(yy1[i]))])
pcount = headdata['npartTotal'][0][1]
print (pcount)
dummy = np.fromfile(f, dtype=np.int32,count=1)
parts = np.fromfile(f,dtype=part2, count = pcount)
f.close()
return parts
# Calculate z for range of comoving distances
z = np.zeros(3000)
d_c = np.zeros(3000)
for d in range(1,3000):
d_c[d] = d
z[d] = z_at_value(cosmo.comoving_distance, d * uni.Mpc)
# create spine for quick lookup
d2z = spl(d_c,z)
# define Schechter funciton
def schechter(x, phi_star=1.0, a=-1.24): # the luminosity function n(x) with x = L/Lstar
return phi_star * x**a * np.exp(-x)
# calculate cumulative probability distribution function for Schechter distribution
x = np.linspace(0.1,10, 1000)
r = np.zeros(1000)
for i in range(1000):
r[i] = integrate.quad(schechter, x[i], np.inf)[0]
# create spline for looking up luminosity value correspondiong to a given p value
P2L = spl(r[::-1], x[::-1], s=0)
# calculate average particle density and L_min for normalising probability distribution
# Calculate real-time sensitivity + precision
pred_rt = dat_sk_phat.groupby('date')[['y','tp','fp']].sum().reset_index()
cn_cumsum = ['y','tp','fp']
pred_rt[cn_cumsum] = pred_rt[cn_cumsum].apply(np.cumsum)
pred_rt = pred_rt.loc[pred_rt.query('y>0').index.min():].reset_index(None, True)
pred_rt = pred_rt.assign(sens=lambda x: x.tp/x.y, prec=lambda x: x.tp/(x.tp+x.fp))
pred_rt = pred_rt.assign(se_sens=lambda x: np.sqrt(x.sens*(1-x.sens)/x.y),se_prec=lambda x: np.sqrt(x.prec*(1-x.prec)/(x.tp+x.fp)))
pred_rt = pred_rt.assign(zscore=lambda x: (x.sens-sens_trial)/x.se_sens).query('zscore<inf').reset_index(None,True)
pred_rt = pred_rt.assign(pval=lambda x: 1-stats.norm.cdf(x.zscore))
# Calculat the full "n"
holder_n = np.zeros(len(pred_rt))
for i, date in enumerate(pred_rt.date):
holder_n[i] = df_sk.query('date <= @date').shape[0]
pred_rt.insert(0,'n_rt',holder_n.astype(int))
mdl_nsq = spl(x=df_n_power.n_nsq, y=df_n_power.power)
mdl_sk = spl(x=df_n_power.n_sk, y=df_n_power.power)
# Add on power
pred_rt = pred_rt.assign(power_nsq=mdl_nsq(pred_rt.n_rt),power_sk=mdl_sk(pred_rt.n_rt))
# Repeat calculations for model types
tmp_pred = dat_sk_phat.groupby(['mdl','date'])[['y','tp','fp']].sum().reset_index()
tmp_pred[cn_cumsum] = tmp_pred.groupby('mdl')[cn_cumsum].cumsum()
tmp_pred = tmp_pred.assign(sens=lambda x: x.tp/x.y, prec=lambda x: x.tp/(x.tp+x.fp)).dropna()[['date','mdl','sens','prec']]
pred_PR = pd.concat([pred_rt.assign(mdl='all')[tmp_pred.columns],tmp_pred],0)
pred_PR = pred_PR.melt(['date','mdl'],None,'msr','val')
pred_PR = pred_PR.assign(mdl=lambda x: x.mdl.map(di_mdl))
# Print how long trial would take
idx = pred_rt.query('pval < @alpha').index
date_reject = pred_rt.loc[idx[np.where(np.diff(idx.values) > 1)[0].max()+1]].date
days2reject = (date_reject - df_sk.date.min()).days
curvature_of_func = 1.0 / R
return curvature_of_func
if __name__ == "__main__":
#r, cur = calc_curvature(testf, )
xdata = np.linspace(0., 1.5, 150)
curvatures = [ calc_curvature(testf,
x) for x in xdata]
ydata = testf(xdata)
x_somepoint = np.linspace(0., 3, 100)
y_somepoit = [testf(x) for x in x_somepoint]
spfunc = spl(x_somepoint, y_somepoit, s=1)
curvatures_spl = [calc_curvature(spfunc, x) for x in x_somepoint]
fig, ax1 = plt.subplots()
ax1.set_xlabel("X value")
ax1.set_ylabel("Y value")
for tl in ax1.get_yticklabels():
tl.set_color("black")
#ax2 = ax1.twinx()
#ax2.set_ylabel("Curvature", color="r")
line1 = ax1.plot(xdata, ydata, "b-", label="Function Values")
#line2 = ax1.plot(x_somepoint, spfunc(x_somepoint), "g-", label="Spline Value")
# 3 state 2D adt solve
import numpy as np
from scipy.interpolate import InterpolatedUnivariateSpline as spl
from scipy.integrate import solve_ivp
from numpy import sin, cos, tan
dat = np.loadtxt('./tauth_1d.dat')
grid = dat[:, 0]
getTau1, getTau2, getTau3 = spl(grid,
dat[:, 1]), spl(grid,
dat[:,
2]), spl(grid, dat[:, 3])
# for general
def res(t, y): # returns value of the differential function at the point
t12, t13, t23 = getTau1(t), getTau2(t), getTau3(t)
y12 = -t12 - tan(y[1]) * (t13 * sin(y[0]) + t23 * cos(y[0]))
y13 = -t13 * cos(y[0]) + t23 * sin(y[0])
y23 = -(1.0 / cos(y[1])) * (t13 * sin(y[0]) + t23 * cos(y[0]))
return y12, y13, y23
sol = solve_ivp(res, [grid[0], grid[-1]], [0, 0, 0],
method='RK45',
t_eval=grid,
def _addCCSpec(snType, sedFile, oldWave, specList=None, specName=None):
phase, newWave, flux = sncosmo.read_griddata_ascii(sedFile)
wave = np.append(newWave[newWave < oldWave[0]],
newWave[newWave > oldWave[-1]])
if len(wave) == 0:
return (flux)
#flux=flux[:][newWave==wave]
if specList:
spec = createAveSpec(specList)
elif specName:
spec = os.path.join(__dir__, 'data', 'spectra', specName)
allSpec = _readSpec(spec)
else:
tempInd, tempVal = _find_nearest(phase, 0)
if snType == 'II':
snType = 'IIP'
allSpec, bestConst = _findBest(snType, newWave, flux[tempInd])
#spec=os.path.join(__dir__,'data','spectra',_defaultSpec[snType])
#allSpec=_readSpec(spec)
x = [float(x[2:]) for x in allSpec.colnames if x != 'wave']
y = allSpec['wave']
allSpec.remove_column('wave')
func = interp2d(x, y, np.array([list(r) for r in np.array(allSpec)]))
tempFlux = np.transpose(
func(phase[phase >= x[0]][phase[phase >= x[0]] <= x[-1]],
newWave[newWave > 4000][newWave[newWave > 4000] < 7500]))
finalFlux = np.transpose(
func(phase[phase >= x[0]][phase[phase >= x[0]] <= x[-1]],
wave[wave >= y[0]][wave[wave >= y[0]] <= y[-1]]))
if np.max(np.max(finalFlux)) == 0:
return
#if os.path.basename(sedFile)=='SDSS-015339.SED':
# print(finalFlux)
# print(wave[wave>=y[0]][wave[wave>=y[0]]<=y[-1]])
# sys.exit()
splines = []
for p in phase[phase >= x[0]][phase[phase >= x[0]] <= x[-1]]:
ind = np.where(phase == p)[0][0]
mySpline = spl(
newWave[newWave > 4000][newWave[newWave > 4000] < 7500],
flux[ind][newWave < 7500][newWave[newWave < 7500] > 4000],
k=5,
ext=1)
tempSpline = mySpline(
newWave[newWave < 7500][newWave[newWave < 7500] > 4000])
splines.append(
np.log10(
flux[ind][newWave < 7500][newWave[newWave < 7500] > 4000] /
tempSpline))
splines = np.array(splines)
waves = wave[wave >= y[0]][wave[wave >= y[0]] <= y[-1]]
for i in range(len(phase[phase >= x[0]][phase[phase >= x[0]] <= x[-1]])):
#const1=math.fabs(np.max(splines[i])/np.max(tempFlux[i]))
#const2=math.fabs(np.min(splines[i])/np.max(tempFlux[i]))
const = minimize(_specChi,
np.array([bestConst]),
args=(splines[i], tempFlux[i])).x
#const=np.nanmedian([math.fabs(k) for k in splines[i]/tempFlux[i]])
finalFlux[i] *= const
if i == 4:
final = const
#if tempFlux[i][0]>splines[i][0]:
# constMinus=tempFlux[i][0]-splines[i][0]
# finalFlux[i]-=constMinus
#else:
# constMinus=splines[i][0]-tempFlux[i][0]
# tempFlux[i]+=constMinus
#splines[i]+=finalFlux[i]
ind = np.where(phase == phase[phase >= x[0]][
phase[phase >= x[0]] <= x[-1]][i])[0][0]
for j in range(len(waves)):
ind2 = np.where(newWave == waves[j])[0][0]
flux[ind][ind2] = max(0, flux[ind][ind2] + finalFlux[i][j])
#fig=plt.figure()
#ax=fig.gca()
#ax.plot(newWave[newWave<7500][newWave[newWave<7500]>4000],tempFlux[4]*const)
#ax.plot(newWave[newWave<7500][newWave[newWave<7500]>4000],splines[4])
#plt.savefig('test_'+os.path.basename(sedFile[:-3])+'pdf',format='pdf',overwrite=True)
#plt.close()
sncosmo.write_griddata_ascii(phase, newWave, flux, sedFile)
return
