def test_likelihoods():
""" The output should be samples of a chi squared distribution with lik_FT.nz_modes d.o.f. """
cls = [cl_pp, cl_noise, cl_unl]
labels = ["pp", "noise", "unl"]
sim_libs = [lib_pp, lib_noise, lib_cmb_unl]
ress = [HD_res, LD_res, HD_res]
histos = []
def is_ok(sig):
if abs(sig) < 0.01:
return "** deviation suspiciously small **"
if abs(sig) >= 0.01 and abs(sig) < 3.0:
return " Test OK"
if abs(sig) >= 3.0:
return "** deviation suspiciously large **"
for cl, label, sims, res in zip(cls, labels, sim_libs, ress):
histo = []
lik_FT = likelihoods.circulant_Gauss_pdf_fourierspace(cl, (res, res), lside)
for i, idx in enumerate_progress(np.arange(nsims), "+ test_likelihoods::collecting chi2 " + label + " sims"):
sim = sims.get_sim(idx)
Npix = lik_FT.dof
histo.append((lik_FT.eval_log(sim) + lik_FT.lnZ) / Npix)
histo = np.array(histo)
histos.append(histo)
# A chi sqd distribution with k d.o.f. has mean k and variance 2k
# The loglikelihood is - 1/2 chi sqd where d.o.f. is the number of modes with nonzero spectrum.
dev = (1.0 + 2 * np.mean(histo)) / np.sqrt(2.0 / lik_FT.dof / nsims)
print "+++++++++++++++++++++++++++++++++++++++++++++++++++"
print "+ Number of sims :", str(nsims)
print "+ d.o.f. / Npix :", np.round(float(lik_FT.dof) / (2 ** res) ** 2, 2)
print "+ Mean chi2, " + label + " sims, dev. from theory", np.round(dev, 3), " sig. "
print "+ ---->", is_ok((1.0 + 2 * np.mean(histo)) / np.sqrt(2.0 / lik_FT.dof / nsims))
print "+++++++++++++++++++++++++++++++++++++++++++++++++++"
return histos
def get_inverse(self,NR_iter= None,use_Pool = 0):
"""
Returns inverse displacement of the full map.
"""
if use_Pool > 0 :
if not self.is_dxdy_ondisk() :
assert self.has_lib_dir(),"Specify lib. dir. if you want to use Pool."
print "lens_map::writing displacements on disk :"
self.write_npy( self.lib_dir + '/temp_displ' + str(pbs.rank)) # this turns dx and dy to the paths
dx_inv,dy_inv = lens_Pool.get_inverse_Pooled(self.mk_args('',self.dx,self.dy),root_Nthreads = use_Pool)
return ffs_displacement(dx_inv,dy_inv,lib_dir=self.lib_dir,
LD_res=self.LD_res,verbose = self.verbose,spline_order=self.k,NR_iter= self.NR_iter)
if NR_iter is None : NR_iter = self.NR_iter
spliter_lib = map_spliter.periodicmap_spliter() # library to split periodic maps.
dx_inv,dy_inv = np.empty(self.shape),np.empty(self.shape)
label = 'ffs_displacement::calculating inverse displ. field'
for i,N in utils.enumerate_progress(xrange(self.N_chks),label = label):
# Doing chunk N
dx_inv_N,dy_inv_N = self.get_inverse_chk_N(N,NR_iter=NR_iter)
sLDs,sHDs = spliter_lib.get_slices_chk_N(N,self.LD_res,self.HD_res,self.buffers,inverse = True)
# Pasting it onto the full map
dx_inv[sHDs[0]] = dx_inv_N[sLDs[0]]
dy_inv[sHDs[0]] = dy_inv_N[sLDs[0]]
return ffs_displacement(dx_inv,dy_inv,lib_dir=self.lib_dir,
LD_res=self.LD_res,verbose = self.verbose,spline_order=self.k,NR_iter= self.NR_iter)
def test_cl():
bins_l = np.int64(np.linspace(10.,3000,100))
bins_u = bins_l[1:] -1
bins_l = bins_l[0:len(bins_l)-1]
binner = jc_utils.binner(bins_l,bins_u)
del bins_l,bins_u
import pylab as pl
pl.ioff()
from matplotlib.backends.backend_pdf import PdfPages
stats_len = jc_utils.stats(binner.Nbins())
for i,idx in enumerate_progress(xrange(nsims),label = 'test_cl::collecting cls'):
sim_cl_len = lib_cmb_unl.map2cl(lib_cmb_len.get_sim(idx))
stats_len.add(binner.bin_that(np.arange(len(sim_cl_len)),sim_cl_len))
camb_binned = binner.bin_that(np.arange(len(cl_len)),cl_len)
camb_unl_binned = binner.bin_that(np.arange(len(cl_unl)),cl_unl)
pp = PdfPages(path_to_figs+'/lenclvscamb.pdf')
pl.figure()
pl.title('len Cl vs CAMB, ' +str(nsims) + ' sims.')
pl.plot(binner.bin_centers(),stats_len.mean()/camb_binned -1.,label = 'sim/camb -1.,100 bins, res ' + str(HD_res))
pl.xlabel('$\ell$')
pl.ylim(-0.05,0.05)
pl.hlines([-0.001,0.001],np.min(binner.bins_l),np.max(binner.bins_r),linestyles='--',color = 'grey')
pl.legend(frameon = False)
pp.savefig()
pl.figure()
pl.title('cl_len / cl_unlCAMB, ' +str(nsims) + ' sims.')
pl.plot(binner.bin_centers(),stats_len.mean()/camb_unl_binned -1.,label = 'sim/camb_unl -1.,binned, res ' + str(HD_res))
pl.plot(binner.bin_centers(),camb_binned/camb_unl_binned -1.,label = 'camb_len/camb_unl -1.,binned.')
pl.xlabel('$\ell$')
pl.legend(frameon = False)
pp.savefig()
pp.close()
pl.close()
def generate_sims(sim_lib, label=''):
if pbs.rank == 0 :
for i, idx in enumerate_progress(np.arange(par.nsims)[::pbs.size], 'building sims:' + label):
sim = sim_lib.get_sim(idx)
else :
for idx in np.arange(par.nsims)[pbs.rank::pbs.size] :
sim = sim_lib.get_sim(idx)
def plot_gradients(number_of_sims=1):
"""
Plot the total curvature (F. info) for phi and Omega for different volumes.
"""
import pylab as pl
pl.ioff()
FT_lik = likelihoods.circulant_Gauss_pdf_fourierspace(cl_pp, (HD_res, HD_res), lside)
curvp, curvO, curvpO = Fisher_lib.get_curvature_matrix_PhiOmega_ell()
curvp = (1j + 1.0) * curvp
# curvp += FT_lik.eval_curv()
grad = curvp * 0
print "+++++++++++++++++++++++++++++++++++++++++++++++++++"
for i, idx in enumerate_progress(np.arange(number_of_sims), label="plot_gradients::collecting sims"):
dat = sim_lib_fixedphi.get_sim(idx)
umap = len_cov.apply_cg_inverse(dat)[0].reshape(LD_shape)
grad += len_cov.likelihood_gradient_phiOmega_ell(umap, nodisplacement=True)[0]
# Estimate of phi
nz = np.where(curvp != 0.0)
phik_est = grad * 0
phik_est[nz] = -grad[nz] / curvp[nz]
phi_est = np.fft.irfft2(phik_est, HD_shape) * np.prod(HD_shape) / np.sqrt(len_cov.HD_cub.vol())
counts, Cl_raw = rfft2_utils.rfft2cl(np.fft.rfft2(phi_est), lside)
ell = np.arange(len(counts))
nz = np.where(counts > 0.0)
N0 = np.zeros(len(counts))
ell_N0 = rfft2_utils.k2ell(Fisher_lib.kx_from_index(np.arange(1, Fisher_lib.N_2p1 - 1)))
N0[ell_N0] = 1.0 / (0.5 * curvp.real[0, 1 : Fisher_lib.N_2p1 - 1])
N0[10:] = np.exp(np.interp(np.log(ell[10:]), np.log(ell[ell_N0]), np.log(N0[ell_N0])))
pl.figure()
pl.title("Reconstruction")
pl.loglog(binner.bin_centers(), binner.bin_that(ell[nz], (Cl_raw * ell ** 4 / np.pi / 2.0)[nz]), label="Clraw")
pl.plot(
binner.bin_centers(),
binner.bin_that(ell[nz], ((Cl_raw - number_of_sims * N0) * ell ** 4 / np.pi / 2.0)[nz]),
label="Clraw - N0",
)
pl.plot(np.arange(len(cl_pp)), cl_pp * np.arange(len(cl_pp)) ** 4 / np.pi / 2.0)
nz0 = np.where(N0 > 0.0)
pl.plot(ell[nz0], number_of_sims * (N0 * ell ** 4 / 2.0 / np.pi)[nz0], label="N0")
pl.legend(frameon=False)
pl.xlabel("L")
pl.ylabel("$C_L L^2 (L+1)^2 /2 \pi$")
pl.ylim(1e-10, 2e-7)
pl.savefig(path_to_figs + "/reconstruction.pdf")
pl.close()
print "+++++++++++++++++++++++++++++++++++++++++++++++++++"
def lens_map(self,map,use_Pool = 0):
"""
Lens the input map according to the displacement fields dx dy. 'map' typically would be (8192 * 8192) np array,
or the path to the array on disk.
Does this by splitting the job in chunks (of typically (256 * 256), as specified by the LD_res parameters)
allowing a buffer size to ensure the junctions are properly performed.
Set use_Pool to a power of two to use explicit threading via the multiprocessing module.
'use_Pool' ** 2 is the number of threads. On laptop and Darwin use_Pool = 16 has the best performances.
It use_Pool is set, then 'map' must be the path to the map to lens.
"""
if use_Pool > 0 :
if not isinstance(map,str) :
assert self.has_lib_dir(),"Specify lib. dir. if you want to use Pool."
np.save(self.lib_dir + '/temp_maptolens.npy',map)
if not self.is_dxdy_ondisk() :
assert self.has_lib_dir(),"Specify lib. dir. if you want to use Pool."
print "lens_map::writing displacements on disk :"
self.write_npy( self.lib_dir + '/temp_displ' + str(pbs.rank)) # this turns dx and dy to the paths
path_to_map = map if isinstance(map,str) else self.lib_dir + '/temp_maptolens.npy'
return lens_Pool.get_lens_Pooled(self.mk_args(path_to_map,self.dx,self.dy),root_Nthreads = use_Pool)
assert map.shape == self.shape,map.shape
s = self.chk_shape
idc0,idc1 = np.indices(s) # Two (256 * 256) maps
dx_gu = np.empty(s) # will dx displ. in grid units of each chunk (typ. (256 * 256) )
dy_gu = np.empty(s) # will dy displ. in grid units of each chunk (typ. (256 * 256) )
map_chk = np.empty(s) # Will be map chunk
# (typ. (256 * 256) )
lensed_map = np.empty(self.shape)
spliter_lib = map_spliter.periodicmap_spliter() # library to split periodic maps.
for i,N in utils.enumerate_progress(xrange(self.N_chks),label = 'ffs_displacement::lensing map'):
# doing chunk N
sLDs,sHDs = spliter_lib.get_slices_chk_N(N,self.LD_res,self.HD_res,self.buffers)
for sLD,sHD in zip(sLDs,sHDs):
# Displacements chunk in grid units, and map chunk to displace.
dx_gu[sLD] = self.get_dx()[sHD] / self.rmin[1]
dy_gu[sLD] = self.get_dy()[sHD] / self.rmin[0]
map_chk[sLD] = map[sHD]
lx = (idc1 + dx_gu).flatten() # No need to enforce periodicity here.
ly = (idc0 + dy_gu).flatten() # No need to enforce periodicity here.
sLDs,sHDs = spliter_lib.get_slices_chk_N(N,self.LD_res,self.HD_res,self.buffers,inverse = True)
lensed_map[sHDs[0]] = interpolate.RectBivariateSpline(np.arange(s[0]), np.arange(s[1]),
map_chk, kx=self.k, ky=self.k).ev(ly, lx).reshape(self.chk_shape)[sLDs[0]]
return lensed_map
def Pool_generic(func,arg,root_Nthreads):
assert len(arg) == 9,arg
path_to_map,path_to_dx,path_to_dy,buff0,buff1,HD_res0,HD_res1,NR_iter,kspl = arg
assert os.path.exists(path_to_dx) and os.path.exists(path_to_dy)
assert IsPowerOfTwo(root_Nthreads)
diff0,diff1 = Log2ofPowerof2( (root_Nthreads,root_Nthreads) )
HD_shape = (2 ** HD_res0, 2** HD_res1)
LD = (HD_res0 - diff0,HD_res1 - diff1)
pool = setup_Pool()
ret_list = pool.map(func,[[i,path_to_map,path_to_dx,path_to_dy,buff0,buff1,HD_res0,HD_res1,NR_iter,kspl,LD[0],LD[1]]
for i in range(root_Nthreads ** 2)])
pool.close()
pool.join()
# Recombines from the lensed_chks :
spliter_lib = periodicmap_spliter() # library to split periodic maps.
ret = [] # one map for lens, two for inverse
for i in range(len(ret_list[0])) :
map = np.empty(HD_shape)
for j,N in enumerate_progress(xrange(root_Nthreads ** 2),label = 'Pool_generic:patching chks together'):
sLDs,sHDs = spliter_lib.get_slices_chk_N(N,LD,(HD_res0,HD_res1),(buff0,buff1),inverse=True)
map[sHDs[0]] = ret_list[N][i][sLDs[0]]
ret.append(map)
return ret
def generate_sims(sim_lib, label=""):
for i, idx in enumerate_progress(xrange(nsims), "test_suite::regenerating sims: " + label):
sim = sim_lib.get_sim(idx)
def plot_cls(parfile,type = 'len',mc_sims = None,Nbins = 100):
"""
Plots the sims cls in comparison to input cl_len.
"""
assert type in ['len','unl','pp','OO']
par = imp.load_source('par',parfile)
lib = {'len':par.lib_cmb_len,'unl':par.lib_cmb_unl,'pp':par.lib_pp,'OO':par.lib_OO}[type]
if lib is None :
print 'no such lib, returning'
return
input_cl = {'len':par.cl_len,'unl':par.cl_unl,'pp':par.cl_pp,'OO':par.lib_OO}[type]
if not os.path.exists(par.lib_dir + '/figs') : os.mkdir(par.lib_dir + '/figs')
bins_l = np.int64(np.linspace(10.,3000,Nbins))
bins_u = bins_l[1:] -1
bins_l = bins_l[0:len(bins_l)-1]
binner = utils.binner(bins_l,bins_u)
stats_len = utils.stats(binner.Nbins())
if mc_sims is None : mc_sims = np.arange(par.nsims)
for i,idx in utils.enumerate_progress(mc_sims,label = 'test_cl::collecting cls'):
sim_cl = par.lib_cmb_unl.map2cl(lib.get_sim(idx))
ell = np.arange(len(sim_cl))
stats_len.add(binner.bin_that(ell,sim_cl))
if i == 0 :
cl_unb = sim_cl.copy()
sig_unb = sim_cl.copy() ** 2
else :
cl_unb += sim_cl
sig_unb += sim_cl ** 2
cl_unb /= (i + 1.)
sig_unb /= (i + 1.)
sig_unb = np.sqrt(sig_unb - cl_unb ** 2) # Unbinned sigmas.
cl_input_binned = binner.bin_that(np.arange(len(par.cl_len)),input_cl)
prefix = '' if not hasattr(par,'prefix') else par.prefix
if not os.path.exists('./figs/' + prefix) :
os.mkdir('./figs/' + prefix)
pp = PdfPages('./figs/' + prefix+'/cl'+type +'_vs_input. Nb'+str(Nbins)+'.pdf')
pl.ioff()
pl.figure('Cl ratios')
pl.title('Binned cl vs input, ' +str(len(mc_sims)) + ' sims.')
pl.plot(binner.bin_centers(),stats_len.mean()/cl_input_binned -1.,label = 'sim/input -1.,binned, res ' + str(par.HD_res))
pl.xlabel('$\ell$')
pl.ylim(-0.05,0.05)
pl.hlines([-0.001,0.001],np.min(binner.bins_l),np.max(binner.bins_r),linestyles='--',color = 'grey')
pl.legend(frameon = False)
pp.savefig()
pl.close()
pl.figure('Unbinned Cl mean')
pl.title('Unbinned Cl mean')
counts = par.lib_cmb_unl.get_ellcounts()
ell = np.arange(len(counts))
nz = np.where(counts[0:len(input_cl)-1] > 0)
pl.plot(ell[nz],cl_unb[nz]/input_cl[nz] -1.,label = 'cl sim / input -1 ')
pl.plot(ell[nz], np.sqrt(1./counts[nz]/len(mc_sims)),linestyle = '--',color = 'black',label = '$\sigma $ Gauss.')
pl.plot(ell[nz], -np.sqrt(1./counts[nz]/len(mc_sims)),linestyle = '--',color = 'black')
pl.plot(ell[nz],- np.sqrt(2./(2*ell[nz] + 1)/len(mc_sims)),linestyle = '--',color = 'black' )
pl.plot(ell[nz], np.sqrt(2./(2*ell[nz] + 1)/len(mc_sims)),linestyle = '--',color = 'black' )
pl.hlines([0.],0.,max(ell[nz]),color = 'grey',linestyle = '--')
pl.legend(frameon= False)
pp.savefig()
pl.close()
pl.figure('Unbinned Cl variance')
pl.title('Unbinned Cl variance')
counts = par.lib_cmb_unl.get_ellcounts()
ell = np.arange(len(counts))
nz = np.where(counts[0:len(input_cl) - 1] > 0)
pl.plot(ell[nz],sig_unb[nz]/np.sqrt(2 * input_cl[nz] ** 2 / (2 * ell[nz] + 1.)) -1.,label = 'sim sig./ Gauss sig -1. ')
pl.plot(ell[nz],sig_unb[nz]/np.sqrt(input_cl[nz] ** 2 / counts[nz]) -1.,label = 'incl. discrtness')
pl.ylim(-1,1.)
pl.hlines([0.],0.,max(ell[nz]),color = 'grey',linestyle = '--')
pl.legend(frameon = False)
pp.savefig()
pl.close()
pp.close()
par = imp.load_source('par',parfile)
if pbs.rank == 0 :
print "++++ Setup ok."
print "I see ",pbs.size," MPI proc."
if os.getenv("OMP_NUM_THREADS") is not None :
print "I see ",int(os.getenv("OMP_NUM_THREADS"))," OMP NUM THREADS"
pbs.barrier()
def generate_sims(sim_lib, label=''):
if pbs.rank == 0 :
for i, idx in enumerate_progress(np.arange(par.nsims)[::pbs.size], 'building sims:' + label):
sim = sim_lib.get_sim(idx)
else :
for idx in np.arange(par.nsims)[pbs.rank::pbs.size] :
sim = sim_lib.get_sim(idx)
# Generating lowest levels sims (in fact only storing the rng states) :
np.random.set_state(mk_session_seed(verbose = False))
for lib in par.base_libs:
if lib is not None and not lib.is_full() : generate_sims(lib,label = lib.lib_dir)
# Performing lensing operations on sims :
if pbs.rank == 0 :
for i, idx in enumerate_progress(np.arange(par.nsims)[::pbs.size], 'lensing sims: '):
sim = par.lib_cmb_len.get_sim(idx)
else :
for idx in np.arange(par.nsims)[pbs.rank::pbs.size] :
sim = par.lib_cmb_len.get_sim(idx)
pbs.barrier()
pbs.finalize()
