def test_likelihoods(): """ The output should be samples of a chi squared distribution with lik_FT.nz_modes d.o.f. """ cls = [cl_pp, cl_noise, cl_unl] labels = ["pp", "noise", "unl"] sim_libs = [lib_pp, lib_noise, lib_cmb_unl] ress = [HD_res, LD_res, HD_res] histos = [] def is_ok(sig): if abs(sig) < 0.01: return "** deviation suspiciously small **" if abs(sig) >= 0.01 and abs(sig) < 3.0: return " Test OK" if abs(sig) >= 3.0: return "** deviation suspiciously large **" for cl, label, sims, res in zip(cls, labels, sim_libs, ress): histo = [] lik_FT = likelihoods.circulant_Gauss_pdf_fourierspace(cl, (res, res), lside) for i, idx in enumerate_progress(np.arange(nsims), "+ test_likelihoods::collecting chi2 " + label + " sims"): sim = sims.get_sim(idx) Npix = lik_FT.dof histo.append((lik_FT.eval_log(sim) + lik_FT.lnZ) / Npix) histo = np.array(histo) histos.append(histo) # A chi sqd distribution with k d.o.f. has mean k and variance 2k # The loglikelihood is - 1/2 chi sqd where d.o.f. is the number of modes with nonzero spectrum. dev = (1.0 + 2 * np.mean(histo)) / np.sqrt(2.0 / lik_FT.dof / nsims) print "+++++++++++++++++++++++++++++++++++++++++++++++++++" print "+ Number of sims :", str(nsims) print "+ d.o.f. / Npix :", np.round(float(lik_FT.dof) / (2 ** res) ** 2, 2) print "+ Mean chi2, " + label + " sims, dev. from theory", np.round(dev, 3), " sig. " print "+ ---->", is_ok((1.0 + 2 * np.mean(histo)) / np.sqrt(2.0 / lik_FT.dof / nsims)) print "+++++++++++++++++++++++++++++++++++++++++++++++++++" return histos
def get_inverse(self,NR_iter= None,use_Pool = 0): """ Returns inverse displacement of the full map. """ if use_Pool > 0 : if not self.is_dxdy_ondisk() : assert self.has_lib_dir(),"Specify lib. dir. if you want to use Pool." print "lens_map::writing displacements on disk :" self.write_npy( self.lib_dir + '/temp_displ' + str(pbs.rank)) # this turns dx and dy to the paths dx_inv,dy_inv = lens_Pool.get_inverse_Pooled(self.mk_args('',self.dx,self.dy),root_Nthreads = use_Pool) return ffs_displacement(dx_inv,dy_inv,lib_dir=self.lib_dir, LD_res=self.LD_res,verbose = self.verbose,spline_order=self.k,NR_iter= self.NR_iter) if NR_iter is None : NR_iter = self.NR_iter spliter_lib = map_spliter.periodicmap_spliter() # library to split periodic maps. dx_inv,dy_inv = np.empty(self.shape),np.empty(self.shape) label = 'ffs_displacement::calculating inverse displ. field' for i,N in utils.enumerate_progress(xrange(self.N_chks),label = label): # Doing chunk N dx_inv_N,dy_inv_N = self.get_inverse_chk_N(N,NR_iter=NR_iter) sLDs,sHDs = spliter_lib.get_slices_chk_N(N,self.LD_res,self.HD_res,self.buffers,inverse = True) # Pasting it onto the full map dx_inv[sHDs[0]] = dx_inv_N[sLDs[0]] dy_inv[sHDs[0]] = dy_inv_N[sLDs[0]] return ffs_displacement(dx_inv,dy_inv,lib_dir=self.lib_dir, LD_res=self.LD_res,verbose = self.verbose,spline_order=self.k,NR_iter= self.NR_iter)
def test_cl(): bins_l = np.int64(np.linspace(10.,3000,100)) bins_u = bins_l[1:] -1 bins_l = bins_l[0:len(bins_l)-1] binner = jc_utils.binner(bins_l,bins_u) del bins_l,bins_u import pylab as pl pl.ioff() from matplotlib.backends.backend_pdf import PdfPages stats_len = jc_utils.stats(binner.Nbins()) for i,idx in enumerate_progress(xrange(nsims),label = 'test_cl::collecting cls'): sim_cl_len = lib_cmb_unl.map2cl(lib_cmb_len.get_sim(idx)) stats_len.add(binner.bin_that(np.arange(len(sim_cl_len)),sim_cl_len)) camb_binned = binner.bin_that(np.arange(len(cl_len)),cl_len) camb_unl_binned = binner.bin_that(np.arange(len(cl_unl)),cl_unl) pp = PdfPages(path_to_figs+'/lenclvscamb.pdf') pl.figure() pl.title('len Cl vs CAMB, ' +str(nsims) + ' sims.') pl.plot(binner.bin_centers(),stats_len.mean()/camb_binned -1.,label = 'sim/camb -1.,100 bins, res ' + str(HD_res)) pl.xlabel('$\ell$') pl.ylim(-0.05,0.05) pl.hlines([-0.001,0.001],np.min(binner.bins_l),np.max(binner.bins_r),linestyles='--',color = 'grey') pl.legend(frameon = False) pp.savefig() pl.figure() pl.title('cl_len / cl_unlCAMB, ' +str(nsims) + ' sims.') pl.plot(binner.bin_centers(),stats_len.mean()/camb_unl_binned -1.,label = 'sim/camb_unl -1.,binned, res ' + str(HD_res)) pl.plot(binner.bin_centers(),camb_binned/camb_unl_binned -1.,label = 'camb_len/camb_unl -1.,binned.') pl.xlabel('$\ell$') pl.legend(frameon = False) pp.savefig() pp.close() pl.close()
def generate_sims(sim_lib, label=''): if pbs.rank == 0 : for i, idx in enumerate_progress(np.arange(par.nsims)[::pbs.size], 'building sims:' + label): sim = sim_lib.get_sim(idx) else : for idx in np.arange(par.nsims)[pbs.rank::pbs.size] : sim = sim_lib.get_sim(idx)
def plot_gradients(number_of_sims=1): """ Plot the total curvature (F. info) for phi and Omega for different volumes. """ import pylab as pl pl.ioff() FT_lik = likelihoods.circulant_Gauss_pdf_fourierspace(cl_pp, (HD_res, HD_res), lside) curvp, curvO, curvpO = Fisher_lib.get_curvature_matrix_PhiOmega_ell() curvp = (1j + 1.0) * curvp # curvp += FT_lik.eval_curv() grad = curvp * 0 print "+++++++++++++++++++++++++++++++++++++++++++++++++++" for i, idx in enumerate_progress(np.arange(number_of_sims), label="plot_gradients::collecting sims"): dat = sim_lib_fixedphi.get_sim(idx) umap = len_cov.apply_cg_inverse(dat)[0].reshape(LD_shape) grad += len_cov.likelihood_gradient_phiOmega_ell(umap, nodisplacement=True)[0] # Estimate of phi nz = np.where(curvp != 0.0) phik_est = grad * 0 phik_est[nz] = -grad[nz] / curvp[nz] phi_est = np.fft.irfft2(phik_est, HD_shape) * np.prod(HD_shape) / np.sqrt(len_cov.HD_cub.vol()) counts, Cl_raw = rfft2_utils.rfft2cl(np.fft.rfft2(phi_est), lside) ell = np.arange(len(counts)) nz = np.where(counts > 0.0) N0 = np.zeros(len(counts)) ell_N0 = rfft2_utils.k2ell(Fisher_lib.kx_from_index(np.arange(1, Fisher_lib.N_2p1 - 1))) N0[ell_N0] = 1.0 / (0.5 * curvp.real[0, 1 : Fisher_lib.N_2p1 - 1]) N0[10:] = np.exp(np.interp(np.log(ell[10:]), np.log(ell[ell_N0]), np.log(N0[ell_N0]))) pl.figure() pl.title("Reconstruction") pl.loglog(binner.bin_centers(), binner.bin_that(ell[nz], (Cl_raw * ell ** 4 / np.pi / 2.0)[nz]), label="Clraw") pl.plot( binner.bin_centers(), binner.bin_that(ell[nz], ((Cl_raw - number_of_sims * N0) * ell ** 4 / np.pi / 2.0)[nz]), label="Clraw - N0", ) pl.plot(np.arange(len(cl_pp)), cl_pp * np.arange(len(cl_pp)) ** 4 / np.pi / 2.0) nz0 = np.where(N0 > 0.0) pl.plot(ell[nz0], number_of_sims * (N0 * ell ** 4 / 2.0 / np.pi)[nz0], label="N0") pl.legend(frameon=False) pl.xlabel("L") pl.ylabel("$C_L L^2 (L+1)^2 /2 \pi$") pl.ylim(1e-10, 2e-7) pl.savefig(path_to_figs + "/reconstruction.pdf") pl.close() print "+++++++++++++++++++++++++++++++++++++++++++++++++++"
def lens_map(self,map,use_Pool = 0): """ Lens the input map according to the displacement fields dx dy. 'map' typically would be (8192 * 8192) np array, or the path to the array on disk. Does this by splitting the job in chunks (of typically (256 * 256), as specified by the LD_res parameters) allowing a buffer size to ensure the junctions are properly performed. Set use_Pool to a power of two to use explicit threading via the multiprocessing module. 'use_Pool' ** 2 is the number of threads. On laptop and Darwin use_Pool = 16 has the best performances. It use_Pool is set, then 'map' must be the path to the map to lens. """ if use_Pool > 0 : if not isinstance(map,str) : assert self.has_lib_dir(),"Specify lib. dir. if you want to use Pool." np.save(self.lib_dir + '/temp_maptolens.npy',map) if not self.is_dxdy_ondisk() : assert self.has_lib_dir(),"Specify lib. dir. if you want to use Pool." print "lens_map::writing displacements on disk :" self.write_npy( self.lib_dir + '/temp_displ' + str(pbs.rank)) # this turns dx and dy to the paths path_to_map = map if isinstance(map,str) else self.lib_dir + '/temp_maptolens.npy' return lens_Pool.get_lens_Pooled(self.mk_args(path_to_map,self.dx,self.dy),root_Nthreads = use_Pool) assert map.shape == self.shape,map.shape s = self.chk_shape idc0,idc1 = np.indices(s) # Two (256 * 256) maps dx_gu = np.empty(s) # will dx displ. in grid units of each chunk (typ. (256 * 256) ) dy_gu = np.empty(s) # will dy displ. in grid units of each chunk (typ. (256 * 256) ) map_chk = np.empty(s) # Will be map chunk # (typ. (256 * 256) ) lensed_map = np.empty(self.shape) spliter_lib = map_spliter.periodicmap_spliter() # library to split periodic maps. for i,N in utils.enumerate_progress(xrange(self.N_chks),label = 'ffs_displacement::lensing map'): # doing chunk N sLDs,sHDs = spliter_lib.get_slices_chk_N(N,self.LD_res,self.HD_res,self.buffers) for sLD,sHD in zip(sLDs,sHDs): # Displacements chunk in grid units, and map chunk to displace. dx_gu[sLD] = self.get_dx()[sHD] / self.rmin[1] dy_gu[sLD] = self.get_dy()[sHD] / self.rmin[0] map_chk[sLD] = map[sHD] lx = (idc1 + dx_gu).flatten() # No need to enforce periodicity here. ly = (idc0 + dy_gu).flatten() # No need to enforce periodicity here. sLDs,sHDs = spliter_lib.get_slices_chk_N(N,self.LD_res,self.HD_res,self.buffers,inverse = True) lensed_map[sHDs[0]] = interpolate.RectBivariateSpline(np.arange(s[0]), np.arange(s[1]), map_chk, kx=self.k, ky=self.k).ev(ly, lx).reshape(self.chk_shape)[sLDs[0]] return lensed_map
def Pool_generic(func,arg,root_Nthreads): assert len(arg) == 9,arg path_to_map,path_to_dx,path_to_dy,buff0,buff1,HD_res0,HD_res1,NR_iter,kspl = arg assert os.path.exists(path_to_dx) and os.path.exists(path_to_dy) assert IsPowerOfTwo(root_Nthreads) diff0,diff1 = Log2ofPowerof2( (root_Nthreads,root_Nthreads) ) HD_shape = (2 ** HD_res0, 2** HD_res1) LD = (HD_res0 - diff0,HD_res1 - diff1) pool = setup_Pool() ret_list = pool.map(func,[[i,path_to_map,path_to_dx,path_to_dy,buff0,buff1,HD_res0,HD_res1,NR_iter,kspl,LD[0],LD[1]] for i in range(root_Nthreads ** 2)]) pool.close() pool.join() # Recombines from the lensed_chks : spliter_lib = periodicmap_spliter() # library to split periodic maps. ret = [] # one map for lens, two for inverse for i in range(len(ret_list[0])) : map = np.empty(HD_shape) for j,N in enumerate_progress(xrange(root_Nthreads ** 2),label = 'Pool_generic:patching chks together'): sLDs,sHDs = spliter_lib.get_slices_chk_N(N,LD,(HD_res0,HD_res1),(buff0,buff1),inverse=True) map[sHDs[0]] = ret_list[N][i][sLDs[0]] ret.append(map) return ret
def generate_sims(sim_lib, label=""): for i, idx in enumerate_progress(xrange(nsims), "test_suite::regenerating sims: " + label): sim = sim_lib.get_sim(idx)
def plot_cls(parfile,type = 'len',mc_sims = None,Nbins = 100): """ Plots the sims cls in comparison to input cl_len. """ assert type in ['len','unl','pp','OO'] par = imp.load_source('par',parfile) lib = {'len':par.lib_cmb_len,'unl':par.lib_cmb_unl,'pp':par.lib_pp,'OO':par.lib_OO}[type] if lib is None : print 'no such lib, returning' return input_cl = {'len':par.cl_len,'unl':par.cl_unl,'pp':par.cl_pp,'OO':par.lib_OO}[type] if not os.path.exists(par.lib_dir + '/figs') : os.mkdir(par.lib_dir + '/figs') bins_l = np.int64(np.linspace(10.,3000,Nbins)) bins_u = bins_l[1:] -1 bins_l = bins_l[0:len(bins_l)-1] binner = utils.binner(bins_l,bins_u) stats_len = utils.stats(binner.Nbins()) if mc_sims is None : mc_sims = np.arange(par.nsims) for i,idx in utils.enumerate_progress(mc_sims,label = 'test_cl::collecting cls'): sim_cl = par.lib_cmb_unl.map2cl(lib.get_sim(idx)) ell = np.arange(len(sim_cl)) stats_len.add(binner.bin_that(ell,sim_cl)) if i == 0 : cl_unb = sim_cl.copy() sig_unb = sim_cl.copy() ** 2 else : cl_unb += sim_cl sig_unb += sim_cl ** 2 cl_unb /= (i + 1.) sig_unb /= (i + 1.) sig_unb = np.sqrt(sig_unb - cl_unb ** 2) # Unbinned sigmas. cl_input_binned = binner.bin_that(np.arange(len(par.cl_len)),input_cl) prefix = '' if not hasattr(par,'prefix') else par.prefix if not os.path.exists('./figs/' + prefix) : os.mkdir('./figs/' + prefix) pp = PdfPages('./figs/' + prefix+'/cl'+type +'_vs_input. Nb'+str(Nbins)+'.pdf') pl.ioff() pl.figure('Cl ratios') pl.title('Binned cl vs input, ' +str(len(mc_sims)) + ' sims.') pl.plot(binner.bin_centers(),stats_len.mean()/cl_input_binned -1.,label = 'sim/input -1.,binned, res ' + str(par.HD_res)) pl.xlabel('$\ell$') pl.ylim(-0.05,0.05) pl.hlines([-0.001,0.001],np.min(binner.bins_l),np.max(binner.bins_r),linestyles='--',color = 'grey') pl.legend(frameon = False) pp.savefig() pl.close() pl.figure('Unbinned Cl mean') pl.title('Unbinned Cl mean') counts = par.lib_cmb_unl.get_ellcounts() ell = np.arange(len(counts)) nz = np.where(counts[0:len(input_cl)-1] > 0) pl.plot(ell[nz],cl_unb[nz]/input_cl[nz] -1.,label = 'cl sim / input -1 ') pl.plot(ell[nz], np.sqrt(1./counts[nz]/len(mc_sims)),linestyle = '--',color = 'black',label = '$\sigma $ Gauss.') pl.plot(ell[nz], -np.sqrt(1./counts[nz]/len(mc_sims)),linestyle = '--',color = 'black') pl.plot(ell[nz],- np.sqrt(2./(2*ell[nz] + 1)/len(mc_sims)),linestyle = '--',color = 'black' ) pl.plot(ell[nz], np.sqrt(2./(2*ell[nz] + 1)/len(mc_sims)),linestyle = '--',color = 'black' ) pl.hlines([0.],0.,max(ell[nz]),color = 'grey',linestyle = '--') pl.legend(frameon= False) pp.savefig() pl.close() pl.figure('Unbinned Cl variance') pl.title('Unbinned Cl variance') counts = par.lib_cmb_unl.get_ellcounts() ell = np.arange(len(counts)) nz = np.where(counts[0:len(input_cl) - 1] > 0) pl.plot(ell[nz],sig_unb[nz]/np.sqrt(2 * input_cl[nz] ** 2 / (2 * ell[nz] + 1.)) -1.,label = 'sim sig./ Gauss sig -1. ') pl.plot(ell[nz],sig_unb[nz]/np.sqrt(input_cl[nz] ** 2 / counts[nz]) -1.,label = 'incl. discrtness') pl.ylim(-1,1.) pl.hlines([0.],0.,max(ell[nz]),color = 'grey',linestyle = '--') pl.legend(frameon = False) pp.savefig() pl.close() pp.close()
par = imp.load_source('par',parfile) if pbs.rank == 0 : print "++++ Setup ok." print "I see ",pbs.size," MPI proc." if os.getenv("OMP_NUM_THREADS") is not None : print "I see ",int(os.getenv("OMP_NUM_THREADS"))," OMP NUM THREADS" pbs.barrier() def generate_sims(sim_lib, label=''): if pbs.rank == 0 : for i, idx in enumerate_progress(np.arange(par.nsims)[::pbs.size], 'building sims:' + label): sim = sim_lib.get_sim(idx) else : for idx in np.arange(par.nsims)[pbs.rank::pbs.size] : sim = sim_lib.get_sim(idx) # Generating lowest levels sims (in fact only storing the rng states) : np.random.set_state(mk_session_seed(verbose = False)) for lib in par.base_libs: if lib is not None and not lib.is_full() : generate_sims(lib,label = lib.lib_dir) # Performing lensing operations on sims : if pbs.rank == 0 : for i, idx in enumerate_progress(np.arange(par.nsims)[::pbs.size], 'lensing sims: '): sim = par.lib_cmb_len.get_sim(idx) else : for idx in np.arange(par.nsims)[pbs.rank::pbs.size] : sim = par.lib_cmb_len.get_sim(idx) pbs.barrier() pbs.finalize()