def gradient_flat(map_in, lmax=2000): """Return the gradient maps of map_in. Compute the gradient of the map_in in Fourier space. Simultaneously low-pass the maps such that they only include modes ell < lmax. Parameters ---------- map_in: ndmap, ndarray The input map. lmax: int The cutoff ell for low-passing the maps. Returns ------- dy: ndmap, ndarray The gradient in the y-direction dx: ndmap, ndarray The gradient in the x-direction """ ly, lx = map_in.lmap() lmod = map_in.modlmap() lp = enmap.zeros(map_in.shape) lp[np.where(lmod < lmax)] = 1. map_fft = enmap.fft(map_in) dx = enmap.ifft(map_fft*lp*lx*1j).real dy = enmap.ifft(map_fft*lp*ly*1j).real return dy, dx
def test_fft(self): # Tests that ifft(ifft(imap))==imap, i.e. default normalizations are consistent shape, wcs = enmap.geometry(pos=(0, 0), shape=(3, 100, 100), res=0.01) imap = enmap.enmap(np.random.random(shape), wcs) assert np.all( np.isclose( imap, enmap.ifft(enmap.fft(imap, normalize='phy'), normalize='phy').real)) assert np.all(np.isclose(imap, enmap.ifft(enmap.fft(imap)).real))
def gather_patches_cos(self): """Assemble patch statistics for small scale lensing with cos filter. Compute the small scale (ell > 3000) temperature power at different patches across the sky as well as the average amplitude of the background temperature gradient (ell < 2000). For the small scale statistics, also apply a filter in Fourier space such that: .. math:: f_\\ell = \\cos(\\hat{\\ell}\\cdot\\hat{\\nabla T}) """ self._edge = 5 # Edge pixels to throw away p = self._p m_fft = enmap.fft(self.map_in) hp = np.zeros(self.map_in.shape) hp[np.where((self._lmod > self.lmin) & (self._lmod < self.lmax))] = 1. # Apply pre-whitening or Wiener/inverse variance filters, then top hat if self._aW and self.fid is not None: cs = CubicSpline(self.fid[0], self.fid[1]) # (ell, Cl) m_fft = m_fft / cs(self._lmod) self._Tss = enmap.ifft(m_fft * hp) self._dTy, self._dTx = gradient_flat(self.map_in, self.ldT) # Scale geometry for lower res map of patches pshp, pwcs = enmap.scale_geometry(self.map_in.shape, self.map_in.wcs, 1. / self._p) if not self.savesteps: del self.map_in, m_fft self._T2patch = enmap.zeros(pshp, pwcs) self._dTxpatch = enmap.zeros(pshp, pwcs) self._dTypatch = enmap.zeros(pshp, pwcs) self._T_sub = np.zeros((pshp[-2], pshp[-1], p, p)) for i in range(self._T2patch.shape[-2]): for j in range(self._T2patch.shape[-1]): self._dTypatch[i, j] = np.mean(self._dTy[i * p:(i + 1) * p, j * p:(j + 1) * p]) self._dTxpatch[i, j] = np.mean(self._dTx[i * p:(i + 1) * p, j * p:(j + 1) * p]) Tp = self._Tss[i * p:(i + 1) * p, j * p:(j + 1) * p] lsuby, lsubx = Tp.lmap() lsubmod = Tp.modlmap() lsubmod[0, 0] = 1. fl = 1.j*(lsubx*self._dTxpatch[i,j] + \ lsuby*self._dTypatch[i,j]) / \ (lsubmod * np.sqrt(self._dTxpatch[i,j]**2 + \ self._dTypatch[i,j]**2)) fl[0, 0] = 0. self._T_sub[i, j, :, :] = enmap.ifft(enmap.fft(Tp) * fl).real # Throw away pixels with edge effects self._T2patch[i, j] = np.var(self._T_sub[i, j, 5:-5, 5:-5]) self._dT2patch = self._dTxpatch**2 + self._dTypatch**2 if not self.savesteps: del self._dTypatch, self._dTxpatch, self._dTy, self._dTx, self._Tss del self._T_sub
def gather_patches_cos2(self): """Assemble patch statistics for small scale lensing with cos^2 filter. Compute the small scale (ell > 3000) temperature power at different patches across the sky as well as the average amplitude of the background temperature gradient (ell < 2000). For the small scale statistics, also apply a filter in Fourier space such that: .. math:: f_\\ell = \\cos^2(\\hat{\\ell}\\cdot\\hat{\\nabla T}) """ self._edge = 3 # Edge pixels to throw away p = self._p m_fft = enmap.fft(self.map_in) hp = np.zeros(self.map_in.shape) hp[np.where((self._lmod > self.lmin) & (self._lmod < self.lmax))] = 1. self._Tss = enmap.ifft(m_fft * hp) self._dTy, self._dTx = gradient_flat(self.map_in, self.ldT) # Scale geometry for lower res map of patches pshp, pwcs = enmap.scale_geometry(self.map_in.shape, self.map_in.wcs, 1. / self._p) if not self.savesteps: del self.map_in, m_fft self._T2patch = enmap.zeros(pshp, pwcs) self._dTxpatch = enmap.zeros(pshp, pwcs) self._dTypatch = enmap.zeros(pshp, pwcs) self._T_sub = np.zeros((pshp[-2], pshp[-1], p, p)) for i in range(self._T2patch.shape[-2]): for j in range(self._T2patch.shape[-1]): self._dTypatch[i, j] = np.mean(self._dTy[i * p:(i + 1) * p, j * p:(j + 1) * p]) self._dTxpatch[i, j] = np.mean(self._dTx[i * p:(i + 1) * p, j * p:(j + 1) * p]) Tp = self._Tss[i * p:(i + 1) * p, j * p:(j + 1) * p] lsuby, lsubx = Tp.lmap() lsubmod = Tp.modlmap() lsubmod[0, 0] = 1. # Avoid divide by 0; set fl here to 0 later fl = (lsubx*self._dTxpatch[i,j] + \ lsuby*self._dTypatch[i,j])**2 / \ (lsubmod * np.sqrt(self._dTxpatch[i,j]**2 + \ self._dTypatch[i,j]**2))**2 fl[0, 0] = 0. self._T_sub[i, j, :, :] = enmap.ifft(enmap.fft(Tp) * fl).real # Throw away pixels with edge effects self._T2patch[i, j] = np.var(self._T_sub[i, j, 3:-3, 3:-3]) self._dT2patch = self._dTxpatch**2 + self._dTypatch**2 if not self.savesteps: del self._dTypatch, self._dTxpatch, self._dTy, self._dTx, self._Tss del self._T_sub
def filter_weighted(imap, ivar, filter, tol=1e-4, ref=0.9): """Filter enmap imap with the given 2d fourier filter while weithing spatially with ivar""" filter = 1 - filter omap = enmap.ifft(filter * enmap.fft(imap * ivar)).real div = enmap.ifft(filter * enmap.fft(ivar)).real del filter # Avoid division by very low values div = np.maximum(div, np.percentile(ivar[::10, ::10], ref * 100) * tol) # omap = imap - rhs/div omap /= div del div omap *= -1 omap += imap omap *= imap != 0 return omap
def gather_patches_plain(self): """Assemble patch statistics relevant to lensing at small scales. Compute the small scale (ell > lmin) temperature power at different patches across the sky as well as the average amplitude of the background temperature gradient (ell < ldT). """ self._edge = 0 # Not throwing away edge pixels m_fft = enmap.fft(self.map_in) hp = np.zeros(self.map_in.shape) hp[np.where((self._lmod > self.lmin) & (self._lmod < self.lmax))] = 1. self._Tss = enmap.ifft(m_fft * hp) self._dTy, self._dTx = gradient_flat(self.map_in, self.ldT) # Scale geometry for lower res map of patches pshp, pwcs = enmap.scale_geometry(self.map_in.shape, self.map_in.wcs, 1. / self._p) if not self.savesteps: del self.map_in, m_fft self._T2patch = enmap.zeros(pshp, pwcs) self._dTxpatch = enmap.zeros(pshp, pwcs) self._dTypatch = enmap.zeros(pshp, pwcs) Trs = self._Tss[:pshp[-2] * self._p, :pshp[-1] * self._p].reshape( [pshp[-2], self._p, pshp[-1], self._p]) dTxrs = self._dTx[:pshp[-2] * self._p, :pshp[-1] * self._p].reshape( [pshp[-2], self._p, pshp[-1], self._p]) dTyrs = self._dTy[:pshp[-2] * self._p, :pshp[-1] * self._p].reshape( [pshp[-2], self._p, pshp[-1], self._p]) self._T2patch[:, :] = np.var(Trs, axis=(1, 3)) self._dTypatch[:, :] = np.mean(dTyrs, axis=(1, 3)) self._dTxpatch[:, :] = np.mean(dTxrs, axis=(1, 3)) self._dT2patch = self._dTxpatch**2 + self._dTypatch**2 if not self.savesteps: del self._dTypatch, self._dTxpatch, self._dTy, self._dTx, self._Tss
def cutout_rec(shape, wcs, feed_dict, cmask, kmask, map1_k, map2_k): """ cutout lensing reconstruction """ feed_dict['X'] = map1_k feed_dict['Y'] = map2_k # unnormalized lensing map in fourier space ukappa_k = s.unnormalized_quadratic_estimator(shape, wcs, feed_dict, "hu_ok", "TT", xmask=cmask, ymask=cmask) # normaliztion norm_k = s.A_l(shape, wcs, feed_dict, "hu_ok", "TT", xmask=cmask, ymask=cmask, kmask=kmask) # noise noise_2d = s.N_l_from_A_l_optimal(shape, wcs, norm_k) # normalized Fourier space CMB lensing convergence map kappa_k = norm_k * ukappa_k # real space CMB lensing convergence map kappa = enmap.ifft(kappa_k, normalize='phys') return kappa, noise_2d
def cmbmap2alm(i, mtype, p, f, r): fmap = enmap.read_map( f.imap[mtype][i]) # load flatsky K-space combined map # FT print('2D ell filtering in F-space and assign to healpix map') alm = enmap.fft(fmap) # remove some Fourier modes shape, wcs = fmap.shape, fmap.wcs kmask = mask_kspace(shape, wcs, lmin=p.lcut, lmax=4000, lxcut=90, lycut=50) alm[kmask < 0.5] = 0 # alm -> map fmap = enmap.ifft(alm).real # transform cmb map to healpix hpmap = enmap.to_healpix(fmap, nside=p.nside) # from map to alm hpmap = r.w * hpmap # masking #hp.fitsfunc.write_map(f.omap[mtype][i],hpmap,overwrite=True) alm = curvedsky.utils.hp_map2alm(p.nside, p.lmax, p.lmax, hpmap) print("save to file") pickle.dump((alm), open(f.alm[mtype][i], "wb"), protocol=pickle.HIGHEST_PROTOCOL)
def generate_noise_sim(covsqrt, ivars, seed=None, dtype=None): """ Supports only two cases 1) nfreqs>=1,npol=3 2) nfreqs=1,npol=1 """ if isinstance(seed, int): seed = (seed, ) assert np.all(np.isfinite(covsqrt)) shape, wcs = covsqrt.shape, covsqrt.wcs Ny, Nx = shape[-2:] ncomps = covsqrt.shape[0] assert ncomps == covsqrt.shape[1] assert ((ncomps % 3) == 0) or (ncomps == 1) nfreqs = 1 if ncomps == 1 else ncomps // 3 if ncomps == 1: npol = 1 else: npol = 3 wmaps = enmap.extract(ivars, shape[-2:], wcs) nsplits = wmaps.shape[1] if dtype is np.float32: ctype = np.complex64 elif dtype is np.float64: ctype = np.complex128 # Old way with loop kmap = [] for i in range(nsplits): if seed is None: np.random.seed(None) else: np.random.seed(seed + (i, )) rmap = enmap.rand_gauss_harm((ncomps, Ny, Nx), covsqrt.wcs).astype(ctype) kmap.append(enmap.map_mul(covsqrt, rmap)) del covsqrt, rmap kmap = enmap.enmap(np.stack(kmap), wcs) outmaps = enmap.ifft(kmap, normalize="phys").real del kmap # Need to test this more ; it's only marginally faster and has different seed behaviour # covsqrt = icovsqrt # np.random.seed(seed) # rmap = enmap.rand_gauss_harm((nsplits,ncomps,Ny, Nx),covsqrt.wcs) # kmap = enmap.samewcs(np.einsum("abyx,cbyx->cayx", covsqrt, rmap),rmap) # outmaps = enmap.ifft(kmap, normalize="phys").real # isivars = 1/np.sqrt(wmaps) with np.errstate(divide='ignore', invalid='ignore'): isivars = ((1. / wmaps) - (1. / wmaps.sum(axis=1)[:, None, ...]))**0.5 isivars[~np.isfinite(isivars)] = 0 assert np.all(np.isfinite(outmaps)) # Divide by hits for ifreq in range(nfreqs): outmaps[:, ifreq * npol:(ifreq + 1) * npol, ...] = outmaps[:, ifreq * npol:(ifreq + 1) * npol, ...] * isivars[ifreq, ...] * np.sqrt(nsplits) retmaps = outmaps.reshape((nsplits, nfreqs, npol, Ny, Nx)).swapaxes(0, 1) assert np.all(np.isfinite(retmaps)) return retmaps, wmaps
def sim_noise_oneoverf(ivar, apod, lknee=3000, lmin=50, alpha=-3.5): l = np.maximum(ivar.modlmap(), lmin) fscale = (1 + (l / lknee)**alpha)**0.5 ref = np.mean(ivar[ivar != 0]) sim = enmap.rand_gauss(ivar.shape, ivar.wcs, dtype=ivar.dtype) sim = enmap.ifft(enmap.fft(sim) * fscale).real * np.maximum( ivar, ref * 1e-3)**-0.5 * apod**2 return sim
def filter_riseset(imap, ivar, riseset, filter, tol=1e-4, ref=0.9): """Filter enmap imap with the given 2d fourier filter while weighting spatially with ivar and allowing for rise vs set-dependent pickup. riseset should be a map with values between -1 and 1 that determines the balance between rising and setting scans, with 0 meaning equal weight from both. Overall this is only a marginal improvement over filter_weighted depsite being quite a bit heaver, and it only helps a bit for the pa7 residual stripe issue it was invented to deal with, so it probably isn't worth using. """ # We model the map as m = Pa+n, where a is [2] is the rising and setting pickup in # a single pixel, and P is the map's response to this pickup, which is # P = [x,1-x]*filter, where x=(1-riseset)/2. Given this we can solve for a as: # a = (P'N"P)"P'N"m, where N" is ivar. Written out this becomes, for a single pixel. # rhs = ifft(filter*fft([x,1-x]*ivar*imap)) # div = ifft(filter*fft([x,1-x]*ivar*[x,1-x]*ifft(filter) # Hm.... The problem here is that we're assuming that there's nothing in the other pixels. # That's not what we did in filter_weighted. Let's just do something simple for now. x = (1 - riseset) / 2 x1x = np.array([x, 1 - x]) del x # Broadcast to imap shape x1x = x1x[(slice(None), ) + (None, ) * (imap.ndim - 2)] print(imap.shape, x1x.shape) filter = 1 - filter rhs = enmap.ifft(filter * enmap.fft(x1x * ivar * imap)).real div = enmap.ifft(filter * enmap.fft(x1x[:, None] * x1x[None, :] * ivar)).real del filter # Avoid division by very low values ref = np.percentile(ivar[::10, ::10], ref * 100) * tol for i in range(2): div[i, i] = np.maximum(div[i, i], ref) # Solve the system rhs = np.moveaxis(rhs, 0, -1) div = np.moveaxis(div, (0, 1), (-2, -1)) omap = np.linalg.solve(div, rhs) del rhs, div omap = np.sum(x1x * np.moveaxis(omap, -1, 0), 0) del x1x omap = enmap.ndmap(omap, imap.wcs) omap *= -1 omap += imap omap *= imap != 0 return omap
def beam_match(imap, f1, f2): """f1, f2 are fcodes instead of the actual frequency centers. It assumes that the first one (f1) has larger beam, so f2 will be matched to it. """ from common import fwhms l = imap.modlmap() bmap_f1 = np.exp(-0.5 * l**2 * (fwhms[f1] * u.fwhm * u.arcmin)**2) bmap_f2 = np.exp(-0.5 * l**2 * (fwhms[f2] * u.fwhm * u.arcmin)**2) rmap = enmap.ifft(enmap.fft(imap) * (bmap_f1 / np.maximum(bmap_f2, 1e-3))).real return rmap
def smooth_ps_gauss(ps2d, lsigma=200): """Smooth a 2d power spectrum to the target resolution in l. Simple gaussian smoothing avoids ringing.""" # This hasn't been tested in isolation, but breaks when used in smooth_ps_mixed # First get our pixel size in l ly, lx = enmap.laxes(ps2d.shape, ps2d.wcs) ires = np.array([ly[1], lx[1]]) sigma_pix = np.abs(lsigma / ires) fmap = enmap.fft(ps2d) ky = np.fft.fftfreq(ps2d.shape[-2]) * sigma_pix[0] kx = np.fft.fftfreq(ps2d.shape[-1]) * sigma_pix[1] kr2 = ky[:, None]**2 + kx[None, :]**2 fmap *= np.exp(-0.5 * kr2) return enmap.ifft(fmap).real
def remove_lxly(fmap, lmin=100, lmax=4096, lxcut=90, lycut=50): alm = enmap.fft(fmap) shape, wcs = fmap.shape, fmap.wcs kmask = define_lmask(shape, wcs, lmin=lmin, lmax=lmax, lxcut=lxcut, lycut=lycut) #print(np.shape(kmask),np.shape(alm)) alm[kmask < 0.5] = 0 fmap_fl = enmap.ifft(alm).real return fmap_fl
def filter_fast(map, vk_mask=None, hk_mask=None, normalize="phys", inv_pixwin_lxly=None): """Filter the map in Fourier space removing modes in a horizontal and vertical band defined by hk_mask and vk_mask. This is a faster version that what is implemented in pspy We also include an option for removing the pixel window function Parameters --------- map: ``so_map`` the map to be filtered vk_mask: list with 2 elements format is fourier modes [-lx,+lx] hk_mask: list with 2 elements format is fourier modes [-ly,+ly] normalize: string optional normalisation of the Fourier transform inv_pixwin_lxly: 2d array the inverse of the pixel window function in fourier space """ lymap, lxmap = map.data.lmap() ly, lx = lymap[:,0], lxmap[0,:] # filtered_map = map.copy() ft = enmap.fft(map.data, normalize=normalize) if vk_mask is not None: id_vk = np.where((lx > vk_mask[0]) & (lx < vk_mask[1])) if hk_mask is not None: id_hk = np.where((ly > hk_mask[0]) & (ly < hk_mask[1])) if map.ncomp == 1: if vk_mask is not None: ft[: , id_vk] = 0. if hk_mask is not None: ft[id_hk , :] = 0. if map.ncomp == 3: for i in range(3): if vk_mask is not None: ft[i, : , id_vk] = 0. if hk_mask is not None: ft[i, id_hk , :] = 0. if inv_pixwin_lxly is not None: ft *= inv_pixwin_lxly map.data[:] = np.real(enmap.ifft(ft, normalize=normalize)) return map
def process_map(imap, box=None, deconvolve=False, freq=None, smooth=None): if deconvolve: fwhm = fwhms[freq] l = imap.modlmap() bmap = np.exp(-0.5*l**2*(fwhm*u.fwhm*u.arcmin)**2) bmap = np.maximum(bmap, 1e-3) fmap = enmap.fft(imap) omap = enmap.ifft(fmap/bmap).real else: omap = imap if smooth is not None: omap = enmap.smooth_gauss(omap, smooth*u.fwhm*u.arcmin) if box is not None: omap = omap.submap(box) if args.comp == 0: return np.log10(omap[0]) if args.comp == 12: return np.sum(omap[1:]**2, axis=0)**0.5 else: return omap[args.comp]
def get_filtered_map(map, binary, vk_mask, hk_mask, normalize=False): """Filter the map in Fourier space removing modes in a horizontal and vertical band defined by hk_mask and vk_mask. Note that we mutliply the maps by a binary mask before doing this operation in order to remove pathological pixels Parameters --------- orig_map: ``so_map`` the map to be filtered binary: ``so_map`` a binary mask removing pathological pixels vk_mask: list with 2 elements format is fourier modes [-lx,+lx] hk_mask: list with 2 elements format is fourier modes [-ly,+ly] """ if map.ncomp == 1: map.data *= binary.data else: map.data[:] *= binary.data lymap, lxmap = map.data.lmap() ly, lx = lymap[:, 0], lxmap[0, :] # filtered_map = map.copy() ft = enmap.fft(map.data, normalize=normalize) if vk_mask is not None: id_vk = np.where((lx > vk_mask[0]) & (lx < vk_mask[1])) if hk_mask is not None: id_hk = np.where((ly > hk_mask[0]) & (ly < hk_mask[1])) if map.ncomp == 1: if vk_mask is not None: ft[:, id_vk] = 0.0 if hk_mask is not None: ft[id_hk, :] = 0.0 if map.ncomp == 3: if vk_mask is not None: ft[:, :, id_vk] = 0.0 if hk_mask is not None: ft[:, id_hk, :] = 0.0 map.data[:] = np.real(enmap.ifft(ft, normalize=normalize)) return map
def smooth_ps_grid(ps, res, alpha=4, log=False, ndof=2): """Smooth a 2d power spectrum to the target resolution in l""" # First get our pixel size in l lx, ly = enmap.laxes(ps.shape, ps.wcs) ires = np.array([lx[1],ly[1]]) smooth = np.abs(res/ires) # We now know how many pixels to somoth by in each direction, # so perform the actual smoothing if log: ps = np.log(ps) fmap = enmap.fft(ps) ky = np.fft.fftfreq(ps.shape[-2]) kx = np.fft.fftfreq(ps.shape[-1]) fmap /= 1 + np.abs(2*ky[:,None]*smooth[0])**alpha fmap /= 1 + np.abs(2*kx[None,:]*smooth[1])**alpha ps = enmap.ifft(fmap).real if log: ps = np.exp(ps)*log_smooth_corrections[ndof] return ps
def map_act_hist(patches, numerics, wf, masks, apo_masks, \ negbins, posbins) :#{{{ # @TODO: check final pixel count! # @TODO check changing cmb or noise power changes variance as expected. it will. # @TODO plot histogram with and without cmb # @TODO plot histogram with and without WF # maps are sub-divided into six patches; analyze each separately and then get the PDF of the whole map dat_tot = np.array([]) for i in range(numerics['n_patch']): patch = patches[i] mask = copy.deepcopy(apo_masks[i]) patch *= mask #apply mask modlmap = patch.modlmap( ) #map of |\vec{ell}| in 2D Fourier domain for the patch Fell2d = interp1d(wf[:, 0], wf[:, 1], bounds_error=False, fill_value=0.)( modlmap) #get filter in 2D Fourier domain kmap = enmap.fft(patch, normalize="phys") #FFT the patch to 2D Fourier domain kfilt = kmap * Fell2d patch_filt = enmap.ifft(kfilt, normalize="phys").real # point source mask (different for each patch) newMask = copy.deepcopy(masks[i]) newMask[363, :] = 1. newMask[:, 2181] = 1. la = np.where(newMask < 0.1) mask[la] = 0. ### pick out unmasked data loc2 = np.where(mask > 0.9) dat = patch_filt[loc2] #*fraction dat_tot = np.append(dat_tot, np.ravel(dat)) # histogram the unmasked data histneg, bin_edgesneg = np.histogram(dat_tot, negbins) histpos, bin_edgespos = np.histogram(dat_tot, posbins[::-1]) histall = np.concatenate((histneg, histpos)) PDFpatch = histall / float(len(dat_tot)) return PDFpatch
def kspace_filter_fast(map, vk_mask=None, hk_mask=None, normalize="phys"): """Filter the map in Fourier space removing modes in a horizontal and vertical band defined by hk_mask and vk_mask. This is a faster version that what is implemented in pspy Parameters --------- map: ``so_map`` the map to be filtered vk_mask: list with 2 elements format is fourier modes [-lx,+lx] hk_mask: list with 2 elements format is fourier modes [-ly,+ly] """ lymap, lxmap = map.data.lmap() ly, lx = lymap[:,0], lxmap[0,:] # filtered_map = map.copy() ft = enmap.fft(map.data, normalize=normalize) if vk_mask is not None: id_vk = np.where((lx > vk_mask[0]) & (lx < vk_mask[1])) if hk_mask is not None: id_hk = np.where((ly > hk_mask[0]) & (ly < hk_mask[1])) if map.ncomp == 1: if vk_mask is not None: ft[: , id_vk] = 0. if hk_mask is not None: ft[id_hk , :] = 0. if map.ncomp == 3: for i in range(3): if vk_mask is not None: ft[i, : , id_vk] = 0. if hk_mask is not None: ft[i, id_hk , :] = 0. map.data[:] = np.real(enmap.ifft(ft, normalize=normalize)) return map
def deconvolve_pixwin_CAR(orig_map, binary, inv_pixwin_lxly, norm=0): """Deconvolve the two dimensional CAR pixel window function Parameters --------- orig_map: ``so_map`` the map to be filtered binary: ``so_map`` a binary mask removing pathological pixels inv_pixwin_lxly: 2d array the inverse of the pixel window function in fourier space norm: integer everytime we do a FFT + IFFT operation we get a normalisation cst to take care of this is counting the number of occurence of it, note that we could correct directly by using normalize="physical", but dividing the map by a number is slow so we prefer to do it later in the code. """ orig_map.data *= binary.data ft = enmap.fft(orig_map.data, normalize=False) ft *= inv_pixwin_lxly orig_map.data = enmap.ifft(ft, normalize=False).real norm += 1 return norm, orig_map
def compute_map(self,oshape,owcs,qid,pixwin_taper_deg=0.3,pixwin_pad_deg=0.3, include_cmb=True,include_tsz=True,include_fgres=True,sht_beam=True): """ 1. get total alm 2. apply beam, and pixel window if Planck 3. ISHT 4. if ACT, apply a small taper and apply pixel window in Fourier space """ # pad to a slightly larger geometry tot_pad_deg = pixwin_taper_deg + pixwin_pad_deg res = maps.resolution(oshape,owcs) pix = np.deg2rad(tot_pad_deg)/res omap = enmap.pad(enmap.zeros((3,)+oshape,owcs),pix) ishape,iwcs = omap.shape[-2:],omap.wcs # get data model dm = sints.models[sints.arrays(qid,'data_model')](region_shape=ishape,region_wcs=iwcs,calibrated=True) # 1. get total alm array_index = self.qids.index(qid) tot_alm = int(include_cmb)*self.alms['cmb'] if include_tsz: try: assert self.tsz_fnu.ndim==2 tot_alm[0] = tot_alm[0] + hp.almxfl(self.alms['comptony'][0] ,self.tsz_fnu[array_index]) except: tot_alm[0] = tot_alm[0] + self.alms['comptony'][0] * self.tsz_fnu[array_index] if self.cfgres is not None: tot_alm[0] = tot_alm[0] + int(include_fgres)*self.alms['fgres'][array_index] assert tot_alm.ndim==2 assert tot_alm.shape[0]==3 ells = np.arange(self.lmax+1) # 2. get beam, and pixel window for Planck if sht_beam: beam = tutils.get_kbeam(qid,ells,sanitize=False,planck_pixwin=False) # NEVER SANITIZE THE BEAM IN A SIMULATION!!! for i in range(3): tot_alm[i] = hp.almxfl(tot_alm[i],beam) if dm.name=='planck_hybrid': pixwint,pixwinp = hp.pixwin(nside=tutils.get_nside(qid),lmax=self.lmax,pol=True) tot_alm[0] = hp.almxfl(tot_alm[0],pixwint) tot_alm[1] = hp.almxfl(tot_alm[1],pixwinp) tot_alm[2] = hp.almxfl(tot_alm[2],pixwinp) # 3. ISHT omap = curvedsky.alm2map(np.complex128(tot_alm),omap,spin=[0,2]) assert omap.ndim==3 assert omap.shape[0]==3 if not(sht_beam): taper,_ = maps.get_taper_deg(ishape,iwcs,taper_width_degrees=pixwin_taper_deg,pad_width_degrees=pixwin_pad_deg) modlmap = omap.modlmap() beam = tutils.get_kbeam(qid,modlmap,sanitize=False,planck_pixwin=True) kmap = enmap.fft(omap*taper,normalize='phys') kmap = kmap * beam # 4. if ACT, apply a small taper and apply pixel window in Fourier space if dm.name=='act_mr3': if sht_beam: taper,_ = maps.get_taper_deg(ishape,iwcs,taper_width_degrees=pixwin_taper_deg,pad_width_degrees=pixwin_pad_deg) pwin = tutils.get_pixwin(ishape[-2:]) if sht_beam: omap = maps.filter_map(omap*taper,pwin) else: kmap = kmap * pwin if not(sht_beam): omap = enmap.ifft(kmap,normalize='phys').real return enmap.extract(omap,(3,)+oshape[-2:],owcs)
# wmap = enmap.ifft(enmap.fft(wmap)*matched_filter).real**2 # lim = max(np.median(wmap)*tol1**2, np.max(wmap)*tol2**2) # mask |= wmap > lim # return mask comm = mpi.COMM_WORLD beam1d = get_beam(args.beam) ifiles = sorted(sum([glob.glob(ifile) for ifile in args.ifiles], [])) for ind in range(comm.rank, len(ifiles), comm.size): ifile = ifiles[ind] if args.verbose: print(ifile) ofile = args.odir + "/" + ifile imap = enmap.read_map(ifile) if args.mask is not None: mask = imap == args.mask if args.apodize: imap = imap.apod(args.apodize) # We will apply a semi-matched-filter to T l = np.maximum(1, imap.modlmap()) beam2d = enmap.samewcs(np.interp(l, np.arange(len(beam1d)), beam1d), imap) matched_filter = (1 + (l / args.lknee)**args.alpha)**-1 * beam2d fmap = enmap.map2harm(imap, iau=True) fmap[0] *= matched_filter omap = enmap.ifft(fmap).real if args.mask is not None: omap[mask] = 0 del mask utils.mkdir(os.path.dirname(ofile)) enmap.write_map(ofile, omap) del omap
kmask = maps.mask_kspace(shape,wcs,lmin=Lmin,lmax=Lmax) feed_dict['uC_T_T'] = cov feed_dict['tC_T_T'] = cov + (wnoise*np.pi/180/60)**2 / kbeam**2 qe = symlens.QE(shape,wcs,feed_dict,'hu_ok','TT',xmask=xmask,ymask=ymask,kmask=kmask) s = stats.Stats() for task in my_tasks: cmb = maps.filter_map(mgen.get_map(seed=(1,task))[0],kbeam) nseed = (2,task) nmap = maps.white_noise(shape,wcs,noise_muK_arcmin=None,seed=nseed,ipsizemap=pmap,div=ivar) obs = cmb + nmap kobs = enmap.fft(obs,normalize='phys')/kbeam kobs[~np.isfinite(kobs)] = 0 feed_dict['X'] = kobs feed_dict['Y'] = kobs krecon = qe.reconstruct(feed_dict) print(cmb.shape,nmap.shape,krecon.shape) s.add_to_stack('kreal',krecon.real) s.add_to_stack('kimag',krecon.imag) s.get_stacks() mf = enmap.ifft(s.stacks['kreal'] + 1j*s.stacks['kimag'],normalize='phys') io.hplot(mf,'mf.png')
pathMap = "/global/cscratch1/sd/eschaan/project_ksz_act_planck/data/planck_act_coadd_2020_02_28_r2/" + "act_planck_s08_s18_cmb_f150_daynight_map.fits" # read maps iMap = enmap.read_map(pathMap) # do the reconvolution in Fourier space mapF = enmap.fft(iMap) lMap = np.sqrt(np.sum(iMap.lmap()**2,0)) ########################################################################## # Reconvolve the map print("Reconvolve 150 to 90") pathOutMap = "/global/cscratch1/sd/eschaan/project_ksz_act_planck/data/planck_act_coadd_2020_02_28_r2/" + "act_planck_s08_s18_cmb_f150_daynight_map_reconvto90.fits" oMap = enmap.ifft(mapF * fBeam90F(lMap) / fBeam150F(lMap)).real enmap.write_map(pathOutMap, oMap) print("check finite sum "+str(np.sum(oMap))) print("Reconvolve 150 to TileC deproj") pathOutMap = "/global/cscratch1/sd/eschaan/project_ksz_act_planck/data/planck_act_coadd_2020_02_28_r2/" + "act_planck_s08_s18_cmb_f150_daynight_map_reconvtotilecdeproj.fits" oMap = enmap.ifft(mapF * fBeamTilecDeprojF(lMap) / fBeam150F(lMap)).real enmap.write_map(pathOutMap, oMap) print("check finite sum "+str(np.sum(oMap))) print("Reconvolve 150 to TileC") pathOutMap = "/global/cscratch1/sd/eschaan/project_ksz_act_planck/data/planck_act_coadd_2020_02_28_r2/" + "act_planck_s08_s18_cmb_f150_daynight_map_reconvtotilec.fits" oMap = enmap.ifft(mapF * fBeamTilecF(lMap) / fBeam150F(lMap)).real enmap.write_map(pathOutMap, oMap)
def build_and_save_ilc(arrays,region,version,cov_version,beam_version, solutions,beams,chunk_size, effective_freq,overwrite,maxval,unsanitized_beam=False,do_weights=False, pa1_shift = None, pa2_shift = None, pa3_150_shift = None, pa3_090_shift = None, no_act_color_correction=False, ccor_exp = -1, isotropize=False, isotropize_width=20): print("Chunk size is ", chunk_size*64./8./1024./1024./1024., " GB.") def warn(): print("WARNING: no bandpass file found. Assuming array ",dm.c['id']," has no response to CMB, tSZ and CIB.") aspecs = tutils.ASpecs().get_specs bandpasses = not(effective_freq) savedir = tutils.get_save_path(version,region) covdir = tutils.get_save_path(cov_version,region) assert os.path.exists(covdir) if not(overwrite): assert not(os.path.exists(savedir)), \ "This version already exists on disk. Please use a different version identifier." try: os.makedirs(savedir) except: if overwrite: pass else: raise mask = enmap.read_map(covdir+"tilec_mask.fits") shape,wcs = mask.shape,mask.wcs Ny,Nx = shape modlmap = enmap.modlmap(shape,wcs) arrays = arrays.split(',') narrays = len(arrays) kcoadds = [] kbeams = [] bps = [] names = [] lmins = [] lmaxs = [] shifts = [] cfreqs = [] lbeams = [] ells = np.arange(0,modlmap.max()) for i,qid in enumerate(arrays): dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=True) lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid) cfreqs.append(cfreq) lmins.append(lmin) lmaxs.append(lmax) names.append(qid) if dm.name=='act_mr3': season,array1,array2 = sints.arrays(qid,'season'),sints.arrays(qid,'array'),sints.arrays(qid,'freq') array = '_'.join([array1,array2]) elif dm.name=='planck_hybrid': season,patch,array = None,None,sints.arrays(qid,'freq') else: raise ValueError kcoadd_name = covdir + "kcoadd_%s.npy" % qid kmask = maps.mask_kspace(shape,wcs,lmin=lmin,lmax=lmax) kcoadd = enmap.enmap(np.load(kcoadd_name),wcs) dtype = kcoadd.dtype kcoadds.append(kcoadd.copy()*kmask) kbeam = tutils.get_kbeam(qid,modlmap,sanitize=not(unsanitized_beam),version=beam_version,planck_pixwin=True) if dm.name=='act_mr3': lbeam = tutils.get_kbeam(qid,ells,sanitize=not(unsanitized_beam),version=beam_version,planck_pixwin=False) # note no pixwin but doesnt matter since no ccorr for planck elif dm.name=='planck_hybrid': lbeam = None else: raise ValueError lbeams.append(lbeam) kbeams.append(kbeam.copy()) if bandpasses: try: fname = dm.get_bandpass_file_name(array) bps.append("data/"+fname) if (pa1_shift is not None) and 'PA1' in fname: shifts.append(pa1_shift) elif (pa2_shift is not None) and 'PA2' in fname: shifts.append(pa2_shift) elif (pa3_150_shift is not None) and ('PA3' in fname) and ('150' in fname): shifts.append(pa3_150_shift) elif (pa3_090_shift is not None) and ('PA3' in fname) and ('090' in fname): shifts.append(pa3_90_shift) else: shifts.append(0) except: warn() bps.append(None) else: try: bps.append(cfreq) except: warn() bps.append(None) kcoadds = enmap.enmap(np.stack(kcoadds),wcs) # Read Covmat cov = maps.SymMat(narrays,shape[-2:]) for aindex1 in range(narrays): for aindex2 in range(aindex1,narrays): icov = enmap.enmap(np.load(covdir+"tilec_hybrid_covariance_%s_%s.npy" % (names[aindex1],names[aindex2])),wcs) if isotropize: bin_edges = np.append([0.],np.arange(min(lmins),modlmap.max(),isotropize_width)) binner = stats.bin2D(modlmap,bin_edges) ls,c1d = binner.bin(icov) icov = maps.interp(ls,c1d)(modlmap) if aindex1==aindex2: icov[modlmap<lmins[aindex1]] = maxval icov[modlmap>lmaxs[aindex1]] = maxval cov[aindex1,aindex2] = icov cov.data = enmap.enmap(cov.data,wcs,copy=False) covfunc = lambda sel: cov.to_array(sel,flatten=True) assert cov.data.shape[0]==((narrays*(narrays+1))/2) # FIXME: generalize assert np.all(np.isfinite(cov.data)) # Make responses responses = {} for comp in ['tSZ','CMB','CIB']: if bandpasses: if no_act_color_correction: responses[comp] = tfg.get_mix_bandpassed(bps, comp, bandpass_shifts=shifts) else: responses[comp] = tfg.get_mix_bandpassed(bps, comp, bandpass_shifts=shifts, ccor_cen_nus=cfreqs, ccor_beams=lbeams, ccor_exps = [ccor_exp] * narrays) else: responses[comp] = tfg.get_mix(bps, comp) ilcgen = ilc.chunked_ilc(modlmap,np.stack(kbeams),covfunc,chunk_size,responses=responses,invert=True) # Initialize containers solutions = solutions.split(',') data = {} kcoadds = kcoadds.reshape((narrays,Ny*Nx)) for solution in solutions: data[solution] = {} comps = solution.split('-') data[solution]['comps'] = comps if len(comps)<=2: data[solution]['noise'] = enmap.zeros((Ny*Nx),wcs) if len(comps)==2: data[solution]['cnoise'] = enmap.zeros((Ny*Nx),wcs) data[solution]['kmap'] = enmap.zeros((Ny*Nx),wcs,dtype=dtype) # FIXME: reduce dtype? if do_weights and len(comps)<=2: for qid in arrays: data[solution]['weight_%s' % qid] = enmap.zeros((Ny*Nx),wcs) for chunknum,(hilc,selchunk) in enumerate(ilcgen): print("ILC on chunk ", chunknum+1, " / ",int(modlmap.size/chunk_size)+1," ...") for solution in solutions: comps = data[solution]['comps'] if len(comps)==1: # GENERALIZE data[solution]['noise'][selchunk] = hilc.standard_noise(comps[0]) if do_weights: weight = hilc.standard_weight(comps[0]) data[solution]['kmap'][selchunk] = hilc.standard_map(kcoadds[...,selchunk],comps[0]) elif len(comps)==2: data[solution]['noise'][selchunk] = hilc.constrained_noise(comps[0],comps[1]) data[solution]['cnoise'][selchunk] = hilc.cross_noise(comps[0],comps[1]) ret = hilc.constrained_map(kcoadds[...,selchunk],comps[0],comps[1],return_weight=do_weights) if do_weights: data[solution]['kmap'][selchunk],weight = ret else: data[solution]['kmap'][selchunk] = ret elif len(comps)>2: data[solution]['kmap'][selchunk] = np.nan_to_num(hilc.multi_constrained_map(kcoadds[...,selchunk],comps[0],*comps[1:])) if len(comps)<=2 and do_weights: for qind,qid in enumerate(arrays): data[solution]['weight_%s' % qid][selchunk] = weight[qind] del ilcgen,cov # Reshape into maps name_map = {'CMB':'cmb','tSZ':'comptony','CIB':'cib'} beams = beams.split(',') for solution,beam in zip(solutions,beams): comps = "tilec_single_tile_"+region+"_" comps = comps + name_map[data[solution]['comps'][0]]+"_" if len(data[solution]['comps'])>1: comps = comps + "deprojects_"+ '_'.join([name_map[x] for x in data[solution]['comps'][1:]]) + "_" comps = comps + version if do_weights and len(data[solution]['comps'])<=2: for qind,qid in enumerate(arrays): enmap.write_map("%s/%s_%s_weight.fits" % (savedir,comps,qid), enmap.enmap(data[solution]['weight_%s' % qid].reshape((Ny,Nx)),wcs)) try: noise = enmap.enmap(data[solution]['noise'].reshape((Ny,Nx)),wcs) enmap.write_map("%s/%s_noise.fits" % (savedir,comps),noise) except: pass try: cnoise = enmap.enmap(data[solution]['cnoise'].reshape((Ny,Nx)),wcs) enmap.write_map("%s/%s_cross_noise.fits" % (savedir,comps),cnoise) except: pass ells = np.arange(0,modlmap.max(),1) try: fbeam = float(beam) kbeam = maps.gauss_beam(modlmap,fbeam) lbeam = maps.gauss_beam(ells,fbeam) except: qid = beam bfunc = lambda x: tutils.get_kbeam(qid,x,version=beam_version,sanitize=not(unsanitized_beam),planck_pixwin=False) kbeam = bfunc(modlmap) lbeam = bfunc(ells) kmap = enmap.enmap(data[solution]['kmap'].reshape((Ny,Nx)),wcs) smap = enmap.ifft(kbeam*kmap,normalize='phys').real enmap.write_map("%s/%s.fits" % (savedir,comps),smap) io.save_cols("%s/%s_beam.txt" % (savedir,comps),(ells,lbeam),header="ell beam") enmap.write_map(savedir+"/tilec_mask.fits",mask)
def map_ifft(x): return enmap.ifft(x).real def corrfun_thumb(corr, n):
def filter_simple(imap, filter): return enmap.ifft(enmap.fft(imap) * filter).real
# Beam infos: # https://phy-wiki.princeton.edu/polwiki/pmwiki.php?n=BeamAnalysis.BeamDistributionCenter # - 150GHz S16 PA2: Omega = 191 nsr, ie fwhm = 1.41 arcmin # - 90GHz S16 PA3: Omega = 516 nsr, ie fwhm = 2.32 arcmin # In[29]: # beam sigmas in radians (convert from fwhm to sigma) s150 = 1.41 * np.pi / (180. * 60.) / np.sqrt(8. * np.log(2.)) s90 = 2.32 * np.pi / (180. * 60.) / np.sqrt(8. * np.log(2.)) # do the reconvolution in Fourier space pact90ReconvF = enmap.fft(pact90) l2 = np.sum(pact90.lmap()**2, 0) pact90ReconvF *= np.exp(-0.5 * l2 * (s150**2 - s90**2)) pact90 = enmap.ifft(pact90ReconvF).real # # Measure power spectra # In[ ]: #hMask = enmap.to_healpix(fullMask) #h150 = enmap.to_healpix(pact150) #lCen, ClPACT150, sClPACT150 = powerSpectrum(h150, mask=hMask, theory=[flensedTT, ftotal], fsCl=None, nBins=201, lRange=None, plot=False, path=None, save=False) #h90 = enmap.to_healpix(pact90) #lCen, ClPACT90, sClPACT90 = powerSpectrum(h90, mask=hMask, theory=[flensedTT, ftotal], fsCl=None, nBins=201, lRange=None, plot=False, path=None, save=False) # In[ ]:
kmask = maps.mask_kspace(mask.shape, mask.wcs, lmin=lmin, lmax=30000) # Get coadd map of reference array kmap = enmap.fft( dm.get_coadd("s15", region, array, srcfree=True, ncomp=1)[0] * mask) # Get pixel window correction pwin = tutils.get_pixwin(mask.shape[-2:]) bpfile = "data/" + dm.get_bandpass_file_name(array) # Apply a filter that converts array map to Compton Y units (Eq 8) f = tfg.get_mix_bandpassed([bpfile], 'tSZ', ccor_cen_nus=[freqs[array]], ccor_beams=[lbeam])[0] f2d = maps.interp(ells, f)(modlmap) filt = kmask / pwin / f2d filt[~np.isfinite(filt)] = 0 imap = enmap.ifft(kmap * filt).real # Reconvolve Y map ymap = maps.filter_map(enmap.read_map(yfile), beam_ratio * kmask) # Get CMB maps smap = enmap.read_map(yfile2) cmap = enmap.read_map(yfile3) # Width in pixels of stamp cutout pix = 60 # Initialize stacks i = 0 istack = 0 ystack = 0