def gradient_flat(map_in, lmax=2000):
"""Return the gradient maps of map_in.
Compute the gradient of the map_in in Fourier space. Simultaneously
low-pass the maps such that they only include modes ell < lmax.
Parameters
----------
map_in: ndmap, ndarray
The input map.
lmax: int
The cutoff ell for low-passing the maps.
Returns
-------
dy: ndmap, ndarray
The gradient in the y-direction
dx: ndmap, ndarray
The gradient in the x-direction
"""
ly, lx = map_in.lmap()
lmod = map_in.modlmap()
lp = enmap.zeros(map_in.shape)
lp[np.where(lmod < lmax)] = 1.
map_fft = enmap.fft(map_in)
dx = enmap.ifft(map_fft*lp*lx*1j).real
dy = enmap.ifft(map_fft*lp*ly*1j).real
return dy, dx
def test_fft(self):
# Tests that ifft(ifft(imap))==imap, i.e. default normalizations are consistent
shape, wcs = enmap.geometry(pos=(0, 0), shape=(3, 100, 100), res=0.01)
imap = enmap.enmap(np.random.random(shape), wcs)
assert np.all(
np.isclose(
imap,
enmap.ifft(enmap.fft(imap, normalize='phy'),
normalize='phy').real))
assert np.all(np.isclose(imap, enmap.ifft(enmap.fft(imap)).real))
def gather_patches_cos(self):
"""Assemble patch statistics for small scale lensing with cos filter.
Compute the small scale (ell > 3000) temperature power at different
patches across the sky as well as the average amplitude of the
background temperature gradient (ell < 2000). For the small scale
statistics, also apply a filter in Fourier space such that:
.. math::
f_\\ell = \\cos(\\hat{\\ell}\\cdot\\hat{\\nabla T})
"""
self._edge = 5 # Edge pixels to throw away
p = self._p
m_fft = enmap.fft(self.map_in)
hp = np.zeros(self.map_in.shape)
hp[np.where((self._lmod > self.lmin) & (self._lmod < self.lmax))] = 1.
# Apply pre-whitening or Wiener/inverse variance filters, then top hat
if self._aW and self.fid is not None:
cs = CubicSpline(self.fid[0], self.fid[1]) # (ell, Cl)
m_fft = m_fft / cs(self._lmod)
self._Tss = enmap.ifft(m_fft * hp)
self._dTy, self._dTx = gradient_flat(self.map_in, self.ldT)
# Scale geometry for lower res map of patches
pshp, pwcs = enmap.scale_geometry(self.map_in.shape, self.map_in.wcs,
1. / self._p)
if not self.savesteps:
del self.map_in, m_fft
self._T2patch = enmap.zeros(pshp, pwcs)
self._dTxpatch = enmap.zeros(pshp, pwcs)
self._dTypatch = enmap.zeros(pshp, pwcs)
self._T_sub = np.zeros((pshp[-2], pshp[-1], p, p))
for i in range(self._T2patch.shape[-2]):
for j in range(self._T2patch.shape[-1]):
self._dTypatch[i, j] = np.mean(self._dTy[i * p:(i + 1) * p,
j * p:(j + 1) * p])
self._dTxpatch[i, j] = np.mean(self._dTx[i * p:(i + 1) * p,
j * p:(j + 1) * p])
Tp = self._Tss[i * p:(i + 1) * p, j * p:(j + 1) * p]
lsuby, lsubx = Tp.lmap()
lsubmod = Tp.modlmap()
lsubmod[0, 0] = 1.
fl = 1.j*(lsubx*self._dTxpatch[i,j] + \
lsuby*self._dTypatch[i,j]) / \
(lsubmod * np.sqrt(self._dTxpatch[i,j]**2 + \
self._dTypatch[i,j]**2))
fl[0, 0] = 0.
self._T_sub[i, j, :, :] = enmap.ifft(enmap.fft(Tp) * fl).real
# Throw away pixels with edge effects
self._T2patch[i, j] = np.var(self._T_sub[i, j, 5:-5, 5:-5])
self._dT2patch = self._dTxpatch**2 + self._dTypatch**2
if not self.savesteps:
del self._dTypatch, self._dTxpatch, self._dTy, self._dTx, self._Tss
del self._T_sub
def gather_patches_cos2(self):
"""Assemble patch statistics for small scale lensing with cos^2 filter.
Compute the small scale (ell > 3000) temperature power at different
patches across the sky as well as the average amplitude of the
background temperature gradient (ell < 2000). For the small scale
statistics, also apply a filter in Fourier space such that:
.. math::
f_\\ell = \\cos^2(\\hat{\\ell}\\cdot\\hat{\\nabla T})
"""
self._edge = 3 # Edge pixels to throw away
p = self._p
m_fft = enmap.fft(self.map_in)
hp = np.zeros(self.map_in.shape)
hp[np.where((self._lmod > self.lmin) & (self._lmod < self.lmax))] = 1.
self._Tss = enmap.ifft(m_fft * hp)
self._dTy, self._dTx = gradient_flat(self.map_in, self.ldT)
# Scale geometry for lower res map of patches
pshp, pwcs = enmap.scale_geometry(self.map_in.shape, self.map_in.wcs,
1. / self._p)
if not self.savesteps:
del self.map_in, m_fft
self._T2patch = enmap.zeros(pshp, pwcs)
self._dTxpatch = enmap.zeros(pshp, pwcs)
self._dTypatch = enmap.zeros(pshp, pwcs)
self._T_sub = np.zeros((pshp[-2], pshp[-1], p, p))
for i in range(self._T2patch.shape[-2]):
for j in range(self._T2patch.shape[-1]):
self._dTypatch[i, j] = np.mean(self._dTy[i * p:(i + 1) * p,
j * p:(j + 1) * p])
self._dTxpatch[i, j] = np.mean(self._dTx[i * p:(i + 1) * p,
j * p:(j + 1) * p])
Tp = self._Tss[i * p:(i + 1) * p, j * p:(j + 1) * p]
lsuby, lsubx = Tp.lmap()
lsubmod = Tp.modlmap()
lsubmod[0, 0] = 1. # Avoid divide by 0; set fl here to 0 later
fl = (lsubx*self._dTxpatch[i,j] + \
lsuby*self._dTypatch[i,j])**2 / \
(lsubmod * np.sqrt(self._dTxpatch[i,j]**2 + \
self._dTypatch[i,j]**2))**2
fl[0, 0] = 0.
self._T_sub[i, j, :, :] = enmap.ifft(enmap.fft(Tp) * fl).real
# Throw away pixels with edge effects
self._T2patch[i, j] = np.var(self._T_sub[i, j, 3:-3, 3:-3])
self._dT2patch = self._dTxpatch**2 + self._dTypatch**2
if not self.savesteps:
del self._dTypatch, self._dTxpatch, self._dTy, self._dTx, self._Tss
del self._T_sub
def filter_weighted(imap, ivar, filter, tol=1e-4, ref=0.9):
"""Filter enmap imap with the given 2d fourier filter while
weithing spatially with ivar"""
filter = 1 - filter
omap = enmap.ifft(filter * enmap.fft(imap * ivar)).real
div = enmap.ifft(filter * enmap.fft(ivar)).real
del filter
# Avoid division by very low values
div = np.maximum(div, np.percentile(ivar[::10, ::10], ref * 100) * tol)
# omap = imap - rhs/div
omap /= div
del div
omap *= -1
omap += imap
omap *= imap != 0
return omap
def gather_patches_plain(self):
"""Assemble patch statistics relevant to lensing at small scales.
Compute the small scale (ell > lmin) temperature power at different
patches across the sky as well as the average amplitude of the
background temperature gradient (ell < ldT).
"""
self._edge = 0 # Not throwing away edge pixels
m_fft = enmap.fft(self.map_in)
hp = np.zeros(self.map_in.shape)
hp[np.where((self._lmod > self.lmin) & (self._lmod < self.lmax))] = 1.
self._Tss = enmap.ifft(m_fft * hp)
self._dTy, self._dTx = gradient_flat(self.map_in, self.ldT)
# Scale geometry for lower res map of patches
pshp, pwcs = enmap.scale_geometry(self.map_in.shape, self.map_in.wcs,
1. / self._p)
if not self.savesteps:
del self.map_in, m_fft
self._T2patch = enmap.zeros(pshp, pwcs)
self._dTxpatch = enmap.zeros(pshp, pwcs)
self._dTypatch = enmap.zeros(pshp, pwcs)
Trs = self._Tss[:pshp[-2] * self._p, :pshp[-1] * self._p].reshape(
[pshp[-2], self._p, pshp[-1], self._p])
dTxrs = self._dTx[:pshp[-2] * self._p, :pshp[-1] * self._p].reshape(
[pshp[-2], self._p, pshp[-1], self._p])
dTyrs = self._dTy[:pshp[-2] * self._p, :pshp[-1] * self._p].reshape(
[pshp[-2], self._p, pshp[-1], self._p])
self._T2patch[:, :] = np.var(Trs, axis=(1, 3))
self._dTypatch[:, :] = np.mean(dTyrs, axis=(1, 3))
self._dTxpatch[:, :] = np.mean(dTxrs, axis=(1, 3))
self._dT2patch = self._dTxpatch**2 + self._dTypatch**2
if not self.savesteps:
del self._dTypatch, self._dTxpatch, self._dTy, self._dTx, self._Tss
def cutout_rec(shape, wcs, feed_dict, cmask, kmask, map1_k, map2_k):
""" cutout lensing reconstruction """
feed_dict['X'] = map1_k
feed_dict['Y'] = map2_k
# unnormalized lensing map in fourier space
ukappa_k = s.unnormalized_quadratic_estimator(shape,
wcs,
feed_dict,
"hu_ok",
"TT",
xmask=cmask,
ymask=cmask)
# normaliztion
norm_k = s.A_l(shape,
wcs,
feed_dict,
"hu_ok",
"TT",
xmask=cmask,
ymask=cmask,
kmask=kmask)
# noise
noise_2d = s.N_l_from_A_l_optimal(shape, wcs, norm_k)
# normalized Fourier space CMB lensing convergence map
kappa_k = norm_k * ukappa_k
# real space CMB lensing convergence map
kappa = enmap.ifft(kappa_k, normalize='phys')
return kappa, noise_2d
def cmbmap2alm(i, mtype, p, f, r):
fmap = enmap.read_map(
f.imap[mtype][i]) # load flatsky K-space combined map
# FT
print('2D ell filtering in F-space and assign to healpix map')
alm = enmap.fft(fmap)
# remove some Fourier modes
shape, wcs = fmap.shape, fmap.wcs
kmask = mask_kspace(shape, wcs, lmin=p.lcut, lmax=4000, lxcut=90, lycut=50)
alm[kmask < 0.5] = 0
# alm -> map
fmap = enmap.ifft(alm).real
# transform cmb map to healpix
hpmap = enmap.to_healpix(fmap, nside=p.nside)
# from map to alm
hpmap = r.w * hpmap # masking
#hp.fitsfunc.write_map(f.omap[mtype][i],hpmap,overwrite=True)
alm = curvedsky.utils.hp_map2alm(p.nside, p.lmax, p.lmax, hpmap)
print("save to file")
pickle.dump((alm),
open(f.alm[mtype][i], "wb"),
protocol=pickle.HIGHEST_PROTOCOL)
def generate_noise_sim(covsqrt, ivars, seed=None, dtype=None):
"""
Supports only two cases
1) nfreqs>=1,npol=3
2) nfreqs=1,npol=1
"""
if isinstance(seed, int): seed = (seed, )
assert np.all(np.isfinite(covsqrt))
shape, wcs = covsqrt.shape, covsqrt.wcs
Ny, Nx = shape[-2:]
ncomps = covsqrt.shape[0]
assert ncomps == covsqrt.shape[1]
assert ((ncomps % 3) == 0) or (ncomps == 1)
nfreqs = 1 if ncomps == 1 else ncomps // 3
if ncomps == 1: npol = 1
else: npol = 3
wmaps = enmap.extract(ivars, shape[-2:], wcs)
nsplits = wmaps.shape[1]
if dtype is np.float32: ctype = np.complex64
elif dtype is np.float64: ctype = np.complex128
# Old way with loop
kmap = []
for i in range(nsplits):
if seed is None:
np.random.seed(None)
else:
np.random.seed(seed + (i, ))
rmap = enmap.rand_gauss_harm((ncomps, Ny, Nx),
covsqrt.wcs).astype(ctype)
kmap.append(enmap.map_mul(covsqrt, rmap))
del covsqrt, rmap
kmap = enmap.enmap(np.stack(kmap), wcs)
outmaps = enmap.ifft(kmap, normalize="phys").real
del kmap
# Need to test this more ; it's only marginally faster and has different seed behaviour
# covsqrt = icovsqrt
# np.random.seed(seed)
# rmap = enmap.rand_gauss_harm((nsplits,ncomps,Ny, Nx),covsqrt.wcs)
# kmap = enmap.samewcs(np.einsum("abyx,cbyx->cayx", covsqrt, rmap),rmap)
# outmaps = enmap.ifft(kmap, normalize="phys").real
# isivars = 1/np.sqrt(wmaps)
with np.errstate(divide='ignore', invalid='ignore'):
isivars = ((1. / wmaps) - (1. / wmaps.sum(axis=1)[:, None, ...]))**0.5
isivars[~np.isfinite(isivars)] = 0
assert np.all(np.isfinite(outmaps))
# Divide by hits
for ifreq in range(nfreqs):
outmaps[:, ifreq * npol:(ifreq + 1) * npol,
...] = outmaps[:, ifreq * npol:(ifreq + 1) * npol,
...] * isivars[ifreq, ...] * np.sqrt(nsplits)
retmaps = outmaps.reshape((nsplits, nfreqs, npol, Ny, Nx)).swapaxes(0, 1)
assert np.all(np.isfinite(retmaps))
return retmaps, wmaps
def sim_noise_oneoverf(ivar, apod, lknee=3000, lmin=50, alpha=-3.5):
l = np.maximum(ivar.modlmap(), lmin)
fscale = (1 + (l / lknee)**alpha)**0.5
ref = np.mean(ivar[ivar != 0])
sim = enmap.rand_gauss(ivar.shape, ivar.wcs, dtype=ivar.dtype)
sim = enmap.ifft(enmap.fft(sim) * fscale).real * np.maximum(
ivar, ref * 1e-3)**-0.5 * apod**2
return sim
def filter_riseset(imap, ivar, riseset, filter, tol=1e-4, ref=0.9):
"""Filter enmap imap with the given 2d fourier filter while
weighting spatially with ivar and allowing for rise vs set-dependent
pickup. riseset should be a map with values between -1 and 1 that determines
the balance between rising and setting scans, with 0 meaning equal weight from both.
Overall this is only a marginal improvement over filter_weighted depsite being
quite a bit heaver, and it only helps a bit for the pa7 residual stripe issue it
was invented to deal with, so it probably isn't worth using.
"""
# We model the map as m = Pa+n, where a is [2] is the rising and setting pickup in
# a single pixel, and P is the map's response to this pickup, which is
# P = [x,1-x]*filter, where x=(1-riseset)/2. Given this we can solve for a as:
# a = (P'N"P)"P'N"m, where N" is ivar. Written out this becomes, for a single pixel.
# rhs = ifft(filter*fft([x,1-x]*ivar*imap))
# div = ifft(filter*fft([x,1-x]*ivar*[x,1-x]*ifft(filter)
# Hm.... The problem here is that we're assuming that there's nothing in the other pixels.
# That's not what we did in filter_weighted. Let's just do something simple for now.
x = (1 - riseset) / 2
x1x = np.array([x, 1 - x])
del x
# Broadcast to imap shape
x1x = x1x[(slice(None), ) + (None, ) * (imap.ndim - 2)]
print(imap.shape, x1x.shape)
filter = 1 - filter
rhs = enmap.ifft(filter * enmap.fft(x1x * ivar * imap)).real
div = enmap.ifft(filter *
enmap.fft(x1x[:, None] * x1x[None, :] * ivar)).real
del filter
# Avoid division by very low values
ref = np.percentile(ivar[::10, ::10], ref * 100) * tol
for i in range(2):
div[i, i] = np.maximum(div[i, i], ref)
# Solve the system
rhs = np.moveaxis(rhs, 0, -1)
div = np.moveaxis(div, (0, 1), (-2, -1))
omap = np.linalg.solve(div, rhs)
del rhs, div
omap = np.sum(x1x * np.moveaxis(omap, -1, 0), 0)
del x1x
omap = enmap.ndmap(omap, imap.wcs)
omap *= -1
omap += imap
omap *= imap != 0
return omap
def beam_match(imap, f1, f2):
"""f1, f2 are fcodes instead of the actual frequency centers. It
assumes that the first one (f1) has larger beam, so f2 will be matched
to it.
"""
from common import fwhms
l = imap.modlmap()
bmap_f1 = np.exp(-0.5 * l**2 * (fwhms[f1] * u.fwhm * u.arcmin)**2)
bmap_f2 = np.exp(-0.5 * l**2 * (fwhms[f2] * u.fwhm * u.arcmin)**2)
rmap = enmap.ifft(enmap.fft(imap) *
(bmap_f1 / np.maximum(bmap_f2, 1e-3))).real
return rmap
def smooth_ps_gauss(ps2d, lsigma=200):
"""Smooth a 2d power spectrum to the target resolution in l. Simple
gaussian smoothing avoids ringing."""
# This hasn't been tested in isolation, but breaks when used in smooth_ps_mixed
# First get our pixel size in l
ly, lx = enmap.laxes(ps2d.shape, ps2d.wcs)
ires = np.array([ly[1], lx[1]])
sigma_pix = np.abs(lsigma / ires)
fmap = enmap.fft(ps2d)
ky = np.fft.fftfreq(ps2d.shape[-2]) * sigma_pix[0]
kx = np.fft.fftfreq(ps2d.shape[-1]) * sigma_pix[1]
kr2 = ky[:, None]**2 + kx[None, :]**2
fmap *= np.exp(-0.5 * kr2)
return enmap.ifft(fmap).real
def remove_lxly(fmap, lmin=100, lmax=4096, lxcut=90, lycut=50):
alm = enmap.fft(fmap)
shape, wcs = fmap.shape, fmap.wcs
kmask = define_lmask(shape,
wcs,
lmin=lmin,
lmax=lmax,
lxcut=lxcut,
lycut=lycut)
#print(np.shape(kmask),np.shape(alm))
alm[kmask < 0.5] = 0
fmap_fl = enmap.ifft(alm).real
return fmap_fl
def filter_fast(map, vk_mask=None, hk_mask=None, normalize="phys", inv_pixwin_lxly=None):
"""Filter the map in Fourier space removing modes in a horizontal and vertical band
defined by hk_mask and vk_mask. This is a faster version that what is implemented in pspy
We also include an option for removing the pixel window function
Parameters
---------
map: ``so_map``
the map to be filtered
vk_mask: list with 2 elements
format is fourier modes [-lx,+lx]
hk_mask: list with 2 elements
format is fourier modes [-ly,+ly]
normalize: string
optional normalisation of the Fourier transform
inv_pixwin_lxly: 2d array
the inverse of the pixel window function in fourier space
"""
lymap, lxmap = map.data.lmap()
ly, lx = lymap[:,0], lxmap[0,:]
# filtered_map = map.copy()
ft = enmap.fft(map.data, normalize=normalize)
if vk_mask is not None:
id_vk = np.where((lx > vk_mask[0]) & (lx < vk_mask[1]))
if hk_mask is not None:
id_hk = np.where((ly > hk_mask[0]) & (ly < hk_mask[1]))
if map.ncomp == 1:
if vk_mask is not None:
ft[: , id_vk] = 0.
if hk_mask is not None:
ft[id_hk , :] = 0.
if map.ncomp == 3:
for i in range(3):
if vk_mask is not None:
ft[i, : , id_vk] = 0.
if hk_mask is not None:
ft[i, id_hk , :] = 0.
if inv_pixwin_lxly is not None:
ft *= inv_pixwin_lxly
map.data[:] = np.real(enmap.ifft(ft, normalize=normalize))
return map
def process_map(imap, box=None, deconvolve=False, freq=None, smooth=None):
if deconvolve:
fwhm = fwhms[freq]
l = imap.modlmap()
bmap = np.exp(-0.5*l**2*(fwhm*u.fwhm*u.arcmin)**2)
bmap = np.maximum(bmap, 1e-3)
fmap = enmap.fft(imap)
omap = enmap.ifft(fmap/bmap).real
else:
omap = imap
if smooth is not None:
omap = enmap.smooth_gauss(omap, smooth*u.fwhm*u.arcmin)
if box is not None: omap = omap.submap(box)
if args.comp == 0: return np.log10(omap[0])
if args.comp == 12: return np.sum(omap[1:]**2, axis=0)**0.5
else: return omap[args.comp]
def get_filtered_map(map, binary, vk_mask, hk_mask, normalize=False):
"""Filter the map in Fourier space removing modes in a horizontal and vertical band
defined by hk_mask and vk_mask. Note that we mutliply the maps by a binary mask before
doing this operation in order to remove pathological pixels
Parameters
---------
orig_map: ``so_map``
the map to be filtered
binary: ``so_map``
a binary mask removing pathological pixels
vk_mask: list with 2 elements
format is fourier modes [-lx,+lx]
hk_mask: list with 2 elements
format is fourier modes [-ly,+ly]
"""
if map.ncomp == 1:
map.data *= binary.data
else:
map.data[:] *= binary.data
lymap, lxmap = map.data.lmap()
ly, lx = lymap[:, 0], lxmap[0, :]
# filtered_map = map.copy()
ft = enmap.fft(map.data, normalize=normalize)
if vk_mask is not None:
id_vk = np.where((lx > vk_mask[0]) & (lx < vk_mask[1]))
if hk_mask is not None:
id_hk = np.where((ly > hk_mask[0]) & (ly < hk_mask[1]))
if map.ncomp == 1:
if vk_mask is not None:
ft[:, id_vk] = 0.0
if hk_mask is not None:
ft[id_hk, :] = 0.0
if map.ncomp == 3:
if vk_mask is not None:
ft[:, :, id_vk] = 0.0
if hk_mask is not None:
ft[:, id_hk, :] = 0.0
map.data[:] = np.real(enmap.ifft(ft, normalize=normalize))
return map
def smooth_ps_grid(ps, res, alpha=4, log=False, ndof=2):
"""Smooth a 2d power spectrum to the target resolution in l"""
# First get our pixel size in l
lx, ly = enmap.laxes(ps.shape, ps.wcs)
ires = np.array([lx[1],ly[1]])
smooth = np.abs(res/ires)
# We now know how many pixels to somoth by in each direction,
# so perform the actual smoothing
if log: ps = np.log(ps)
fmap = enmap.fft(ps)
ky = np.fft.fftfreq(ps.shape[-2])
kx = np.fft.fftfreq(ps.shape[-1])
fmap /= 1 + np.abs(2*ky[:,None]*smooth[0])**alpha
fmap /= 1 + np.abs(2*kx[None,:]*smooth[1])**alpha
ps = enmap.ifft(fmap).real
if log: ps = np.exp(ps)*log_smooth_corrections[ndof]
return ps
def map_act_hist(patches, numerics, wf, masks, apo_masks, \
negbins, posbins) :#{{{
# @TODO: check final pixel count!
# @TODO check changing cmb or noise power changes variance as expected. it will.
# @TODO plot histogram with and without cmb
# @TODO plot histogram with and without WF
# maps are sub-divided into six patches; analyze each separately and then get the PDF of the whole map
dat_tot = np.array([])
for i in range(numerics['n_patch']):
patch = patches[i]
mask = copy.deepcopy(apo_masks[i])
patch *= mask #apply mask
modlmap = patch.modlmap(
) #map of |\vec{ell}| in 2D Fourier domain for the patch
Fell2d = interp1d(wf[:, 0],
wf[:, 1],
bounds_error=False,
fill_value=0.)(
modlmap) #get filter in 2D Fourier domain
kmap = enmap.fft(patch,
normalize="phys") #FFT the patch to 2D Fourier domain
kfilt = kmap * Fell2d
patch_filt = enmap.ifft(kfilt, normalize="phys").real
# point source mask (different for each patch)
newMask = copy.deepcopy(masks[i])
newMask[363, :] = 1.
newMask[:, 2181] = 1.
la = np.where(newMask < 0.1)
mask[la] = 0.
### pick out unmasked data
loc2 = np.where(mask > 0.9)
dat = patch_filt[loc2] #*fraction
dat_tot = np.append(dat_tot, np.ravel(dat))
# histogram the unmasked data
histneg, bin_edgesneg = np.histogram(dat_tot, negbins)
histpos, bin_edgespos = np.histogram(dat_tot, posbins[::-1])
histall = np.concatenate((histneg, histpos))
PDFpatch = histall / float(len(dat_tot))
return PDFpatch
def kspace_filter_fast(map, vk_mask=None, hk_mask=None, normalize="phys"):
"""Filter the map in Fourier space removing modes in a horizontal and vertical band
defined by hk_mask and vk_mask. This is a faster version that what is implemented in pspy
Parameters
---------
map: ``so_map``
the map to be filtered
vk_mask: list with 2 elements
format is fourier modes [-lx,+lx]
hk_mask: list with 2 elements
format is fourier modes [-ly,+ly]
"""
lymap, lxmap = map.data.lmap()
ly, lx = lymap[:,0], lxmap[0,:]
# filtered_map = map.copy()
ft = enmap.fft(map.data, normalize=normalize)
if vk_mask is not None:
id_vk = np.where((lx > vk_mask[0]) & (lx < vk_mask[1]))
if hk_mask is not None:
id_hk = np.where((ly > hk_mask[0]) & (ly < hk_mask[1]))
if map.ncomp == 1:
if vk_mask is not None:
ft[: , id_vk] = 0.
if hk_mask is not None:
ft[id_hk , :] = 0.
if map.ncomp == 3:
for i in range(3):
if vk_mask is not None:
ft[i, : , id_vk] = 0.
if hk_mask is not None:
ft[i, id_hk , :] = 0.
map.data[:] = np.real(enmap.ifft(ft, normalize=normalize))
return map
def deconvolve_pixwin_CAR(orig_map, binary, inv_pixwin_lxly, norm=0):
"""Deconvolve the two dimensional CAR pixel window function
Parameters
---------
orig_map: ``so_map``
the map to be filtered
binary: ``so_map``
a binary mask removing pathological pixels
inv_pixwin_lxly: 2d array
the inverse of the pixel window function in fourier space
norm: integer
everytime we do a FFT + IFFT operation we get a normalisation cst to take care of
this is counting the number of occurence of it, note that we could correct directly by
using normalize="physical", but dividing the map by a number is slow so we prefer to do it later
in the code.
"""
orig_map.data *= binary.data
ft = enmap.fft(orig_map.data, normalize=False)
ft *= inv_pixwin_lxly
orig_map.data = enmap.ifft(ft, normalize=False).real
norm += 1
return norm, orig_map
def compute_map(self,oshape,owcs,qid,pixwin_taper_deg=0.3,pixwin_pad_deg=0.3,
include_cmb=True,include_tsz=True,include_fgres=True,sht_beam=True):
"""
1. get total alm
2. apply beam, and pixel window if Planck
3. ISHT
4. if ACT, apply a small taper and apply pixel window in Fourier space
"""
# pad to a slightly larger geometry
tot_pad_deg = pixwin_taper_deg + pixwin_pad_deg
res = maps.resolution(oshape,owcs)
pix = np.deg2rad(tot_pad_deg)/res
omap = enmap.pad(enmap.zeros((3,)+oshape,owcs),pix)
ishape,iwcs = omap.shape[-2:],omap.wcs
# get data model
dm = sints.models[sints.arrays(qid,'data_model')](region_shape=ishape,region_wcs=iwcs,calibrated=True)
# 1. get total alm
array_index = self.qids.index(qid)
tot_alm = int(include_cmb)*self.alms['cmb']
if include_tsz:
try:
assert self.tsz_fnu.ndim==2
tot_alm[0] = tot_alm[0] + hp.almxfl(self.alms['comptony'][0] ,self.tsz_fnu[array_index])
except:
tot_alm[0] = tot_alm[0] + self.alms['comptony'][0] * self.tsz_fnu[array_index]
if self.cfgres is not None: tot_alm[0] = tot_alm[0] + int(include_fgres)*self.alms['fgres'][array_index]
assert tot_alm.ndim==2
assert tot_alm.shape[0]==3
ells = np.arange(self.lmax+1)
# 2. get beam, and pixel window for Planck
if sht_beam:
beam = tutils.get_kbeam(qid,ells,sanitize=False,planck_pixwin=False) # NEVER SANITIZE THE BEAM IN A SIMULATION!!!
for i in range(3): tot_alm[i] = hp.almxfl(tot_alm[i],beam)
if dm.name=='planck_hybrid':
pixwint,pixwinp = hp.pixwin(nside=tutils.get_nside(qid),lmax=self.lmax,pol=True)
tot_alm[0] = hp.almxfl(tot_alm[0],pixwint)
tot_alm[1] = hp.almxfl(tot_alm[1],pixwinp)
tot_alm[2] = hp.almxfl(tot_alm[2],pixwinp)
# 3. ISHT
omap = curvedsky.alm2map(np.complex128(tot_alm),omap,spin=[0,2])
assert omap.ndim==3
assert omap.shape[0]==3
if not(sht_beam):
taper,_ = maps.get_taper_deg(ishape,iwcs,taper_width_degrees=pixwin_taper_deg,pad_width_degrees=pixwin_pad_deg)
modlmap = omap.modlmap()
beam = tutils.get_kbeam(qid,modlmap,sanitize=False,planck_pixwin=True)
kmap = enmap.fft(omap*taper,normalize='phys')
kmap = kmap * beam
# 4. if ACT, apply a small taper and apply pixel window in Fourier space
if dm.name=='act_mr3':
if sht_beam: taper,_ = maps.get_taper_deg(ishape,iwcs,taper_width_degrees=pixwin_taper_deg,pad_width_degrees=pixwin_pad_deg)
pwin = tutils.get_pixwin(ishape[-2:])
if sht_beam:
omap = maps.filter_map(omap*taper,pwin)
else:
kmap = kmap * pwin
if not(sht_beam): omap = enmap.ifft(kmap,normalize='phys').real
return enmap.extract(omap,(3,)+oshape[-2:],owcs)
# wmap = enmap.ifft(enmap.fft(wmap)*matched_filter).real**2
# lim = max(np.median(wmap)*tol1**2, np.max(wmap)*tol2**2)
# mask |= wmap > lim
# return mask
comm = mpi.COMM_WORLD
beam1d = get_beam(args.beam)
ifiles = sorted(sum([glob.glob(ifile) for ifile in args.ifiles], []))
for ind in range(comm.rank, len(ifiles), comm.size):
ifile = ifiles[ind]
if args.verbose: print(ifile)
ofile = args.odir + "/" + ifile
imap = enmap.read_map(ifile)
if args.mask is not None: mask = imap == args.mask
if args.apodize:
imap = imap.apod(args.apodize)
# We will apply a semi-matched-filter to T
l = np.maximum(1, imap.modlmap())
beam2d = enmap.samewcs(np.interp(l, np.arange(len(beam1d)), beam1d), imap)
matched_filter = (1 + (l / args.lknee)**args.alpha)**-1 * beam2d
fmap = enmap.map2harm(imap, iau=True)
fmap[0] *= matched_filter
omap = enmap.ifft(fmap).real
if args.mask is not None:
omap[mask] = 0
del mask
utils.mkdir(os.path.dirname(ofile))
enmap.write_map(ofile, omap)
del omap
kmask = maps.mask_kspace(shape,wcs,lmin=Lmin,lmax=Lmax)
feed_dict['uC_T_T'] = cov
feed_dict['tC_T_T'] = cov + (wnoise*np.pi/180/60)**2 / kbeam**2
qe = symlens.QE(shape,wcs,feed_dict,'hu_ok','TT',xmask=xmask,ymask=ymask,kmask=kmask)
s = stats.Stats()
for task in my_tasks:
cmb = maps.filter_map(mgen.get_map(seed=(1,task))[0],kbeam)
nseed = (2,task)
nmap = maps.white_noise(shape,wcs,noise_muK_arcmin=None,seed=nseed,ipsizemap=pmap,div=ivar)
obs = cmb + nmap
kobs = enmap.fft(obs,normalize='phys')/kbeam
kobs[~np.isfinite(kobs)] = 0
feed_dict['X'] = kobs
feed_dict['Y'] = kobs
krecon = qe.reconstruct(feed_dict)
print(cmb.shape,nmap.shape,krecon.shape)
s.add_to_stack('kreal',krecon.real)
s.add_to_stack('kimag',krecon.imag)
s.get_stacks()
mf = enmap.ifft(s.stacks['kreal'] + 1j*s.stacks['kimag'],normalize='phys')
io.hplot(mf,'mf.png')
pathMap = "/global/cscratch1/sd/eschaan/project_ksz_act_planck/data/planck_act_coadd_2020_02_28_r2/" + "act_planck_s08_s18_cmb_f150_daynight_map.fits"
# read maps
iMap = enmap.read_map(pathMap)
# do the reconvolution in Fourier space
mapF = enmap.fft(iMap)
lMap = np.sqrt(np.sum(iMap.lmap()**2,0))
##########################################################################
# Reconvolve the map
print("Reconvolve 150 to 90")
pathOutMap = "/global/cscratch1/sd/eschaan/project_ksz_act_planck/data/planck_act_coadd_2020_02_28_r2/" + "act_planck_s08_s18_cmb_f150_daynight_map_reconvto90.fits"
oMap = enmap.ifft(mapF * fBeam90F(lMap) / fBeam150F(lMap)).real
enmap.write_map(pathOutMap, oMap)
print("check finite sum "+str(np.sum(oMap)))
print("Reconvolve 150 to TileC deproj")
pathOutMap = "/global/cscratch1/sd/eschaan/project_ksz_act_planck/data/planck_act_coadd_2020_02_28_r2/" + "act_planck_s08_s18_cmb_f150_daynight_map_reconvtotilecdeproj.fits"
oMap = enmap.ifft(mapF * fBeamTilecDeprojF(lMap) / fBeam150F(lMap)).real
enmap.write_map(pathOutMap, oMap)
print("check finite sum "+str(np.sum(oMap)))
print("Reconvolve 150 to TileC")
pathOutMap = "/global/cscratch1/sd/eschaan/project_ksz_act_planck/data/planck_act_coadd_2020_02_28_r2/" + "act_planck_s08_s18_cmb_f150_daynight_map_reconvtotilec.fits"
oMap = enmap.ifft(mapF * fBeamTilecF(lMap) / fBeam150F(lMap)).real
enmap.write_map(pathOutMap, oMap)
def build_and_save_ilc(arrays,region,version,cov_version,beam_version,
solutions,beams,chunk_size,
effective_freq,overwrite,maxval,unsanitized_beam=False,do_weights=False,
pa1_shift = None,
pa2_shift = None,
pa3_150_shift = None,
pa3_090_shift = None,
no_act_color_correction=False, ccor_exp = -1,
isotropize=False, isotropize_width=20):
print("Chunk size is ", chunk_size*64./8./1024./1024./1024., " GB.")
def warn(): print("WARNING: no bandpass file found. Assuming array ",dm.c['id']," has no response to CMB, tSZ and CIB.")
aspecs = tutils.ASpecs().get_specs
bandpasses = not(effective_freq)
savedir = tutils.get_save_path(version,region)
covdir = tutils.get_save_path(cov_version,region)
assert os.path.exists(covdir)
if not(overwrite):
assert not(os.path.exists(savedir)), \
"This version already exists on disk. Please use a different version identifier."
try: os.makedirs(savedir)
except:
if overwrite: pass
else: raise
mask = enmap.read_map(covdir+"tilec_mask.fits")
shape,wcs = mask.shape,mask.wcs
Ny,Nx = shape
modlmap = enmap.modlmap(shape,wcs)
arrays = arrays.split(',')
narrays = len(arrays)
kcoadds = []
kbeams = []
bps = []
names = []
lmins = []
lmaxs = []
shifts = []
cfreqs = []
lbeams = []
ells = np.arange(0,modlmap.max())
for i,qid in enumerate(arrays):
dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=True)
lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid)
cfreqs.append(cfreq)
lmins.append(lmin)
lmaxs.append(lmax)
names.append(qid)
if dm.name=='act_mr3':
season,array1,array2 = sints.arrays(qid,'season'),sints.arrays(qid,'array'),sints.arrays(qid,'freq')
array = '_'.join([array1,array2])
elif dm.name=='planck_hybrid':
season,patch,array = None,None,sints.arrays(qid,'freq')
else:
raise ValueError
kcoadd_name = covdir + "kcoadd_%s.npy" % qid
kmask = maps.mask_kspace(shape,wcs,lmin=lmin,lmax=lmax)
kcoadd = enmap.enmap(np.load(kcoadd_name),wcs)
dtype = kcoadd.dtype
kcoadds.append(kcoadd.copy()*kmask)
kbeam = tutils.get_kbeam(qid,modlmap,sanitize=not(unsanitized_beam),version=beam_version,planck_pixwin=True)
if dm.name=='act_mr3':
lbeam = tutils.get_kbeam(qid,ells,sanitize=not(unsanitized_beam),version=beam_version,planck_pixwin=False) # note no pixwin but doesnt matter since no ccorr for planck
elif dm.name=='planck_hybrid':
lbeam = None
else:
raise ValueError
lbeams.append(lbeam)
kbeams.append(kbeam.copy())
if bandpasses:
try:
fname = dm.get_bandpass_file_name(array)
bps.append("data/"+fname)
if (pa1_shift is not None) and 'PA1' in fname:
shifts.append(pa1_shift)
elif (pa2_shift is not None) and 'PA2' in fname:
shifts.append(pa2_shift)
elif (pa3_150_shift is not None) and ('PA3' in fname) and ('150' in fname):
shifts.append(pa3_150_shift)
elif (pa3_090_shift is not None) and ('PA3' in fname) and ('090' in fname):
shifts.append(pa3_90_shift)
else:
shifts.append(0)
except:
warn()
bps.append(None)
else:
try: bps.append(cfreq)
except:
warn()
bps.append(None)
kcoadds = enmap.enmap(np.stack(kcoadds),wcs)
# Read Covmat
cov = maps.SymMat(narrays,shape[-2:])
for aindex1 in range(narrays):
for aindex2 in range(aindex1,narrays):
icov = enmap.enmap(np.load(covdir+"tilec_hybrid_covariance_%s_%s.npy" % (names[aindex1],names[aindex2])),wcs)
if isotropize:
bin_edges = np.append([0.],np.arange(min(lmins),modlmap.max(),isotropize_width))
binner = stats.bin2D(modlmap,bin_edges)
ls,c1d = binner.bin(icov)
icov = maps.interp(ls,c1d)(modlmap)
if aindex1==aindex2:
icov[modlmap<lmins[aindex1]] = maxval
icov[modlmap>lmaxs[aindex1]] = maxval
cov[aindex1,aindex2] = icov
cov.data = enmap.enmap(cov.data,wcs,copy=False)
covfunc = lambda sel: cov.to_array(sel,flatten=True)
assert cov.data.shape[0]==((narrays*(narrays+1))/2) # FIXME: generalize
assert np.all(np.isfinite(cov.data))
# Make responses
responses = {}
for comp in ['tSZ','CMB','CIB']:
if bandpasses:
if no_act_color_correction:
responses[comp] = tfg.get_mix_bandpassed(bps, comp, bandpass_shifts=shifts)
else:
responses[comp] = tfg.get_mix_bandpassed(bps, comp, bandpass_shifts=shifts,
ccor_cen_nus=cfreqs, ccor_beams=lbeams,
ccor_exps = [ccor_exp] * narrays)
else:
responses[comp] = tfg.get_mix(bps, comp)
ilcgen = ilc.chunked_ilc(modlmap,np.stack(kbeams),covfunc,chunk_size,responses=responses,invert=True)
# Initialize containers
solutions = solutions.split(',')
data = {}
kcoadds = kcoadds.reshape((narrays,Ny*Nx))
for solution in solutions:
data[solution] = {}
comps = solution.split('-')
data[solution]['comps'] = comps
if len(comps)<=2:
data[solution]['noise'] = enmap.zeros((Ny*Nx),wcs)
if len(comps)==2:
data[solution]['cnoise'] = enmap.zeros((Ny*Nx),wcs)
data[solution]['kmap'] = enmap.zeros((Ny*Nx),wcs,dtype=dtype) # FIXME: reduce dtype?
if do_weights and len(comps)<=2:
for qid in arrays:
data[solution]['weight_%s' % qid] = enmap.zeros((Ny*Nx),wcs)
for chunknum,(hilc,selchunk) in enumerate(ilcgen):
print("ILC on chunk ", chunknum+1, " / ",int(modlmap.size/chunk_size)+1," ...")
for solution in solutions:
comps = data[solution]['comps']
if len(comps)==1: # GENERALIZE
data[solution]['noise'][selchunk] = hilc.standard_noise(comps[0])
if do_weights: weight = hilc.standard_weight(comps[0])
data[solution]['kmap'][selchunk] = hilc.standard_map(kcoadds[...,selchunk],comps[0])
elif len(comps)==2:
data[solution]['noise'][selchunk] = hilc.constrained_noise(comps[0],comps[1])
data[solution]['cnoise'][selchunk] = hilc.cross_noise(comps[0],comps[1])
ret = hilc.constrained_map(kcoadds[...,selchunk],comps[0],comps[1],return_weight=do_weights)
if do_weights:
data[solution]['kmap'][selchunk],weight = ret
else:
data[solution]['kmap'][selchunk] = ret
elif len(comps)>2:
data[solution]['kmap'][selchunk] = np.nan_to_num(hilc.multi_constrained_map(kcoadds[...,selchunk],comps[0],*comps[1:]))
if len(comps)<=2 and do_weights:
for qind,qid in enumerate(arrays):
data[solution]['weight_%s' % qid][selchunk] = weight[qind]
del ilcgen,cov
# Reshape into maps
name_map = {'CMB':'cmb','tSZ':'comptony','CIB':'cib'}
beams = beams.split(',')
for solution,beam in zip(solutions,beams):
comps = "tilec_single_tile_"+region+"_"
comps = comps + name_map[data[solution]['comps'][0]]+"_"
if len(data[solution]['comps'])>1: comps = comps + "deprojects_"+ '_'.join([name_map[x] for x in data[solution]['comps'][1:]]) + "_"
comps = comps + version
if do_weights and len(data[solution]['comps'])<=2:
for qind,qid in enumerate(arrays):
enmap.write_map("%s/%s_%s_weight.fits" % (savedir,comps,qid), enmap.enmap(data[solution]['weight_%s' % qid].reshape((Ny,Nx)),wcs))
try:
noise = enmap.enmap(data[solution]['noise'].reshape((Ny,Nx)),wcs)
enmap.write_map("%s/%s_noise.fits" % (savedir,comps),noise)
except: pass
try:
cnoise = enmap.enmap(data[solution]['cnoise'].reshape((Ny,Nx)),wcs)
enmap.write_map("%s/%s_cross_noise.fits" % (savedir,comps),cnoise)
except: pass
ells = np.arange(0,modlmap.max(),1)
try:
fbeam = float(beam)
kbeam = maps.gauss_beam(modlmap,fbeam)
lbeam = maps.gauss_beam(ells,fbeam)
except:
qid = beam
bfunc = lambda x: tutils.get_kbeam(qid,x,version=beam_version,sanitize=not(unsanitized_beam),planck_pixwin=False)
kbeam = bfunc(modlmap)
lbeam = bfunc(ells)
kmap = enmap.enmap(data[solution]['kmap'].reshape((Ny,Nx)),wcs)
smap = enmap.ifft(kbeam*kmap,normalize='phys').real
enmap.write_map("%s/%s.fits" % (savedir,comps),smap)
io.save_cols("%s/%s_beam.txt" % (savedir,comps),(ells,lbeam),header="ell beam")
enmap.write_map(savedir+"/tilec_mask.fits",mask)
def map_ifft(x): return enmap.ifft(x).real def corrfun_thumb(corr, n):
def filter_simple(imap, filter):
return enmap.ifft(enmap.fft(imap) * filter).real
# Beam infos: # https://phy-wiki.princeton.edu/polwiki/pmwiki.php?n=BeamAnalysis.BeamDistributionCenter # - 150GHz S16 PA2: Omega = 191 nsr, ie fwhm = 1.41 arcmin # - 90GHz S16 PA3: Omega = 516 nsr, ie fwhm = 2.32 arcmin # In[29]: # beam sigmas in radians (convert from fwhm to sigma) s150 = 1.41 * np.pi / (180. * 60.) / np.sqrt(8. * np.log(2.)) s90 = 2.32 * np.pi / (180. * 60.) / np.sqrt(8. * np.log(2.)) # do the reconvolution in Fourier space pact90ReconvF = enmap.fft(pact90) l2 = np.sum(pact90.lmap()**2, 0) pact90ReconvF *= np.exp(-0.5 * l2 * (s150**2 - s90**2)) pact90 = enmap.ifft(pact90ReconvF).real # # Measure power spectra # In[ ]: #hMask = enmap.to_healpix(fullMask) #h150 = enmap.to_healpix(pact150) #lCen, ClPACT150, sClPACT150 = powerSpectrum(h150, mask=hMask, theory=[flensedTT, ftotal], fsCl=None, nBins=201, lRange=None, plot=False, path=None, save=False) #h90 = enmap.to_healpix(pact90) #lCen, ClPACT90, sClPACT90 = powerSpectrum(h90, mask=hMask, theory=[flensedTT, ftotal], fsCl=None, nBins=201, lRange=None, plot=False, path=None, save=False) # In[ ]:
kmask = maps.mask_kspace(mask.shape, mask.wcs, lmin=lmin, lmax=30000)
# Get coadd map of reference array
kmap = enmap.fft(
dm.get_coadd("s15", region, array, srcfree=True, ncomp=1)[0] * mask)
# Get pixel window correction
pwin = tutils.get_pixwin(mask.shape[-2:])
bpfile = "data/" + dm.get_bandpass_file_name(array)
# Apply a filter that converts array map to Compton Y units (Eq 8)
f = tfg.get_mix_bandpassed([bpfile],
'tSZ',
ccor_cen_nus=[freqs[array]],
ccor_beams=[lbeam])[0]
f2d = maps.interp(ells, f)(modlmap)
filt = kmask / pwin / f2d
filt[~np.isfinite(filt)] = 0
imap = enmap.ifft(kmap * filt).real
# Reconvolve Y map
ymap = maps.filter_map(enmap.read_map(yfile), beam_ratio * kmask)
# Get CMB maps
smap = enmap.read_map(yfile2)
cmap = enmap.read_map(yfile3)
# Width in pixels of stamp cutout
pix = 60
# Initialize stacks
i = 0
istack = 0
ystack = 0
