def process_splits(splits,
wins,
mask,
skip_splits=False,
do_fft_splits=False,
pixwin=False):
assert wins.ndim > 2
with np.errstate(divide='ignore', invalid='ignore'):
coadd = (splits * wins).sum(axis=0) / wins.sum(axis=0)
coadd[~np.isfinite(coadd)] = 0
Ny, Nx = splits.shape[-2:]
assert coadd.shape == (Ny, Nx)
kcoadd = enmap.enmap(enmap.fft(coadd * mask, normalize='phys'), wins.wcs)
if pixwin: pwin = tutils.get_pixwin(coadd.shape[-2:])
else: pwin = 1
kcoadd = kcoadd / pwin
if not (skip_splits):
#data = (splits-coadd)*mask # !!!! wins removed, was (splits-coadd)*wins*mask earlier
data = (splits - coadd) * mask * wins / wins.sum(
axis=0) # !!!! wins removed, was (splits-coadd)*wins*mask earlier
data[~np.isfinite(data)] = 0
kdiffs = enmap.fft(data, normalize='phys')
kdiffs = kdiffs / pwin
else:
kdiffs = None
if do_fft_splits:
ksplits = enmap.fft(splits * wins * mask, normalize='phys')
ksplits = ksplits / pwin
return kdiffs, kcoadd, ksplits
else:
return kdiffs, kcoadd
def get_n2d_data(splits,ivars,mask_a,coadd_estimator=False,flattened=False,plot_fname=None,dtype=None):
assert np.all(np.isfinite(splits))
assert np.all(np.isfinite(ivars))
assert np.all(np.isfinite(mask_a))
if coadd_estimator:
coadd,civars = get_coadd(splits,ivars,axis=1)
data = splits - coadd[:,None,...]
del coadd
else:
data = splits
assert np.all(np.isfinite(data))
if flattened:
# sivars = np.sqrt(ivars)
sivars = ((1./ivars) - (1./civars[:,None,...]))**-0.5
sivars[~np.isfinite(sivars)] = 0
ffts = enmap.fft(data*mask_a*sivars,normalize="phys")
if plot_fname is not None: plot(plot_fname+"_fft_maps",data*mask_a*sivars,quantile=0)
wmaps = mask_a + enmap.zeros(ffts.shape,mask_a.wcs,dtype=dtype) # WARNING: type
del ivars, data, splits
else:
assert np.all(np.isfinite(data*mask_a*ivars))
ffts = enmap.fft(data*mask_a*ivars,normalize="phys")
if plot_fname is not None: plot(plot_fname+"_fft_maps",data*mask_a*ivars,quantile=0)
wmaps = ivars * mask_a
del ivars, data, splits
n2d = get_n2d(ffts,wmaps,coadd_estimator=coadd_estimator,plot_fname=plot_fname,dtype=dtype)
assert np.all(np.isfinite(n2d))
return n2d
def test_fft(self):
# Tests that ifft(ifft(imap))==imap, i.e. default normalizations are consistent
shape, wcs = enmap.geometry(pos=(0, 0), shape=(3, 100, 100), res=0.01)
imap = enmap.enmap(np.random.random(shape), wcs)
assert np.all(
np.isclose(
imap,
enmap.ifft(enmap.fft(imap, normalize='phy'),
normalize='phy').real))
assert np.all(np.isclose(imap, enmap.ifft(enmap.fft(imap)).real))
def gather_patches_cos(self):
"""Assemble patch statistics for small scale lensing with cos filter.
Compute the small scale (ell > 3000) temperature power at different
patches across the sky as well as the average amplitude of the
background temperature gradient (ell < 2000). For the small scale
statistics, also apply a filter in Fourier space such that:
.. math::
f_\\ell = \\cos(\\hat{\\ell}\\cdot\\hat{\\nabla T})
"""
self._edge = 5 # Edge pixels to throw away
p = self._p
m_fft = enmap.fft(self.map_in)
hp = np.zeros(self.map_in.shape)
hp[np.where((self._lmod > self.lmin) & (self._lmod < self.lmax))] = 1.
# Apply pre-whitening or Wiener/inverse variance filters, then top hat
if self._aW and self.fid is not None:
cs = CubicSpline(self.fid[0], self.fid[1]) # (ell, Cl)
m_fft = m_fft / cs(self._lmod)
self._Tss = enmap.ifft(m_fft * hp)
self._dTy, self._dTx = gradient_flat(self.map_in, self.ldT)
# Scale geometry for lower res map of patches
pshp, pwcs = enmap.scale_geometry(self.map_in.shape, self.map_in.wcs,
1. / self._p)
if not self.savesteps:
del self.map_in, m_fft
self._T2patch = enmap.zeros(pshp, pwcs)
self._dTxpatch = enmap.zeros(pshp, pwcs)
self._dTypatch = enmap.zeros(pshp, pwcs)
self._T_sub = np.zeros((pshp[-2], pshp[-1], p, p))
for i in range(self._T2patch.shape[-2]):
for j in range(self._T2patch.shape[-1]):
self._dTypatch[i, j] = np.mean(self._dTy[i * p:(i + 1) * p,
j * p:(j + 1) * p])
self._dTxpatch[i, j] = np.mean(self._dTx[i * p:(i + 1) * p,
j * p:(j + 1) * p])
Tp = self._Tss[i * p:(i + 1) * p, j * p:(j + 1) * p]
lsuby, lsubx = Tp.lmap()
lsubmod = Tp.modlmap()
lsubmod[0, 0] = 1.
fl = 1.j*(lsubx*self._dTxpatch[i,j] + \
lsuby*self._dTypatch[i,j]) / \
(lsubmod * np.sqrt(self._dTxpatch[i,j]**2 + \
self._dTypatch[i,j]**2))
fl[0, 0] = 0.
self._T_sub[i, j, :, :] = enmap.ifft(enmap.fft(Tp) * fl).real
# Throw away pixels with edge effects
self._T2patch[i, j] = np.var(self._T_sub[i, j, 5:-5, 5:-5])
self._dT2patch = self._dTxpatch**2 + self._dTypatch**2
if not self.savesteps:
del self._dTypatch, self._dTxpatch, self._dTy, self._dTx, self._Tss
del self._T_sub
def gather_patches_cos2(self):
"""Assemble patch statistics for small scale lensing with cos^2 filter.
Compute the small scale (ell > 3000) temperature power at different
patches across the sky as well as the average amplitude of the
background temperature gradient (ell < 2000). For the small scale
statistics, also apply a filter in Fourier space such that:
.. math::
f_\\ell = \\cos^2(\\hat{\\ell}\\cdot\\hat{\\nabla T})
"""
self._edge = 3 # Edge pixels to throw away
p = self._p
m_fft = enmap.fft(self.map_in)
hp = np.zeros(self.map_in.shape)
hp[np.where((self._lmod > self.lmin) & (self._lmod < self.lmax))] = 1.
self._Tss = enmap.ifft(m_fft * hp)
self._dTy, self._dTx = gradient_flat(self.map_in, self.ldT)
# Scale geometry for lower res map of patches
pshp, pwcs = enmap.scale_geometry(self.map_in.shape, self.map_in.wcs,
1. / self._p)
if not self.savesteps:
del self.map_in, m_fft
self._T2patch = enmap.zeros(pshp, pwcs)
self._dTxpatch = enmap.zeros(pshp, pwcs)
self._dTypatch = enmap.zeros(pshp, pwcs)
self._T_sub = np.zeros((pshp[-2], pshp[-1], p, p))
for i in range(self._T2patch.shape[-2]):
for j in range(self._T2patch.shape[-1]):
self._dTypatch[i, j] = np.mean(self._dTy[i * p:(i + 1) * p,
j * p:(j + 1) * p])
self._dTxpatch[i, j] = np.mean(self._dTx[i * p:(i + 1) * p,
j * p:(j + 1) * p])
Tp = self._Tss[i * p:(i + 1) * p, j * p:(j + 1) * p]
lsuby, lsubx = Tp.lmap()
lsubmod = Tp.modlmap()
lsubmod[0, 0] = 1. # Avoid divide by 0; set fl here to 0 later
fl = (lsubx*self._dTxpatch[i,j] + \
lsuby*self._dTypatch[i,j])**2 / \
(lsubmod * np.sqrt(self._dTxpatch[i,j]**2 + \
self._dTypatch[i,j]**2))**2
fl[0, 0] = 0.
self._T_sub[i, j, :, :] = enmap.ifft(enmap.fft(Tp) * fl).real
# Throw away pixels with edge effects
self._T2patch[i, j] = np.var(self._T_sub[i, j, 3:-3, 3:-3])
self._dT2patch = self._dTxpatch**2 + self._dTypatch**2
if not self.savesteps:
del self._dTypatch, self._dTxpatch, self._dTy, self._dTx, self._Tss
del self._T_sub
def compute_ps(map1, map2, mask, beamf1, beamf2):
"""Compute the FFTs, multiply, bin
"""
kmap1 = enmap.fft(map1 * mask, normalize="phys")
kmap2 = enmap.fft(map2 * mask, normalize="phys")
power = (kmap1 * np.conj(kmap2)).real
bin_edges = np.arange(0, 8000, 40)
centers = (bin_edges[1:] + bin_edges[:-1]) / 2.
w2 = np.mean(mask**2.)
modlmap = enmap.modlmap(map1.shape, map1.wcs)
binned_power = bin(power / w2 / beamf1(modlmap) / beamf2(modlmap), modlmap,
bin_edges)
return centers, binned_power
def filter_weighted(imap, ivar, filter, tol=1e-4, ref=0.9):
"""Filter enmap imap with the given 2d fourier filter while
weithing spatially with ivar"""
filter = 1 - filter
omap = enmap.ifft(filter * enmap.fft(imap * ivar)).real
div = enmap.ifft(filter * enmap.fft(ivar)).real
del filter
# Avoid division by very low values
div = np.maximum(div, np.percentile(ivar[::10, ::10], ref * 100) * tol)
# omap = imap - rhs/div
omap /= div
del div
omap *= -1
omap += imap
omap *= imap != 0
return omap
def inpaint_map_const_cov(imap,mask,union_sources_version=None,noise_pix = 20,hole_radius = 3.,plots=False):
"""
Inpaints a map under the assumption of constant 2D Fourier covariance. This uses the average PS
of the full map for the noise model at each source location and thus does not handle inhomogenity.
Pros: no products needed other than map-maker outputs of map
Cons:
imap -- (npol,Ny,Nx)
ivar -- (Ny,Nx)
"""
ras,decs = sints.get_act_mr3f_union_sources(version=union_sources_version)
kmap = enmap.fft(mask*imap,normalize='phys')
gtags = []
gdicts = {}
pcoords = []
for i,(ra,dec) in enumerate(zip(ras,decs)):
sel = reproject.cutout(ivar, ra=np.deg2rad(ra), dec=np.deg2rad(dec), pad=1, corner=False,npix=noise_pix,return_slice=True)
if sel is None: continue
civar = ivar[sel]
if np.any(civar<=0): continue
modrmap = civar.modrmap()
modlmap = civar.modlmap()
res = maps.resolution(civar.shape,civar.wcs)
cimap = imap[sel]
print(ra,dec)
if plots:
for p in range(3): io.plot_img(cimap[p],os.environ['WORK']+"/cimap_%d_%s" % (p,str(i).zfill(2)))
mimap = cimap.copy()
mimap[...,modrmap<np.deg2rad(hole_radius/60.)] = np.nan
for p in range(3): io.plot_img(mimap[p],os.environ['WORK']+"/masked_cimap_%d_%s" % (p,str(i).zfill(2)))
scov = pixcov.scov_from_theory(modlmap,cmb_theory_fn,fn_beam,iau=False)
ncov = pixcov.ncov_from_ivar(civar)
pcov = scov + ncov
gdicts[i] = pixcov.make_geometry(hole_radius=np.deg2rad(hole_radius/60.),n=noise_pix,deproject=True,iau=False,pcov=pcov,res=res)
pcoords.append(np.array((dec,ra)))
gtags.append(i)
if len(gtags)>0:
pcoords = np.stack(pcoords).swapaxes(0,1)
result = pixcov.inpaint(imap,pcoords,deproject=True,iau=False,geometry_tags=gtags,geometry_dicts=gdicts,verbose=True)
if plots:
for i,(ra,dec) in enumerate(zip(ras,decs)):
sel = reproject.cutout(ivar, ra=np.deg2rad(ra), dec=np.deg2rad(dec), pad=1, corner=False,npix=noise_pix,return_slice=True)
if sel is None: continue
civar = ivar[sel]
if np.any(civar<=0): continue
modrmap = civar.modrmap()
modlmap = civar.modlmap()
res = maps.resolution(civar.shape,civar.wcs)
cimap = result[sel]
print("Inpainted ", ra,dec)
if plots:
for p in range(3): io.plot_img(cimap[p],os.environ['WORK']+"/inpainted_cimap_%d_%s" % (p,str(i).zfill(2)))
return result
def cmbmap2alm(i, mtype, p, f, r):
fmap = enmap.read_map(
f.imap[mtype][i]) # load flatsky K-space combined map
# FT
print('2D ell filtering in F-space and assign to healpix map')
alm = enmap.fft(fmap)
# remove some Fourier modes
shape, wcs = fmap.shape, fmap.wcs
kmask = mask_kspace(shape, wcs, lmin=p.lcut, lmax=4000, lxcut=90, lycut=50)
alm[kmask < 0.5] = 0
# alm -> map
fmap = enmap.ifft(alm).real
# transform cmb map to healpix
hpmap = enmap.to_healpix(fmap, nside=p.nside)
# from map to alm
hpmap = r.w * hpmap # masking
#hp.fitsfunc.write_map(f.omap[mtype][i],hpmap,overwrite=True)
alm = curvedsky.utils.hp_map2alm(p.nside, p.lmax, p.lmax, hpmap)
print("save to file")
pickle.dump((alm),
open(f.alm[mtype][i], "wb"),
protocol=pickle.HIGHEST_PROTOCOL)
def compute_ps(map1, map2, beamf1, beamf2):
"""Compute the FFTs, multiply, bin
"""
if args.fft:
kmap1 = enmap.fft(map1*mask, normalize="phys")
kmap2 = enmap.fft(map2*mask, normalize="phys")
power = (kmap1*np.conj(kmap2)).real
bin_edges = np.arange(20,8000,40)
centers = (bin_edges[1:] + bin_edges[:-1])/2.
w2 = np.mean(mask**2.)
modlmap = enmap.modlmap(map1.shape,map1.wcs)
binned_power = bin(power/w2/beamf1(modlmap)/beamf2(modlmap),modlmap,bin_edges)
return centers, binned_power
else:
ells,cls = pcalc.get_power_scalarXscalar(map1*mask, map2*mask,ret_dl=False)
return ells,cls/beamf1(ells)/beamf2(ells)
def gather_patches_plain(self):
"""Assemble patch statistics relevant to lensing at small scales.
Compute the small scale (ell > lmin) temperature power at different
patches across the sky as well as the average amplitude of the
background temperature gradient (ell < ldT).
"""
self._edge = 0 # Not throwing away edge pixels
m_fft = enmap.fft(self.map_in)
hp = np.zeros(self.map_in.shape)
hp[np.where((self._lmod > self.lmin) & (self._lmod < self.lmax))] = 1.
self._Tss = enmap.ifft(m_fft * hp)
self._dTy, self._dTx = gradient_flat(self.map_in, self.ldT)
# Scale geometry for lower res map of patches
pshp, pwcs = enmap.scale_geometry(self.map_in.shape, self.map_in.wcs,
1. / self._p)
if not self.savesteps:
del self.map_in, m_fft
self._T2patch = enmap.zeros(pshp, pwcs)
self._dTxpatch = enmap.zeros(pshp, pwcs)
self._dTypatch = enmap.zeros(pshp, pwcs)
Trs = self._Tss[:pshp[-2] * self._p, :pshp[-1] * self._p].reshape(
[pshp[-2], self._p, pshp[-1], self._p])
dTxrs = self._dTx[:pshp[-2] * self._p, :pshp[-1] * self._p].reshape(
[pshp[-2], self._p, pshp[-1], self._p])
dTyrs = self._dTy[:pshp[-2] * self._p, :pshp[-1] * self._p].reshape(
[pshp[-2], self._p, pshp[-1], self._p])
self._T2patch[:, :] = np.var(Trs, axis=(1, 3))
self._dTypatch[:, :] = np.mean(dTyrs, axis=(1, 3))
self._dTxpatch[:, :] = np.mean(dTxrs, axis=(1, 3))
self._dT2patch = self._dTxpatch**2 + self._dTypatch**2
if not self.savesteps:
del self._dTypatch, self._dTxpatch, self._dTy, self._dTx, self._Tss
def gradient_flat(map_in, lmax=2000):
"""Return the gradient maps of map_in.
Compute the gradient of the map_in in Fourier space. Simultaneously
low-pass the maps such that they only include modes ell < lmax.
Parameters
----------
map_in: ndmap, ndarray
The input map.
lmax: int
The cutoff ell for low-passing the maps.
Returns
-------
dy: ndmap, ndarray
The gradient in the y-direction
dx: ndmap, ndarray
The gradient in the x-direction
"""
ly, lx = map_in.lmap()
lmod = map_in.modlmap()
lp = enmap.zeros(map_in.shape)
lp[np.where(lmod < lmax)] = 1.
map_fft = enmap.fft(map_in)
dx = enmap.ifft(map_fft*lp*lx*1j).real
dy = enmap.ifft(map_fft*lp*ly*1j).real
return dy, dx
def sim_noise_oneoverf(ivar, apod, lknee=3000, lmin=50, alpha=-3.5):
l = np.maximum(ivar.modlmap(), lmin)
fscale = (1 + (l / lknee)**alpha)**0.5
ref = np.mean(ivar[ivar != 0])
sim = enmap.rand_gauss(ivar.shape, ivar.wcs, dtype=ivar.dtype)
sim = enmap.ifft(enmap.fft(sim) * fscale).real * np.maximum(
ivar, ref * 1e-3)**-0.5 * apod**2
return sim
def filter_riseset(imap, ivar, riseset, filter, tol=1e-4, ref=0.9):
"""Filter enmap imap with the given 2d fourier filter while
weighting spatially with ivar and allowing for rise vs set-dependent
pickup. riseset should be a map with values between -1 and 1 that determines
the balance between rising and setting scans, with 0 meaning equal weight from both.
Overall this is only a marginal improvement over filter_weighted depsite being
quite a bit heaver, and it only helps a bit for the pa7 residual stripe issue it
was invented to deal with, so it probably isn't worth using.
"""
# We model the map as m = Pa+n, where a is [2] is the rising and setting pickup in
# a single pixel, and P is the map's response to this pickup, which is
# P = [x,1-x]*filter, where x=(1-riseset)/2. Given this we can solve for a as:
# a = (P'N"P)"P'N"m, where N" is ivar. Written out this becomes, for a single pixel.
# rhs = ifft(filter*fft([x,1-x]*ivar*imap))
# div = ifft(filter*fft([x,1-x]*ivar*[x,1-x]*ifft(filter)
# Hm.... The problem here is that we're assuming that there's nothing in the other pixels.
# That's not what we did in filter_weighted. Let's just do something simple for now.
x = (1 - riseset) / 2
x1x = np.array([x, 1 - x])
del x
# Broadcast to imap shape
x1x = x1x[(slice(None), ) + (None, ) * (imap.ndim - 2)]
print(imap.shape, x1x.shape)
filter = 1 - filter
rhs = enmap.ifft(filter * enmap.fft(x1x * ivar * imap)).real
div = enmap.ifft(filter *
enmap.fft(x1x[:, None] * x1x[None, :] * ivar)).real
del filter
# Avoid division by very low values
ref = np.percentile(ivar[::10, ::10], ref * 100) * tol
for i in range(2):
div[i, i] = np.maximum(div[i, i], ref)
# Solve the system
rhs = np.moveaxis(rhs, 0, -1)
div = np.moveaxis(div, (0, 1), (-2, -1))
omap = np.linalg.solve(div, rhs)
del rhs, div
omap = np.sum(x1x * np.moveaxis(omap, -1, 0), 0)
del x1x
omap = enmap.ndmap(omap, imap.wcs)
omap *= -1
omap += imap
omap *= imap != 0
return omap
def powspec(map1, map2, taper, taper_order, modlmap, ellmin, ellmax,
delta_ell):
bin_edges = np.arange(ellmin, ellmax, delta_ell)
binner = utils.bin2D(modlmap, bin_edges)
kmap1 = enmap.fft(map1, normalize='phys')
kmap2 = enmap.fft(map2, normalize='phys')
# kmap1_ave = utils.bin2D(modlmap, bin_edges)
# kmap2_ave = utils.bin2D(modlmap, bin_edges)
# correct power spectra
w = np.mean(taper**taper_order)
p2d = (kmap1 * kmap2.conj()).real / w
# p2d = ((kmap1-kmap1_ave) * (kmap2.conj()-kmap2_ave.conj())).real / w
# p2d = abs((kmap1 * kmap2.conj()).real / w)
centers, p1d = binner.bin(p2d)
return centers, p1d
def util_bin_FFT_CAR(map1, map2, mask, beam1, beam2, lmax=8000):
"""Compute the FFTs, multiply, bin
Beams are multiplied at bin centers. This is the worst
job you could do for calculating power spectra.
"""
# beam_ells = np.arange(lmax+1)
kmap1 = enmap.fft(map1 * mask, normalize="phys")
kmap2 = enmap.fft(map2 * mask, normalize="phys")
power = (kmap1 * np.conj(kmap2)).real
bin_edges = np.arange(0, lmax, 40)
centers = (bin_edges[1:] + bin_edges[:-1]) / 2.0
w2 = np.mean(mask**2.0)
modlmap = enmap.modlmap(map1.shape, map1.wcs)
binned_power = util_bin_FFTspec_CAR(power / w2, modlmap, bin_edges)
binned_power *= beam1[centers.astype(int)]
binned_power *= beam2[centers.astype(int)]
return centers, binned_power
def beam_match(imap, f1, f2):
"""f1, f2 are fcodes instead of the actual frequency centers. It
assumes that the first one (f1) has larger beam, so f2 will be matched
to it.
"""
from common import fwhms
l = imap.modlmap()
bmap_f1 = np.exp(-0.5 * l**2 * (fwhms[f1] * u.fwhm * u.arcmin)**2)
bmap_f2 = np.exp(-0.5 * l**2 * (fwhms[f2] * u.fwhm * u.arcmin)**2)
rmap = enmap.ifft(enmap.fft(imap) *
(bmap_f1 / np.maximum(bmap_f2, 1e-3))).real
return rmap
def remove_lxly(fmap, lmin=100, lmax=4096, lxcut=90, lycut=50):
alm = enmap.fft(fmap)
shape, wcs = fmap.shape, fmap.wcs
kmask = define_lmask(shape,
wcs,
lmin=lmin,
lmax=lmax,
lxcut=lxcut,
lycut=lycut)
#print(np.shape(kmask),np.shape(alm))
alm[kmask < 0.5] = 0
fmap_fl = enmap.ifft(alm).real
return fmap_fl
def smooth_ps_gauss(ps2d, lsigma=200):
"""Smooth a 2d power spectrum to the target resolution in l. Simple
gaussian smoothing avoids ringing."""
# This hasn't been tested in isolation, but breaks when used in smooth_ps_mixed
# First get our pixel size in l
ly, lx = enmap.laxes(ps2d.shape, ps2d.wcs)
ires = np.array([ly[1], lx[1]])
sigma_pix = np.abs(lsigma / ires)
fmap = enmap.fft(ps2d)
ky = np.fft.fftfreq(ps2d.shape[-2]) * sigma_pix[0]
kx = np.fft.fftfreq(ps2d.shape[-1]) * sigma_pix[1]
kr2 = ky[:, None]**2 + kx[None, :]**2
fmap *= np.exp(-0.5 * kr2)
return enmap.ifft(fmap).real
def filter_fast(map, vk_mask=None, hk_mask=None, normalize="phys", inv_pixwin_lxly=None):
"""Filter the map in Fourier space removing modes in a horizontal and vertical band
defined by hk_mask and vk_mask. This is a faster version that what is implemented in pspy
We also include an option for removing the pixel window function
Parameters
---------
map: ``so_map``
the map to be filtered
vk_mask: list with 2 elements
format is fourier modes [-lx,+lx]
hk_mask: list with 2 elements
format is fourier modes [-ly,+ly]
normalize: string
optional normalisation of the Fourier transform
inv_pixwin_lxly: 2d array
the inverse of the pixel window function in fourier space
"""
lymap, lxmap = map.data.lmap()
ly, lx = lymap[:,0], lxmap[0,:]
# filtered_map = map.copy()
ft = enmap.fft(map.data, normalize=normalize)
if vk_mask is not None:
id_vk = np.where((lx > vk_mask[0]) & (lx < vk_mask[1]))
if hk_mask is not None:
id_hk = np.where((ly > hk_mask[0]) & (ly < hk_mask[1]))
if map.ncomp == 1:
if vk_mask is not None:
ft[: , id_vk] = 0.
if hk_mask is not None:
ft[id_hk , :] = 0.
if map.ncomp == 3:
for i in range(3):
if vk_mask is not None:
ft[i, : , id_vk] = 0.
if hk_mask is not None:
ft[i, id_hk , :] = 0.
if inv_pixwin_lxly is not None:
ft *= inv_pixwin_lxly
map.data[:] = np.real(enmap.ifft(ft, normalize=normalize))
return map
def process_map(imap, box=None, deconvolve=False, freq=None, smooth=None):
if deconvolve:
fwhm = fwhms[freq]
l = imap.modlmap()
bmap = np.exp(-0.5*l**2*(fwhm*u.fwhm*u.arcmin)**2)
bmap = np.maximum(bmap, 1e-3)
fmap = enmap.fft(imap)
omap = enmap.ifft(fmap/bmap).real
else:
omap = imap
if smooth is not None:
omap = enmap.smooth_gauss(omap, smooth*u.fwhm*u.arcmin)
if box is not None: omap = omap.submap(box)
if args.comp == 0: return np.log10(omap[0])
if args.comp == 12: return np.sum(omap[1:]**2, axis=0)**0.5
else: return omap[args.comp]
def get_filtered_map(map, binary, vk_mask, hk_mask, normalize=False):
"""Filter the map in Fourier space removing modes in a horizontal and vertical band
defined by hk_mask and vk_mask. Note that we mutliply the maps by a binary mask before
doing this operation in order to remove pathological pixels
Parameters
---------
orig_map: ``so_map``
the map to be filtered
binary: ``so_map``
a binary mask removing pathological pixels
vk_mask: list with 2 elements
format is fourier modes [-lx,+lx]
hk_mask: list with 2 elements
format is fourier modes [-ly,+ly]
"""
if map.ncomp == 1:
map.data *= binary.data
else:
map.data[:] *= binary.data
lymap, lxmap = map.data.lmap()
ly, lx = lymap[:, 0], lxmap[0, :]
# filtered_map = map.copy()
ft = enmap.fft(map.data, normalize=normalize)
if vk_mask is not None:
id_vk = np.where((lx > vk_mask[0]) & (lx < vk_mask[1]))
if hk_mask is not None:
id_hk = np.where((ly > hk_mask[0]) & (ly < hk_mask[1]))
if map.ncomp == 1:
if vk_mask is not None:
ft[:, id_vk] = 0.0
if hk_mask is not None:
ft[id_hk, :] = 0.0
if map.ncomp == 3:
if vk_mask is not None:
ft[:, :, id_vk] = 0.0
if hk_mask is not None:
ft[:, id_hk, :] = 0.0
map.data[:] = np.real(enmap.ifft(ft, normalize=normalize))
return map
def smooth_ps_grid(ps, res, alpha=4, log=False, ndof=2):
"""Smooth a 2d power spectrum to the target resolution in l"""
# First get our pixel size in l
lx, ly = enmap.laxes(ps.shape, ps.wcs)
ires = np.array([lx[1],ly[1]])
smooth = np.abs(res/ires)
# We now know how many pixels to somoth by in each direction,
# so perform the actual smoothing
if log: ps = np.log(ps)
fmap = enmap.fft(ps)
ky = np.fft.fftfreq(ps.shape[-2])
kx = np.fft.fftfreq(ps.shape[-1])
fmap /= 1 + np.abs(2*ky[:,None]*smooth[0])**alpha
fmap /= 1 + np.abs(2*kx[None,:]*smooth[1])**alpha
ps = enmap.ifft(fmap).real
if log: ps = np.exp(ps)*log_smooth_corrections[ndof]
return ps
def get_pol_powers(tmap, qmap, umap, kbeam, mask_w2):
f = lambda x: enmap.fft(x, normalize='phys')
p = lambda x, y: (x * y.conj()).real / mask_w2 / kbeam**2
kt = f(tmap)
kq = -f(qmap)
ku = f(umap)
tt = p(kt, kt)
tu = p(kt, ku)
tq = p(kt, kq)
qq = p(kq, kq)
uu = p(ku, ku)
qu = p(kq, ku)
p2d = np.array([[tt, tq, tu], [tq, qq, qu], [tu, qu, uu]])
p2d[~np.isfinite(p2d)] = 0
rp2d = enmap.enmap(
maps.rotate_pol_power(tmap.shape, tmap.wcs, p2d, iau=False), tmap.wcs)
return rp2d[0, 0], rp2d[1, 1], rp2d[2, 2]
def map_act_hist(patches, numerics, wf, masks, apo_masks, \
negbins, posbins) :#{{{
# @TODO: check final pixel count!
# @TODO check changing cmb or noise power changes variance as expected. it will.
# @TODO plot histogram with and without cmb
# @TODO plot histogram with and without WF
# maps are sub-divided into six patches; analyze each separately and then get the PDF of the whole map
dat_tot = np.array([])
for i in range(numerics['n_patch']):
patch = patches[i]
mask = copy.deepcopy(apo_masks[i])
patch *= mask #apply mask
modlmap = patch.modlmap(
) #map of |\vec{ell}| in 2D Fourier domain for the patch
Fell2d = interp1d(wf[:, 0],
wf[:, 1],
bounds_error=False,
fill_value=0.)(
modlmap) #get filter in 2D Fourier domain
kmap = enmap.fft(patch,
normalize="phys") #FFT the patch to 2D Fourier domain
kfilt = kmap * Fell2d
patch_filt = enmap.ifft(kfilt, normalize="phys").real
# point source mask (different for each patch)
newMask = copy.deepcopy(masks[i])
newMask[363, :] = 1.
newMask[:, 2181] = 1.
la = np.where(newMask < 0.1)
mask[la] = 0.
### pick out unmasked data
loc2 = np.where(mask > 0.9)
dat = patch_filt[loc2] #*fraction
dat_tot = np.append(dat_tot, np.ravel(dat))
# histogram the unmasked data
histneg, bin_edgesneg = np.histogram(dat_tot, negbins)
histpos, bin_edgespos = np.histogram(dat_tot, posbins[::-1])
histall = np.concatenate((histneg, histpos))
PDFpatch = histall / float(len(dat_tot))
return PDFpatch
def kspace_filter_fast(map, vk_mask=None, hk_mask=None, normalize="phys"):
"""Filter the map in Fourier space removing modes in a horizontal and vertical band
defined by hk_mask and vk_mask. This is a faster version that what is implemented in pspy
Parameters
---------
map: ``so_map``
the map to be filtered
vk_mask: list with 2 elements
format is fourier modes [-lx,+lx]
hk_mask: list with 2 elements
format is fourier modes [-ly,+ly]
"""
lymap, lxmap = map.data.lmap()
ly, lx = lymap[:,0], lxmap[0,:]
# filtered_map = map.copy()
ft = enmap.fft(map.data, normalize=normalize)
if vk_mask is not None:
id_vk = np.where((lx > vk_mask[0]) & (lx < vk_mask[1]))
if hk_mask is not None:
id_hk = np.where((ly > hk_mask[0]) & (ly < hk_mask[1]))
if map.ncomp == 1:
if vk_mask is not None:
ft[: , id_vk] = 0.
if hk_mask is not None:
ft[id_hk , :] = 0.
if map.ncomp == 3:
for i in range(3):
if vk_mask is not None:
ft[i, : , id_vk] = 0.
if hk_mask is not None:
ft[i, id_hk , :] = 0.
map.data[:] = np.real(enmap.ifft(ft, normalize=normalize))
return map
def deconvolve_pixwin_CAR(orig_map, binary, inv_pixwin_lxly, norm=0):
"""Deconvolve the two dimensional CAR pixel window function
Parameters
---------
orig_map: ``so_map``
the map to be filtered
binary: ``so_map``
a binary mask removing pathological pixels
inv_pixwin_lxly: 2d array
the inverse of the pixel window function in fourier space
norm: integer
everytime we do a FFT + IFFT operation we get a normalisation cst to take care of
this is counting the number of occurence of it, note that we could correct directly by
using normalize="physical", but dividing the map by a number is slow so we prefer to do it later
in the code.
"""
orig_map.data *= binary.data
ft = enmap.fft(orig_map.data, normalize=False)
ft *= inv_pixwin_lxly
orig_map.data = enmap.ifft(ft, normalize=False).real
norm += 1
return norm, orig_map
dm = sints.PlanckHybrid(region=mask)
for qid in qids:
split = enmap.read_map("%ssplit_%s.fits" % (froot,qid))
#io.hplot(enmap.downgrade(split,4),"split_%s" % qid)
arrayname = arraynames[qid]
wts = dm.get_splits_ivar(arrayname)[0,:,0,...]
coadd,_ = noise.get_coadd(split[:,0,...],wts,axis=0) * mask
pl = io.Plotter(xyscale='linlog',xlabel='l',ylabel='C')
kmap = enmap.fft(coadd,normalize='phys')
p2d = np.real(kmap*kmap.conj())/w2
cents,p1d = binner.bin(p2d)
pl.add(cents,p1d,label='sim power')
kcoadd = enmap.enmap(np.load("%skcoadd_%s.npy" % (kroot,qid)),wcs)
p2d = np.real(kcoadd*kcoadd.conj())/w2
cents,p1d = binner.bin(p2d)
pl.add(cents,p1d,label='sim power 2')
p2d = enmap.enmap(np.load("%stilec_hybrid_covariance_%s_%s.npy" % (kroot,qid,qid)),wcs)
cents,p1d = binner.bin(p2d)
pl.add(cents,p1d,label='sim cov')
def fitQ(config):
"""Calculates the filter mismatch function Q on a grid of scale sizes (given in terms of theta500 in
arcmin), for each tile in the map. The results are initially cached (with a separate .fits table for each
tile) under the selFn directory, before being combined into a single file at the end of a nemo run.
Use GNFWParams (in config.parDict) if you want to specify a different cluster profile shape.
Args:
config (:obj:`startUp.NemoConfig`): A NemoConfig object.
Note:
See :obj:`loadQ` for how to load in the output of this routine.
"""
if config.parDict['GNFWParams'] == 'default':
GNFWParams = gnfw._default_params
else:
GNFWParams = config.parDict['GNFWParams']
# Spin through the filter kernels
photFilterLabel = config.parDict['photFilter']
filterList = config.parDict['mapFilters']
for f in filterList:
if f['label'] == photFilterLabel:
ref = f
# M, z ranges for Q calc
# NOTE: ref filter that sets scale we compare to must ALWAYS come first
# To safely (numerically, at least) apply Q at z ~ 0.01, we need to go to theta500 ~ 500 arcmin (< 10 deg)
MRange = [ref['params']['M500MSun']]
zRange = [ref['params']['z']]
# This should cover theta500Arcmin range fairly evenly without using too crazy masses
#minTheta500Arcmin=0.5
#maxTheta500Arcmin=20.0
minTheta500Arcmin = 0.1
maxTheta500Arcmin = 500.0
numPoints = 50
#theta500Arcmin_wanted=np.logspace(np.log10(1e-1), np.log10(500), 50)
theta500Arcmin_wanted = np.logspace(np.log10(minTheta500Arcmin),
np.log10(maxTheta500Arcmin), numPoints)
zRange_wanted = np.zeros(numPoints)
zRange_wanted[np.less(theta500Arcmin_wanted, 3.0)] = 2.0
zRange_wanted[np.logical_and(np.greater(theta500Arcmin_wanted, 3.0),
np.less(theta500Arcmin_wanted, 6.0))] = 1.0
zRange_wanted[np.logical_and(np.greater(theta500Arcmin_wanted, 6.0),
np.less(theta500Arcmin_wanted, 10.0))] = 0.5
zRange_wanted[np.logical_and(np.greater(theta500Arcmin_wanted, 10.0),
np.less(theta500Arcmin_wanted, 20.0))] = 0.1
zRange_wanted[np.logical_and(np.greater(theta500Arcmin_wanted, 20.0),
np.less(theta500Arcmin_wanted, 30.0))] = 0.05
zRange_wanted[np.greater(theta500Arcmin_wanted, 30.0)] = 0.01
MRange_wanted = []
for theta500Arcmin, z in zip(theta500Arcmin_wanted, zRange_wanted):
Ez = astCalc.Ez(z)
Hz = astCalc.Ez(z) * astCalc.H0
G = 4.301e-9 # in MSun-1 km2 s-2 Mpc
criticalDensity = (3 * np.power(Hz, 2)) / (8 * np.pi * G)
R500Mpc = np.tan(np.radians(theta500Arcmin / 60.0)) * astCalc.da(z)
M500 = (4 / 3.0) * np.pi * np.power(R500Mpc, 3) * 500 * criticalDensity
MRange_wanted.append(M500)
MRange = MRange + MRange_wanted
zRange = zRange + zRange_wanted.tolist()
# Here we save the fit for each tile separately...
# completeness.tidyUp will put them into one file at the end of a nemo run
for tileName in config.tileNames:
tileQTabFileName = config.selFnDir + os.path.sep + "QFit#%s.fits" % (
tileName)
if os.path.exists(tileQTabFileName) == True:
print("... already done Q fit for tile %s ..." % (tileName))
continue
print("... fitting Q in tile %s ..." % (tileName))
# Load reference scale filter
foundFilt = False
for filt in config.parDict['mapFilters']:
if filt['label'] == config.parDict['photFilter']:
foundFilt = True
break
if foundFilt == False:
raise Exception("couldn't find filter that matches photFilter")
filterClass = eval('filters.%s' % (filt['class']))
filterObj=filterClass(filt['label'], config.unfilteredMapsDictList, filt['params'], \
tileName = tileName,
diagnosticsDir = config.diagnosticsDir+os.path.sep+tileName)
filterObj.loadFilter()
# Real space kernel or Fourier space filter?
if issubclass(filterObj.__class__,
filters.RealSpaceMatchedFilter) == True:
realSpace = True
else:
realSpace = False
# Set-up the beams
beamsDict = {}
for mapDict in config.parDict['unfilteredMaps']:
obsFreqGHz = mapDict['obsFreqGHz']
beamsDict[obsFreqGHz] = mapDict['beamFileName']
# A bit clunky but gets map pixel scale and shrinks map size we'll use for inserting signals
# NOTE: 5 deg is too small for the largest very low-z clusters: it's better to add a z cut and ignore those
# NOTE: 5 deg fell over (ringing) for tiles_v2 RE6_10_0, but 10 deg worked fine
extMap = np.zeros(filterObj.shape)
wcs = filterObj.wcs
RADeg, decDeg = wcs.getCentreWCSCoords()
clipDict = astImages.clipImageSectionWCS(extMap, wcs, RADeg, decDeg,
10.0)
wcs = clipDict['wcs']
extMap = clipDict['data']
# Input signal maps to which we will apply filter(s)
# We do this once and store in a dictionary for speed
theta500Arcmin = []
signalMapDict = {}
signalMap = np.zeros(extMap.shape)
degreesMap = np.ones(signalMap.shape, dtype=float) * 1e6
degreesMap, xBounds, yBounds = nemoCython.makeDegreesDistanceMap(
degreesMap, wcs, RADeg, decDeg, 10.0)
for z, M500MSun in zip(zRange, MRange):
key = '%.2f_%.2f' % (z, np.log10(M500MSun))
signalMaps = []
fSignalMaps = []
y0 = 2e-4
for obsFreqGHz in list(beamsDict.keys()):
if mapDict['obsFreqGHz'] is not None: # Normal case
amplitude = maps.convertToDeltaT(y0, obsFreqGHz)
else: # TILe-C case
amplitude = y0
# NOTE: Q is to adjust for mismatched filter shape
# Yes, this should have the beam in it (certainly for TILe-C)
signalMap = makeArnaudModelSignalMap(
z,
M500MSun,
degreesMap,
wcs,
beamsDict[obsFreqGHz],
amplitude=amplitude,
convolveWithBeam=True,
GNFWParams=config.parDict['GNFWParams'])
if realSpace == True:
signalMaps.append(signalMap)
else:
signalMaps.append(enmap.fft(signalMap))
signalMaps = np.array(signalMaps)
signalMapDict[key] = signalMaps
theta500Arcmin.append(
calcTheta500Arcmin(z, M500MSun, fiducialCosmoModel))
theta500Arcmin = np.array(theta500Arcmin)
# Filter maps with the ref kernel
# NOTE: keep only unique values of Q, theta500Arcmin (or interpolation routines will fail)
Q = []
QTheta500Arcmin = []
count = 0
for z, M500MSun in zip(zRange, MRange):
key = '%.2f_%.2f' % (z, np.log10(M500MSun))
filteredSignal = filterObj.applyFilter(signalMapDict[key])
peakFilteredSignal = filteredSignal.max()
if peakFilteredSignal not in Q:
Q.append(peakFilteredSignal)
QTheta500Arcmin.append(theta500Arcmin[count])
count = count + 1
Q = np.array(Q)
Q = Q / Q[0]
# Sort and do spline fit... save .fits table of theta, Q
QTab = atpy.Table()
QTab.add_column(atpy.Column(Q, 'Q'))
QTab.add_column(atpy.Column(QTheta500Arcmin, 'theta500Arcmin'))
QTab.sort('theta500Arcmin')
QTab.meta['NEMOVER'] = nemo.__version__
QTab.write(tileQTabFileName, overwrite=True)
#rank_QTabDict[tileName]=QTab
# Fit with spline
tck = interpolate.splrep(QTab['theta500Arcmin'], QTab['Q'])
# Plot
plotSettings.update_rcParams()
plt.figure(figsize=(9, 6.5))
ax = plt.axes([0.12, 0.11, 0.86, 0.88])
#plt.tick_params(axis='both', which='major', labelsize=15)
#plt.tick_params(axis='both', which='minor', labelsize=15)
thetaArr = np.linspace(0, 500, 100000)
plt.plot(thetaArr, interpolate.splev(thetaArr, tck), 'k-')
plt.plot(QTheta500Arcmin, Q, 'D', ms=8)
#plt.xlim(0, 9)
plt.ylim(0, 1.9)
#plt.xlim(0, thetaArr.max())
#plt.xlim(0, 15)
plt.xlim(0.1, 500)
plt.semilogx()
plt.xlabel("$\\theta_{\\rm 500c}$ (arcmin)")
plt.ylabel("$Q$ ($M_{\\rm 500c}$, $z$)")
plt.savefig(config.diagnosticsDir + os.path.sep + tileName +
os.path.sep + "QFit_%s.pdf" % (tileName))
plt.savefig(config.diagnosticsDir + os.path.sep + tileName +
os.path.sep + "QFit_%s.png" % (tileName))
plt.close()
print("... Q fit finished [tileName = %s, rank = %d] ..." %
(tileName, config.rank))
def pow(x,y=None):
k = enmap.fft(x,normalize='phys')
ky = enmap.fft(y,normalize='phys') if y is not None else k
p = (k*ky.conj()).real
cents,p1d = binner.bin(p)
return p,cents,p1d
