def generate_noise_sim(covsqrt, ivars, seed=None, dtype=None):
"""
Supports only two cases
1) nfreqs>=1,npol=3
2) nfreqs=1,npol=1
"""
if isinstance(seed, int): seed = (seed, )
assert np.all(np.isfinite(covsqrt))
shape, wcs = covsqrt.shape, covsqrt.wcs
Ny, Nx = shape[-2:]
ncomps = covsqrt.shape[0]
assert ncomps == covsqrt.shape[1]
assert ((ncomps % 3) == 0) or (ncomps == 1)
nfreqs = 1 if ncomps == 1 else ncomps // 3
if ncomps == 1: npol = 1
else: npol = 3
wmaps = enmap.extract(ivars, shape[-2:], wcs)
nsplits = wmaps.shape[1]
if dtype is np.float32: ctype = np.complex64
elif dtype is np.float64: ctype = np.complex128
# Old way with loop
kmap = []
for i in range(nsplits):
if seed is None:
np.random.seed(None)
else:
np.random.seed(seed + (i, ))
rmap = enmap.rand_gauss_harm((ncomps, Ny, Nx),
covsqrt.wcs).astype(ctype)
kmap.append(enmap.map_mul(covsqrt, rmap))
del covsqrt, rmap
kmap = enmap.enmap(np.stack(kmap), wcs)
outmaps = enmap.ifft(kmap, normalize="phys").real
del kmap
# Need to test this more ; it's only marginally faster and has different seed behaviour
# covsqrt = icovsqrt
# np.random.seed(seed)
# rmap = enmap.rand_gauss_harm((nsplits,ncomps,Ny, Nx),covsqrt.wcs)
# kmap = enmap.samewcs(np.einsum("abyx,cbyx->cayx", covsqrt, rmap),rmap)
# outmaps = enmap.ifft(kmap, normalize="phys").real
# isivars = 1/np.sqrt(wmaps)
with np.errstate(divide='ignore', invalid='ignore'):
isivars = ((1. / wmaps) - (1. / wmaps.sum(axis=1)[:, None, ...]))**0.5
isivars[~np.isfinite(isivars)] = 0
assert np.all(np.isfinite(outmaps))
# Divide by hits
for ifreq in range(nfreqs):
outmaps[:, ifreq * npol:(ifreq + 1) * npol,
...] = outmaps[:, ifreq * npol:(ifreq + 1) * npol,
...] * isivars[ifreq, ...] * np.sqrt(nsplits)
retmaps = outmaps.reshape((nsplits, nfreqs, npol, Ny, Nx)).swapaxes(0, 1)
assert np.all(np.isfinite(retmaps))
return retmaps, wmaps
def test_sim_slice(self):
ps = powspec.read_spectrum(DATA_PREFIX+"test_scalCls.dat")[:1,:1]
test_res_arcmin = 10.0
lmax = 2000
fact = 2.
shape,wcs = enmap.fullsky_geometry(res=np.deg2rad(test_res_arcmin/60.),proj='car')
omap = curvedsky.rand_map(shape, wcs, ps,lmax=lmax)
ofunc = lambda ishape,iwcs: fact*enmap.extract(omap,ishape,iwcs)
nmap = reproject.populate(shape,wcs,ofunc,maxpixy = 400,maxpixx = 400)
assert np.all(np.isclose(nmap/omap,2.))
def test_sim_slice():
path = os.path.dirname(enmap.__file__) + "/../tests/"
ps = powspec.read_spectrum(path + "data/test_scalCls.dat")[:1, :1]
test_res_arcmin = 10.0
lmax = 2000
fact = 2.
shape, wcs = enmap.fullsky_geometry(res=np.deg2rad(test_res_arcmin / 60.),
proj='car')
omap = curvedsky.rand_map(shape, wcs, ps, lmax=lmax)
ofunc = lambda ishape, iwcs: fact * enmap.extract(omap, ishape, iwcs)
nmap = reproject.populate(shape, wcs, ofunc, maxpixy=400, maxpixx=400)
assert np.all(np.isclose(nmap / omap, 2.))
def test_extract(self):
# Tests that extraction is sensible
shape, wcs = enmap.geometry(pos=(0, 0), shape=(500, 500), res=0.01)
imap = enmap.enmap(np.random.random(shape), wcs)
smap = imap[200:300, 200:300]
sshape, swcs = smap.shape, smap.wcs
smap2 = enmap.extract(imap, sshape, swcs)
pixbox = enmap.pixbox_of(imap.wcs, sshape, swcs)
# Do write and read test
filename = "temporary_extract_map.fits" # NOT THREAD SAFE
enmap.write_map(filename, imap)
smap3 = enmap.read_map(filename, pixbox=pixbox)
os.remove(filename)
assert np.all(np.isclose(smap, smap2))
assert np.all(np.isclose(smap, smap3))
assert wcsutils.equal(smap.wcs, smap2.wcs)
assert wcsutils.equal(smap.wcs, smap3.wcs)
# generate simulation from the covariance
if nsims > 0:
bin_edges = np.arange(40, 8000, 40)
p1ds = []
for i in range(nsims):
print("Sim %d of %d ..." % (i + 1, nsims))
with bench.show("simgen"):
sims = ngen.generate_sim(season=season,
patch=patch,
array=array,
seed=i,
mask_patch=mask_patch)
print(sims.nbytes / 1024. / 1024. / 1024., " GB", sims.shape,
sims.dtype)
if args.extract_mask is not None:
ivars2 = enmap.extract(ivars, eshape, ewcs)
modlmap = enmap.modlmap(eshape, ewcs)
else:
ivars2 = ivars
if args.debug and i == 0: noise.plot(pout + "_sims", sims)
if not (args.no_write):
ngen.save_sims(i,
sims,
season,
patch,
array,
coadd=coadd,
mask_patch=mask_patch)
n2d_sim = noise.get_n2d_data(sims,
ivars2,
def _footprint(self, imap):
if not (self._refoot): return imap
else: return enmap.extract(imap, self._eshape, self._wcs)
Ncrop = 400
navg = lambda x,delta : covtools.noise_block_average(x,nsplits=1,delta_ell=delta,
radial_fit=False,lmax=None,
wnoise_annulus=None,
lmin = 40,
bin_annulus=None,fill_lmax=None,
log=False)
loc = "/home/r/rbond/sigurdkn/project/actpol/maps/mr3f_20190328/transfun/release/"
mask = sints.get_act_mr3_crosslinked_mask('deep56')
binner = stats.bin2D(mask.modlmap(),bin_edges)
bin = lambda x: binner.bin(x)
isim1 = enmap.extract(enmap.read_map(loc+'../sims/deep56_00.fits'),mask.shape,mask.wcs) * mask
isim2 = enmap.extract(enmap.read_map(loc+'../sims/deep56_01.fits'),mask.shape,mask.wcs) * mask
tmap1 = enmap.extract(enmap.read_map(loc+'s15_deep56_pa2_f150_nohwp_night_sim00_3pass_4way_coadd_transmap.fits'),mask.shape,mask.wcs) * mask
tmap2 = enmap.extract(enmap.read_map(loc+'s15_deep56_pa2_f150_nohwp_night_sim01_3pass_4way_coadd_transmap.fits'),mask.shape,mask.wcs) * mask
lmap = mask.lmap()
lymap,lxmap = lmap
# def model(x,width,amplitude,sigma):
# mmap = 1-amplitude * np.exp(-lymap**2./2./sigma**2.)
# mmap[lxmap>width/2.] = 1
# mmap[lxmap<-width/2.] = 1
# return mmap
def model(x,width,amplitude,sigma):
mmap = (1-amplitude * np.exp(-lymap**2./2./sigma**2.))* (1-np.exp(-lxmap**2./2./width**2.))
def compute_map(self,oshape,owcs,qid,pixwin_taper_deg=0.3,pixwin_pad_deg=0.3,
include_cmb=True,include_tsz=True,include_fgres=True,sht_beam=True):
"""
1. get total alm
2. apply beam, and pixel window if Planck
3. ISHT
4. if ACT, apply a small taper and apply pixel window in Fourier space
"""
# pad to a slightly larger geometry
tot_pad_deg = pixwin_taper_deg + pixwin_pad_deg
res = maps.resolution(oshape,owcs)
pix = np.deg2rad(tot_pad_deg)/res
omap = enmap.pad(enmap.zeros((3,)+oshape,owcs),pix)
ishape,iwcs = omap.shape[-2:],omap.wcs
# get data model
dm = sints.models[sints.arrays(qid,'data_model')](region_shape=ishape,region_wcs=iwcs,calibrated=True)
# 1. get total alm
array_index = self.qids.index(qid)
tot_alm = int(include_cmb)*self.alms['cmb']
if include_tsz:
try:
assert self.tsz_fnu.ndim==2
tot_alm[0] = tot_alm[0] + hp.almxfl(self.alms['comptony'][0] ,self.tsz_fnu[array_index])
except:
tot_alm[0] = tot_alm[0] + self.alms['comptony'][0] * self.tsz_fnu[array_index]
if self.cfgres is not None: tot_alm[0] = tot_alm[0] + int(include_fgres)*self.alms['fgres'][array_index]
assert tot_alm.ndim==2
assert tot_alm.shape[0]==3
ells = np.arange(self.lmax+1)
# 2. get beam, and pixel window for Planck
if sht_beam:
beam = tutils.get_kbeam(qid,ells,sanitize=False,planck_pixwin=False) # NEVER SANITIZE THE BEAM IN A SIMULATION!!!
for i in range(3): tot_alm[i] = hp.almxfl(tot_alm[i],beam)
if dm.name=='planck_hybrid':
pixwint,pixwinp = hp.pixwin(nside=tutils.get_nside(qid),lmax=self.lmax,pol=True)
tot_alm[0] = hp.almxfl(tot_alm[0],pixwint)
tot_alm[1] = hp.almxfl(tot_alm[1],pixwinp)
tot_alm[2] = hp.almxfl(tot_alm[2],pixwinp)
# 3. ISHT
omap = curvedsky.alm2map(np.complex128(tot_alm),omap,spin=[0,2])
assert omap.ndim==3
assert omap.shape[0]==3
if not(sht_beam):
taper,_ = maps.get_taper_deg(ishape,iwcs,taper_width_degrees=pixwin_taper_deg,pad_width_degrees=pixwin_pad_deg)
modlmap = omap.modlmap()
beam = tutils.get_kbeam(qid,modlmap,sanitize=False,planck_pixwin=True)
kmap = enmap.fft(omap*taper,normalize='phys')
kmap = kmap * beam
# 4. if ACT, apply a small taper and apply pixel window in Fourier space
if dm.name=='act_mr3':
if sht_beam: taper,_ = maps.get_taper_deg(ishape,iwcs,taper_width_degrees=pixwin_taper_deg,pad_width_degrees=pixwin_pad_deg)
pwin = tutils.get_pixwin(ishape[-2:])
if sht_beam:
omap = maps.filter_map(omap*taper,pwin)
else:
kmap = kmap * pwin
if not(sht_beam): omap = enmap.ifft(kmap,normalize='phys').real
return enmap.extract(omap,(3,)+oshape[-2:],owcs)
's15_boss_pa3_f090_nohwp_night_3pass_4way_coadd_hits.fits',
's15_boss_pa3_f150_nohwp_night_3pass_4way_coadd_hits.fits',
]
# a sample sim just to extract the shape and wcs. This is because
# the data map has a different size as the sims.
sample_sim_deep56 = enmap.read_map(
'/global/cscratch1/sd/yguan/sims/v0.3/arrays/deep56_s15_pa1_set0_id0.fits')
sample_sim_boss = enmap.read_map(
'/global/cscratch1/sd/yguan/sims/v0.3/arrays/boss_s15_pa1_set0_id0.fits')
# get Nhits map
first = True
for f in deep56_files:
filename = op.join(input_dir, f)
imap = enmap.extract(enmap.read_map(filename), sample_sim_deep56.shape,
sample_sim_deep56.wcs)
if first:
omap = imap.copy()
first = False
else:
omap += imap
hits_map = omap
# seed the random number
np.random.seed(10)
# generate alpha with standard gaussian
alpha_0 = np.random.randn(*omap.shape) * alpha_error
alpha_map_0 = enmap.samewcs(alpha_0, omap)
alpha_map = alpha_map_0 * hits_map**-0.5
alpha_map[~np.isfinite(alpha_map)] = 0
# print(optile[px[0]-1,px[1]])
# print(modlmap[px[0],px[1]])
# print(modlmap[px[0]-1,px[1]])
# # pftile = ptile
# # pftile[modlmap>300] = 0
# # print(np.sort(pftile[pftile>0]))
# # print(modlmap[np.isclose(ptile,1.52256073e+02)])
# # io.plot_img(np.log10(np.fft.fftshift(ptile)),os.environ['WORK']+"/tiling/ptile_%d_smap" % i)
# # io.hplot(enmap.enmap(np.log10(np.fft.fftshift(ptile)),ewcs),os.environ['WORK']+"/tiling/phtile_%d_smap" % i)
smap = enmap.ifft(kbeam * enmap.enmap(ksol, ewcs),
normalize='phys').real
if solution == 'CMB':
io.hplot(smap, os.environ['WORK'] + "/tiling/tile_%d_smap" % i)
io.power_crop(np.real(ksol * ksol.conj()), 100,
os.environ['WORK'] + "/tiling/ptile_%d.png" % i)
# sys.exit()
ta.update_output(solution, smap, inserter)
#ta.update_output("processed",c*civar,inserter)
#ta.update_output("processed_ivar",civar,inserter)
#pmap = ilc.do_ilc
#ta.update_output("processed",pmap,inserter)
print("Rank %d done" % comm.rank)
for solution in solutions:
pmap = ta.get_final_output(solution)
if comm.rank == 0:
io.hplot(pmap, os.environ['WORK'] + "/tiling/map_%s" % solution)
mask = sints.get_act_mr3_crosslinked_mask("deep56")
io.hplot(
enmap.extract(pmap, mask.shape, mask.wcs) * mask,
os.environ['WORK'] + "/tiling/mmap_%s" % solution)
def preprocess_fourier(
target_enmap, shape=None, wcs=None,
unpixwin=True,
kx_cut=90, ky_cut=50, apo_width=40,
legacy_steve=False):
"""Correct an enmap for the pixel window and filter in Fourier space.
This function performs the typical map preprocessing that occurs in
ACT map analysis. A CAR map has an anisotropic pixel window, which is
easily corrected in Fourier space. It is convenient to combine this step
with any filtering of Fourier modes that made be necessary while you have
the Fourier-transformed maps, such as removing horizontal noise modes.
If you specify map geometry information by passing `shape` and `wcs`,
the map will be extracted to that geometry *before* applying the Fourier
transform. In order to avoid ringing at the map edges, it's useful to
generate a rectangular taper, which is done in this function according to
this specified shape and WCS information. If these are not specified, it
is assumed that the full map will be used as the region for power spectrum
calculation, and the taper will be constructed at the edges of the full
map inwards. The taper width is specified by `apo_width`.
Parameters
----------
target_enmap : enmap
The enmap to be modified
shape : [tuple of ints], optional
extraction map shape, by default None
wcs : astropy.wcs, optional
extraction map WCS information, by default None
unpixwin : bool, optional
Correct for the CAR pixel window if True, by default True
kx_cut : int, optional
We cut modes with wavenumber: math: `| k_x | < k_x ^ {\mathrm{cut}}`,
by default 90
ky_cut : int, optional
We cut modes with wavenumber: math: `| k_y | < k_y ^ {\mathrm{cut}}`,
by default 50
apo_width : int, optional
width of the rectangular taper for the apodized mask, by default 40
legacy_steve : bool, optional
Use a slightly different filter if True, to reproduce Steve's pipeline.
Steve's k-space filter as of June 2019 had a bug where two k-space
modes (the most positive cut mode in x and most negative cut mode in y)
were not cut. This has a very small effect on the spectrum, but for
reproducibility purposes we offer this behavior. By default False
Returns
-------
enmap
the final processed map
"""
# get the shape and WCS from map if we didn't get it
# this means the `extract` call later will just make a copy if
# no shape/wcs are passed.
if shape is None or wcs is None:
shape, wcs = target_enmap.shape, target_enmap.wcs
# construct a tapered edge (an apodized rectangle) to avoid ringing
apo = rectangular_apodization(shape, wcs, 40)
maps = target_enmap.copy()
if legacy_steve:
# generate an apodization in the region specified
legacy_steve_shift(maps)
if maps.ndim == 2:
maps = np.expand_dims(maps, axis=0)
maps = enmap.extract(maps, shape, wcs)
# perform the pixel window and filter
ncomp = maps.shape[0] # loop over components
for i in range(ncomp):
result = kfilter_map(
maps[i,:,:], apo, kx_cut, ky_cut,
unpixwin, legacy_steve)
maps[i,:,:] = result
if target_enmap.ndim == 2:
return maps[0,:,:]
else:
return maps
def get_maps(self, rot_angle1, rot_angle2, compts=None, use_sht=True, ret_alm=True, transfer=None,
load_processed=False, save_processed=False, flux_cut=None):
if compts is None: compts = self.compts
shape, wcs = self.geometry
nshape = (len(compts),) + shape[-2:]
ret = enmap.zeros(nshape, wcs)
if load_processed and not ret_alm:
for i, compt_idx in enumerate(compts):
input_file = self.get_fits_path(self.processed_dir, rot_angle1, rot_angle2, compt_idx)
print("loading", input_file)
temp = enmap.read_map(input_file)
ret[i, ...] = enmap.extract(temp, shape, wcs).copy()
del temp
return ret
else:
for i, compt_idx in enumerate(compts):
if "pts" not in compt_idx:
input_file = self.get_fits_path(self.input_dir, rot_angle1, rot_angle2, compt_idx)
print("loading", input_file)
alm = np.complex128(hp.read_alm(input_file, hdu=(1)))
ret[i, ...] = curvedsky.alm2map(alm, enmap.zeros(nshape[1:], wcs))
else:
input_file = self.get_fits_path(self.input_dir, rot_angle1, rot_angle2, compt_idx,
fits_type="enmap")
print("loading", input_file)
temp = enmap.read_map(input_file)
ret[i, ...] = enmap.extract(temp, shape, wcs).copy()
del temp
alms = None
if transfer is not None:
l, f = transfer
interp_func = scipy.interpolate.interp1d(l, f, bounds_error=False, fill_value=0.)
if use_sht:
l_intp = np.arange(self.lmax + 1)
f_int = interp_func(l_intp)
alms = curvedsky.map2alm(ret, lmax=self.lmax, spin=0)
for i in range(len(compts)):
alms[i] = hp.almxfl(alms[i], f_int)
ret = curvedsky.alm2map(alms, ret, spin=0)
else:
ftmap = enmap.fft(ret)
f_int = interp_func(enmap.modlmap(shape, wcs).ravel())
ftmap = ftmap * np.reshape(f_int, (shape[-2:]))
ret = enmap.ifft(ftmap).real;
del ftmap
if save_processed:
raise NotImplemented()
if flux_cut is not None:
flux_map = flux_cut / enmap.pixsizemap(shape, wcs)
flux_map *= 1e-3 * jysr2thermo(148)
for i, compt_idx in enumerate(compts):
if "pts" not in compt_idx: continue
loc = np.where(ret[i] > flux_map)
ret[i][loc] = 0.
del flux_map
if ret_alm and alms is None:
alms = curvedsky.map2alm(ret, lmax=self.lmax, spin=0)
return ret if not ret_alm else (ret, alms)
def get_maps(self, rot_angle1, rot_angle2, compts=None, use_sht=True, ret_alm=True, transfer=None,
load_processed=False, save_processed=False, flux_cut=None):
if compts is None: compts = self.compts
shape, wcs = self.geometry
nshape = (len(compts),) + shape[-2:]
ret = enmap.zeros(nshape, wcs)
if load_processed and not ret_alm:
for i, compt_idx in enumerate(compts):
input_file = self.get_fits_path(self.processed_dir, rot_angle1, rot_angle2, compt_idx)
print("loading", input_file)
temp = enmap.read_map(input_file)
ret[i, ...] = enmap.extract(temp, shape, wcs).copy()
del temp
return ret
else:
for i, compt_idx in enumerate(compts):
input_file = self.get_fits_path(self.input_dir, rot_angle1, rot_angle2, compt_idx)
print("loading", input_file)
alm = np.complex128(hp.read_alm(input_file, hdu=(1)))
ret[i, ...] = curvedsky.alm2map(alm, enmap.zeros(nshape[1:], wcs))
del alm
if compt_idx in self.highflux_cats:
print("adding high flux cats")
hiflux_cat = np.load(self.get_highflux_cat_path(compt_idx))
hiflux_cat[:, :2] = car2hp_coords(hiflux_cat[:, :2])
mat_rot, _, _ = hp.rotator.get_rotation_matrix(
(rot_angle1 * utils.degree * -1, rot_angle2 * utils.degree, 0))
uvec = hp.ang2vec(hiflux_cat[:, 0], hiflux_cat[:, 1])
rot_vec = np.inner(mat_rot, uvec).T
temppos = hp.vec2ang(rot_vec)
rot_pos = np.zeros(hiflux_cat[:, :2].shape)
rot_pos[:, 0] = temppos[0]
rot_pos[:, 1] = temppos[1]
rot_pos = hp2car_coords(rot_pos)
del temppos
rot_pix = np.round(enmap.sky2pix(nshape[-2:], wcs, rot_pos.T).T).astype(np.int)
loc = np.where((rot_pix[:, 0] >= 0) & (rot_pix[:, 0] < nshape[-2]) & (rot_pix[:, 1] >= 0.) & (
rot_pix[:, 1] < nshape[-1]))
hiflux_cat = hiflux_cat[loc[0], 2]
rot_pix = rot_pix[loc[0], :]
hiflux_map = enmap.zeros(nshape[-2:], wcs)
hiflux_map[rot_pix[:, 0], rot_pix[:, 1]] = hiflux_cat
if flux_cut is not None:
tmin = flux_cut * 1e-3 * jysr2thermo(148)
loc = np.where(hiflux_map > tmin)
hiflux_map[loc] = 0
hiflux_map = hiflux_map / enmap.pixsizemap(shape, wcs)
ret[i, ...] = ret[i, ...] + hiflux_map
del hiflux_map
alms = None
if transfer is not None:
l, f = transfer
interp_func = scipy.interpolate.interp1d(l, f, bounds_error=False, fill_value=0.)
if use_sht:
l_intp = np.arange(self.lmax + 1)
f_int = interp_func(l_intp)
alms = curvedsky.map2alm(ret, lmax=self.lmax, spin=0)
for i in range(len(compts)):
alms[i] = hp.almxfl(alms[i], f_int)
ret = curvedsky.alm2map(alms, ret, spin=0)
else:
ftmap = enmap.fft(ret)
f_int = interp_func(enmap.modlmap(shape, wcs).ravel())
ftmap = ftmap * np.reshape(f_int, (shape[-2:]))
ret = enmap.ifft(ftmap).real;
del ftmap
if save_processed:
raise NotImplemented()
if ret_alm and alms is None:
alms = curvedsky.map2alm(ret, lmax=self.lmax, spin=0)
return ret if not ret_alm else (ret, alms)
