def test_homogeneous_noise():
seed = 1234
tube = "ST3"
simulator = mapsims.SONoiseSimulator(nside=nside, homogeneous=False)
output_map = simulator.simulate(tube, seed=seed)
assert hp.mask_bad(output_map).sum() > 100
simulator = mapsims.SONoiseSimulator(nside=nside, homogeneous=True)
output_map = simulator.simulate(tube, seed=seed)
assert hp.mask_bad(output_map).sum() == 0
assert output_map[0][0][1].std() < 4
def __getMaskedPixels(self, pixels):
pixels_masked = healpy.ma(pixels)
if (self.Configuration.isConfigured(ChartHealpixOptions.MASK)):
pixels_masked.mask = healpy.mask_bad(pixels, badval = self.__getMaskedValue())
return pixels_masked.filled(fill_value = numpy.nan)
def ivar_hp_to_cyl(hmap, shape, wcs, rot=False,do_mask=True,extensive=True):
from . import mpi, utils
import healpy as hp
comm = mpi.COMM_WORLD
rstep = 100
dtype = np.float32
nside = hp.npix2nside(hmap.size)
dec, ra = enmap.posaxes(shape, wcs)
pix = np.zeros(shape, np.int32)
# Get the pixel area. We assume a rectangular pixelization, so this is just
# a function of y
ipixsize = 4 * np.pi / (12 * nside ** 2)
opixsize = get_pixsize_rect(shape, wcs)
nblock = (shape[-2] + rstep - 1) // rstep
for bi in range(comm.rank, nblock, comm.size):
if bi % comm.size != comm.rank:
continue
i = bi * rstep
rdec = dec[i : i + rstep]
opos = np.zeros((2, len(rdec), len(ra)))
opos[0] = rdec[:, None]
opos[1] = ra[None, :]
if rot:
# This is unreasonably slow
ipos = coordinates.transform("equ", "gal", opos[::-1], pol=True)
else:
ipos = opos[::-1]
pix[i : i + rstep, :] = hp.ang2pix(nside, np.pi / 2 - ipos[1], ipos[0])
del ipos, opos
for i in range(0, shape[-2], rstep):
pix[i : i + rstep] = utils.allreduce(pix[i : i + rstep], comm)
omap = enmap.zeros((1,) + shape, wcs, dtype)
imap = np.array(hmap).astype(dtype)
imap = imap[None]
if do_mask:
bad = hp.mask_bad(imap)
bad |= imap <= 0
imap[bad] = 0
del bad
# Read off the nearest neighbor values
omap[:] = imap[:, pix]
if extensive: omap *= opixsize[:, None] / ipixsize
# We ignore QU mixing during rotation for the noise level, so
# it makes no sense to maintain distinct levels for them
if do_mask:
mask = omap[1:] > 0
omap[1:] = np.mean(omap[1:], 0)
omap[1:] *= mask
del mask
return omap
runit *= factor_857
npol = 1
elif ("smica" in name) or ("WPR2" in name):
npol = 1
elif ("COM_Mask_Lensing" in name):
npol = 1
runit = 1
else:
npol = 3
fields = range(npol)
omap = enmap.zeros((npol,)+shape, wcs, dtype)
progress("%s TQU read" % name)
imap = np.array(healpy.read_map(ifile, fields)).astype(dtype)
nside = healpy.npix2nside(imap.shape[-1])
progress("%s TQU mask" % name)
imap[healpy.mask_bad(imap)] = 0
progress("%s TQU scale" % name)
imap *= runit
nside = healpy.npix2nside(imap.shape[-1])
lmax = args.lmax or 3*nside
progress("%s TQU alm2map" % name)
alm = curvedsky.map2alm_healpix(imap, lmax=lmax)
del imap
# work around healpix bug
progress("%s TQU rotate_alm" % name)
alm = alm.astype(np.complex128,copy=False)
healpy.rotate_alm(alm, euler[0], euler[1], euler[2])
alm = alm.astype(ctype,copy=False)
progress("%s TQU map2alm" % name)
curvedsky.alm2map_cyl(alm, omap)
del alm
# disk_plot(vec_l[i],dopp_vec_nilc[i],0.1,nside)
for i in range(vec_l.size):
disk_plot(vec_l[i], abdopp_vec_nilc[i], 0.1, nside)
#for i in range(5):
# disk_plot(vec_l[i],ab_vec_smica[i],0.1,nside)
#for i in range(5):
# disk_plot(vec_l[i],dopp_vec_smica[i],0.1,nside)
for i in range(vec_l.size):
disk_plot(vec_l[i], abdopp_vec_smica[i], 0.1, nside)
m[m == 0] = hp.UNSEEN
hp.mask_bad(m)
hp.mollview(m,
badcolor='white',
unit=r'$\ell_{max}$',
cbar=None,
cmap=cmap_test,
xsize=2048,
title="")
hp.graticule()
########################################
########################################
phi = np.linspace(0, 2. * np.pi, 180)
#
#aberror=np.array([laterror_smica[0,0],longerror_smica[0,0]]);
#dopperror=np.array([laterror_smica[1,1],longerror_smica[1,1]]);
def test_binned(self):
rank = 0
if self.comm is not None:
rank = self.comm.rank
# flag data outside valid intervals
gapflagger = OpFlagGaps()
gapflagger.exec(self.data)
# generate noise timestreams from the noise model
nsig = OpSimNoise()
nsig.exec(self.data)
# make a simple pointing matrix
pointing = OpPointingHpix(nside=self.map_nside, nest=True, mode="IQU")
pointing.exec(self.data)
handle = None
if rank == 0:
handle = open(os.path.join(self.outdir, "info.txt"), "w")
self.data.info(handle=handle)
if rank == 0:
handle.close()
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = DistPixels(self.data,
comm=self.data.comm.comm_world,
nnz=6,
dtype=np.float64)
invnpp.data.fill(0.0)
zmap = DistPixels(self.data,
comm=self.data.comm.comm_world,
nnz=3,
dtype=np.float64)
zmap.data.fill(0.0)
hits = DistPixels(self.data,
comm=self.data.comm.comm_world,
nnz=1,
dtype=np.int64)
hits.data.fill(0)
# accumulate the inverse covariance and noise weighted map.
# Use detector weights based on the analytic NET.
tod = self.data.obs[0]["tod"]
nse = self.data.obs[0]["noise"]
detweights = {}
for d in tod.local_dets:
detweights[d] = 1.0 / (self.rate * nse.NET(d)**2)
build_invnpp = OpAccumDiag(detweights=detweights,
invnpp=invnpp,
hits=hits,
zmap=zmap,
name="noise")
build_invnpp.exec(self.data)
invnpp.allreduce()
hits.allreduce()
zmap.allreduce()
hits.write_healpix_fits(os.path.join(self.outdir, "hits.fits"))
invnpp.write_healpix_fits(os.path.join(self.outdir, "invnpp.fits"))
zmap.write_healpix_fits(os.path.join(self.outdir, "zmap.fits"))
# invert it
covariance_invert(invnpp, 1.0e-3)
invnpp.write_healpix_fits(os.path.join(self.outdir, "npp.fits"))
# compute the binned map, N_pp x Z
covariance_apply(invnpp, zmap)
zmap.write_healpix_fits(os.path.join(self.outdir, "binned.fits"))
# compare with MADAM
madam_out = os.path.join(self.outdir, "madam")
if rank == 0:
if os.path.isdir(madam_out):
shutil.rmtree(madam_out)
os.mkdir(madam_out)
pars = {}
pars["temperature_only"] = "F"
pars["force_pol"] = "T"
pars["kfirst"] = "F"
pars["base_first"] = 1.0
pars["fsample"] = self.rate
pars["nside_map"] = self.map_nside
pars["nside_cross"] = self.map_nside
pars["nside_submap"] = self.map_nside
pars["pixlim_cross"] = 1.0e-2
pars["pixlim_map"] = 1.0e-3
pars["write_map"] = "F"
pars["write_binmap"] = "T"
pars["write_matrix"] = "T"
pars["write_wcov"] = "T"
pars["write_hits"] = "T"
pars["kfilter"] = "F"
pars["path_output"] = madam_out
pars["info"] = 0
madam = OpMadam(params=pars, detweights=detweights, name="noise")
if madam.available:
madam.exec(self.data)
if rank == 0:
import matplotlib.pyplot as plt
hitsfile = os.path.join(madam_out, "madam_hmap.fits")
hits = hp.read_map(hitsfile, nest=True, verbose=False)
outfile = "{}.png".format(hitsfile)
hp.mollview(hits, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
toastfile = os.path.join(self.outdir, "hits.fits")
toasthits = hp.read_map(toastfile, nest=True, verbose=False)
nt.assert_equal(hits, toasthits)
tothits = np.sum(hits)
nt.assert_equal(self.validsamp, tothits)
covfile = os.path.join(madam_out, "madam_wcov_inv.fits")
cov = hp.read_map(covfile,
nest=True,
field=None,
verbose=False)
toastfile = os.path.join(self.outdir, "invnpp.fits")
toastcov = hp.read_map(toastfile,
nest=True,
field=None,
verbose=False)
"""
for p in range(6):
print("elem {} madam min/max = ".format(p),
np.min(cov[p]), " / ", np.max(cov[p]))
print("elem {} toast min/max = ".format(p),
np.min(toastcov[p]), " / ", np.max(toastcov[p]))
print("elem {} invNpp max diff = ".format(p),
np.max(np.absolute(toastcov[p] - cov[p])))
nt.assert_almost_equal(cov[p], toastcov[p])
"""
covfile = os.path.join(madam_out, "madam_wcov.fits")
cov = hp.read_map(covfile,
nest=True,
field=None,
verbose=False)
toastfile = os.path.join(self.outdir, "npp.fits")
toastcov = hp.read_map(toastfile,
nest=True,
field=None,
verbose=False)
"""
for p in range(6):
covdiff = toastcov[p] - cov[p]
print("elem {} madam min/max = ".format(p),
np.min(cov[p]), " / ", np.max(cov[p]))
print("elem {} toast min/max = ".format(p),
np.min(toastcov[p]), " / ", np.max(toastcov[p]))
print("elem {} Npp max diff = ".format(p),
np.max(np.absolute(covdiff[p])))
print("elem {} Npp mean / rms diff = ".format(p),
np.mean(covdiff[p]), " / ", np.std(covdiff[p]))
print("elem {} Npp relative diff mean / rms = ".format(p),
np.mean(np.absolute(covdiff[p]/cov[p])), " / ",
np.std(np.absolute(covdiff[p]/cov[p])))
nt.assert_almost_equal(cov[p], toastcov[p])
"""
binfile = os.path.join(madam_out, "madam_bmap.fits")
bins = hp.read_map(binfile,
nest=True,
field=None,
verbose=False)
mask = hp.mask_bad(bins[0])
bins[0][mask] = 0.0
mask = hp.mask_bad(bins[1])
bins[1][mask] = 0.0
mask = hp.mask_bad(bins[2])
bins[2][mask] = 0.0
outfile = "{}_I.png".format(binfile)
hp.mollview(bins[0], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_Q.png".format(binfile)
hp.mollview(bins[1], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_U.png".format(binfile)
hp.mollview(bins[2], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
toastfile = os.path.join(self.outdir, "binned.fits")
toastbins = hp.read_map(toastfile,
nest=True,
field=None,
verbose=False)
outfile = "{}_I.png".format(toastfile)
hp.mollview(toastbins[0], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_Q.png".format(toastfile)
hp.mollview(toastbins[1], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_U.png".format(toastfile)
hp.mollview(toastbins[2], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
# compare binned map to madam output
diffmap = toastbins[0] - bins[0]
mask = bins[0] != 0
"""
print("toast/madam I diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("toast/madam I diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[0][mask]), " / ",
np.max(diffmap[mask]/bins[0][mask]))
"""
outfile = "{}_diff_madam_I.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
# nt.assert_almost_equal(bins[0][mask], binserial[0][mask],
# decimal=6)
diffmap = toastbins[1] - bins[1]
mask = bins[1] != 0
"""
print("toast/madam Q diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("toast/madam Q diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[1][mask]), " / ",
np.max(diffmap[mask]/bins[1][mask]))
"""
outfile = "{}_diff_madam_Q.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
# nt.assert_almost_equal(bins[1][mask], binserial[1][mask],
# decimal=6)
diffmap = toastbins[2] - bins[2]
mask = bins[2] != 0
"""
print("toast/madam U diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("toast/madam U diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[2][mask]), " / ",
np.max(diffmap[mask]/bins[2][mask]))
"""
outfile = "{}_diff_madam_U.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
# nt.assert_almost_equal(bins[2][mask], binserial[2][mask],
# decimal=6)
# compute the binned map serially as a check
zfile = os.path.join(self.outdir, "zmap.fits")
ztoast = hp.read_map(zfile,
nest=True,
field=None,
verbose=False)
binserial = np.copy(ztoast)
for p in range(self.map_npix):
binserial[0][p] = (toastcov[0][p] * ztoast[0][p] +
toastcov[1][p] * ztoast[1][p] +
toastcov[2][p] * ztoast[2][p])
binserial[1][p] = (toastcov[1][p] * ztoast[0][p] +
toastcov[3][p] * ztoast[1][p] +
toastcov[4][p] * ztoast[2][p])
binserial[2][p] = (toastcov[2][p] * ztoast[0][p] +
toastcov[4][p] * ztoast[1][p] +
toastcov[5][p] * ztoast[2][p])
toastfile = os.path.join(self.outdir, "binned_serial")
outfile = "{}_I.png".format(toastfile)
hp.mollview(binserial[0], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_Q.png".format(toastfile)
hp.mollview(binserial[1], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_U.png".format(toastfile)
hp.mollview(binserial[2], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
# compare binned map to madam output
diffmap = binserial[0] - bins[0]
mask = bins[0] != 0
"""
print("serial/madam I diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("serial/madam I diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[0][mask]), " / ",
np.max(diffmap[mask]/bins[0][mask]))
"""
outfile = "{}_diff_madam_I.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
nt.assert_almost_equal(bins[0][mask],
binserial[0][mask],
decimal=3)
diffmap = binserial[1] - bins[1]
mask = bins[1] != 0
"""
print("serial/madam Q diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("serial/madam Q diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[1][mask]), " / ",
np.max(diffmap[mask]/bins[1][mask]))
"""
outfile = "{}_diff_madam_Q.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
nt.assert_almost_equal(bins[1][mask],
binserial[1][mask],
decimal=3)
diffmap = binserial[2] - bins[2]
mask = bins[2] != 0
"""
print("serial/madam U diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("serial/madam U diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[2][mask]), " / ",
np.max(diffmap[mask]/bins[2][mask]))
"""
outfile = "{}_diff_madam_U.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
nt.assert_almost_equal(bins[2][mask],
binserial[2][mask],
decimal=3)
else:
if rank == 0:
print("libmadam not available, skipping tests", flush=True)
return
def test_binned(self):
# flag data outside valid intervals
gapflagger = OpFlagGaps()
gapflagger.exec(self.data)
# generate noise timestreams from the noise model
nsig = OpSimNoise()
nsig.exec(self.data)
# make a simple pointing matrix
pointing = OpPointingHpix(
nside=self.map_nside, nest=True, mode="IQU", hwprpm=self.hwprpm
)
pointing.exec(self.data)
handle = None
if self.comm.rank == 0:
handle = open(os.path.join(self.outdir, "info.txt"), "w")
self.data.info(handle)
if self.comm.rank == 0:
handle.close()
# get locally hit pixels
lc = OpLocalPixels()
localpix = lc.exec(self.data)
# find the locally hit submaps.
localsm = np.unique(np.floor_divide(localpix, self.subnpix))
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = DistPixels(
comm=self.data.comm.comm_world,
size=self.sim_npix,
nnz=6,
dtype=np.float64,
submap=self.subnpix,
local=localsm,
)
invnpp.data.fill(0.0)
zmap = DistPixels(
comm=self.data.comm.comm_world,
size=self.sim_npix,
nnz=3,
dtype=np.float64,
submap=self.subnpix,
local=localsm,
)
zmap.data.fill(0.0)
hits = DistPixels(
comm=self.data.comm.comm_world,
size=self.sim_npix,
nnz=1,
dtype=np.int64,
submap=self.subnpix,
local=localsm,
)
hits.data.fill(0)
# accumulate the inverse covariance and noise weighted map.
# Use detector weights based on the analytic NET.
tod = self.data.obs[0]["tod"]
nse = self.data.obs[0]["noise"]
detweights = {}
for d in tod.local_dets:
detweights[d] = 1.0 / (self.rate * nse.NET(d) ** 2)
build_invnpp = OpAccumDiag(
detweights=detweights, invnpp=invnpp, hits=hits, zmap=zmap, name="noise"
)
build_invnpp.exec(self.data)
invnpp.allreduce()
hits.allreduce()
zmap.allreduce()
hits.write_healpix_fits(os.path.join(self.outdir, "hits.fits"))
invnpp.write_healpix_fits(os.path.join(self.outdir, "invnpp.fits"))
zmap.write_healpix_fits(os.path.join(self.outdir, "zmap.fits"))
# invert it
covariance_invert(invnpp, 1.0e-3)
invnpp.write_healpix_fits(os.path.join(self.outdir, "npp.fits"))
# compute the binned map, N_pp x Z
covariance_apply(invnpp, zmap)
zmap.write_healpix_fits(os.path.join(self.outdir, "binned.fits"))
# compare with MADAM
madam_out = os.path.join(self.outdir, "madam")
if self.comm.rank == 0:
if os.path.isdir(madam_out):
shutil.rmtree(madam_out)
os.mkdir(madam_out)
pars = {}
pars["temperature_only"] = "F"
pars["force_pol"] = "T"
pars["kfirst"] = "F"
pars["base_first"] = 1.0
pars["fsample"] = self.rate
pars["nside_map"] = self.map_nside
pars["nside_cross"] = self.map_nside
pars["nside_submap"] = self.map_nside
pars["pixlim_cross"] = 1.0e-2
pars["pixlim_map"] = 1.0e-3
pars["write_map"] = "F"
pars["write_binmap"] = "T"
pars["write_matrix"] = "T"
pars["write_wcov"] = "T"
pars["write_hits"] = "T"
pars["kfilter"] = "F"
pars["path_output"] = madam_out
pars["info"] = 0
madam = OpMadam(params=pars, detweights=detweights, name="noise")
if madam.available:
madam.exec(self.data)
if self.comm.rank == 0:
import matplotlib.pyplot as plt
hitsfile = os.path.join(madam_out, "madam_hmap.fits")
hits = hp.read_map(hitsfile, nest=True)
outfile = "{}.png".format(hitsfile)
hp.mollview(hits, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
toastfile = os.path.join(self.outdir, "hits.fits")
toasthits = hp.read_map(toastfile, nest=True)
nt.assert_equal(hits, toasthits)
tothits = np.sum(hits)
nt.assert_equal(self.validsamp, tothits)
covfile = os.path.join(madam_out, "madam_wcov_inv.fits")
cov = hp.read_map(covfile, nest=True, field=None)
toastfile = os.path.join(self.outdir, "invnpp.fits")
toastcov = hp.read_map(toastfile, nest=True, field=None)
"""
for p in range(6):
print("elem {} madam min/max = ".format(p),
np.min(cov[p]), " / ", np.max(cov[p]))
print("elem {} toast min/max = ".format(p),
np.min(toastcov[p]), " / ", np.max(toastcov[p]))
print("elem {} invNpp max diff = ".format(p),
np.max(np.absolute(toastcov[p] - cov[p])))
nt.assert_almost_equal(cov[p], toastcov[p])
"""
covfile = os.path.join(madam_out, "madam_wcov.fits")
cov = hp.read_map(covfile, nest=True, field=None)
toastfile = os.path.join(self.outdir, "npp.fits")
toastcov = hp.read_map(toastfile, nest=True, field=None)
"""
for p in range(6):
covdiff = toastcov[p] - cov[p]
print("elem {} madam min/max = ".format(p),
np.min(cov[p]), " / ", np.max(cov[p]))
print("elem {} toast min/max = ".format(p),
np.min(toastcov[p]), " / ", np.max(toastcov[p]))
print("elem {} Npp max diff = ".format(p),
np.max(np.absolute(covdiff[p])))
print("elem {} Npp mean / rms diff = ".format(p),
np.mean(covdiff[p]), " / ", np.std(covdiff[p]))
print("elem {} Npp relative diff mean / rms = ".format(p),
np.mean(np.absolute(covdiff[p]/cov[p])), " / ",
np.std(np.absolute(covdiff[p]/cov[p])))
nt.assert_almost_equal(cov[p], toastcov[p])
"""
binfile = os.path.join(madam_out, "madam_bmap.fits")
bins = hp.read_map(binfile, nest=True, field=None)
mask = hp.mask_bad(bins[0])
bins[0][mask] = 0.0
mask = hp.mask_bad(bins[1])
bins[1][mask] = 0.0
mask = hp.mask_bad(bins[2])
bins[2][mask] = 0.0
outfile = "{}_I.png".format(binfile)
hp.mollview(bins[0], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_Q.png".format(binfile)
hp.mollview(bins[1], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_U.png".format(binfile)
hp.mollview(bins[2], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
toastfile = os.path.join(self.outdir, "binned.fits")
toastbins = hp.read_map(toastfile, nest=True, field=None)
outfile = "{}_I.png".format(toastfile)
hp.mollview(toastbins[0], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_Q.png".format(toastfile)
hp.mollview(toastbins[1], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_U.png".format(toastfile)
hp.mollview(toastbins[2], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
# compare binned map to madam output
diffmap = toastbins[0] - bins[0]
mask = bins[0] != 0
"""
print("toast/madam I diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("toast/madam I diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[0][mask]), " / ",
np.max(diffmap[mask]/bins[0][mask]))
"""
outfile = "{}_diff_madam_I.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
# nt.assert_almost_equal(bins[0][mask], binserial[0][mask],
# decimal=6)
diffmap = toastbins[1] - bins[1]
mask = bins[1] != 0
"""
print("toast/madam Q diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("toast/madam Q diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[1][mask]), " / ",
np.max(diffmap[mask]/bins[1][mask]))
"""
outfile = "{}_diff_madam_Q.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
# nt.assert_almost_equal(bins[1][mask], binserial[1][mask],
# decimal=6)
diffmap = toastbins[2] - bins[2]
mask = bins[2] != 0
"""
print("toast/madam U diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("toast/madam U diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[2][mask]), " / ",
np.max(diffmap[mask]/bins[2][mask]))
"""
outfile = "{}_diff_madam_U.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
# nt.assert_almost_equal(bins[2][mask], binserial[2][mask],
# decimal=6)
# compute the binned map serially as a check
zfile = os.path.join(self.outdir, "zmap.fits")
ztoast = hp.read_map(zfile, nest=True, field=None)
binserial = np.copy(ztoast)
for p in range(self.map_npix):
binserial[0][p] = (
toastcov[0][p] * ztoast[0][p]
+ toastcov[1][p] * ztoast[1][p]
+ toastcov[2][p] * ztoast[2][p]
)
binserial[1][p] = (
toastcov[1][p] * ztoast[0][p]
+ toastcov[3][p] * ztoast[1][p]
+ toastcov[4][p] * ztoast[2][p]
)
binserial[2][p] = (
toastcov[2][p] * ztoast[0][p]
+ toastcov[4][p] * ztoast[1][p]
+ toastcov[5][p] * ztoast[2][p]
)
toastfile = os.path.join(self.outdir, "binned_serial")
outfile = "{}_I.png".format(toastfile)
hp.mollview(binserial[0], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_Q.png".format(toastfile)
hp.mollview(binserial[1], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_U.png".format(toastfile)
hp.mollview(binserial[2], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
# compare binned map to madam output
diffmap = binserial[0] - bins[0]
mask = bins[0] != 0
"""
print("serial/madam I diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("serial/madam I diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[0][mask]), " / ",
np.max(diffmap[mask]/bins[0][mask]))
"""
outfile = "{}_diff_madam_I.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
nt.assert_almost_equal(bins[0][mask], binserial[0][mask], decimal=3)
diffmap = binserial[1] - bins[1]
mask = bins[1] != 0
"""
print("serial/madam Q diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("serial/madam Q diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[1][mask]), " / ",
np.max(diffmap[mask]/bins[1][mask]))
"""
outfile = "{}_diff_madam_Q.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
nt.assert_almost_equal(bins[1][mask], binserial[1][mask], decimal=3)
diffmap = binserial[2] - bins[2]
mask = bins[2] != 0
"""
print("serial/madam U diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("serial/madam U diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[2][mask]), " / ",
np.max(diffmap[mask]/bins[2][mask]))
"""
outfile = "{}_diff_madam_U.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
nt.assert_almost_equal(bins[2][mask], binserial[2][mask], decimal=3)
else:
if self.comm.rank == 0:
print("libmadam not available, skipping tests", flush=True)
return
hp.gnomview(mapa2r, rot=[40,80,0], reso=8, title='CMB map', unit='K')
pyplot.savefig('gnomonic_view_bad.png')
mapa_save = [mapa2r,mapa[1]]
hp.write_map('CMB_map_Ns1024.fits',mapa_save, coord='G', column_names=list(['I_CMB','Confidence-Mask']), column_units=list(['K',' ']))
#%%
"""
STEP 2: Missing value
"""
m = np.arange(12.)
m[2] = hp.UNSEEN
m_bad = hp.mask_bad(m)
m_good = hp.mask_good(m)
print('m_bad ->', m_bad)
print('m_good ->', m_good)
hp.mollview(m_bad, title='mask_bad')
hp.mollview(m_good, title='mask_good')
#%%
"""
STEP 3: Treating missing pixels
"""
def test_binned(self):
start = MPI.Wtime()
# flag data outside valid intervals
gapflagger = OpFlagGaps()
gapflagger.exec(self.data)
# generate noise timestreams from the noise model
nsig = OpSimNoise()
nsig.exec(self.data)
# make a simple pointing matrix
pointing = OpPointingHpix(nside=self.map_nside,
nest=True,
mode='IQU',
hwprpm=self.hwprpm)
pointing.exec(self.data)
handle = None
if self.comm.rank == 0:
handle = open(os.path.join(self.mapdir, "info.txt"), "w")
self.data.info(handle)
if self.comm.rank == 0:
handle.close()
# get locally hit pixels
lc = OpLocalPixels()
localpix = lc.exec(self.data)
# find the locally hit submaps.
localsm = np.unique(np.floor_divide(localpix, self.subnpix))
# construct distributed maps to store the covariance,
# noise weighted map, and hits
invnpp = DistPixels(comm=self.toastcomm.comm_group,
size=self.sim_npix,
nnz=6,
dtype=np.float64,
submap=self.subnpix,
local=localsm)
invnpp.data.fill(0.0)
zmap = DistPixels(comm=self.toastcomm.comm_group,
size=self.sim_npix,
nnz=3,
dtype=np.float64,
submap=self.subnpix,
local=localsm)
zmap.data.fill(0.0)
hits = DistPixels(comm=self.toastcomm.comm_group,
size=self.sim_npix,
nnz=1,
dtype=np.int64,
submap=self.subnpix,
local=localsm)
hits.data.fill(0)
# accumulate the inverse covariance and noise weighted map.
# Use detector weights based on the analytic NET.
tod = self.data.obs[0]['tod']
nse = self.data.obs[0]['noise']
detweights = {}
for d in tod.local_dets:
detweights[d] = 1.0 / (self.rate * nse.NET(d)**2)
build_invnpp = OpAccumDiag(detweights=detweights,
invnpp=invnpp,
hits=hits,
zmap=zmap,
name="noise")
build_invnpp.exec(self.data)
invnpp.allreduce()
hits.allreduce()
zmap.allreduce()
hits.write_healpix_fits(os.path.join(self.mapdir, "hits.fits"))
invnpp.write_healpix_fits(os.path.join(self.mapdir, "invnpp.fits"))
zmap.write_healpix_fits(os.path.join(self.mapdir, "zmap.fits"))
# invert it
covariance_invert(invnpp, 1.0e-3)
invnpp.write_healpix_fits(os.path.join(self.mapdir, "npp.fits"))
# compute the binned map, N_pp x Z
covariance_apply(invnpp, zmap)
zmap.write_healpix_fits(os.path.join(self.mapdir, "binned.fits"))
# compare with MADAM
madam_out = os.path.join(self.mapdir, "madam")
if self.comm.rank == 0:
if os.path.isdir(madam_out):
shutil.rmtree(madam_out)
os.mkdir(madam_out)
pars = {}
pars['temperature_only'] = 'F'
pars['force_pol'] = 'T'
pars['kfirst'] = 'F'
pars['base_first'] = 1.0
pars['fsample'] = self.rate
pars['nside_map'] = self.map_nside
pars['nside_cross'] = self.map_nside
pars['nside_submap'] = self.map_nside
pars['pixlim_cross'] = 1.0e-2
pars['pixlim_map'] = 1.0e-3
pars['write_map'] = 'F'
pars['write_binmap'] = 'T'
pars['write_matrix'] = 'T'
pars['write_wcov'] = 'T'
pars['write_hits'] = 'T'
pars['kfilter'] = 'F'
pars['path_output'] = madam_out
pars['info'] = 0
madam = OpMadam(params=pars, detweights=detweights, name="noise")
if madam.available:
madam.exec(self.data)
if self.comm.rank == 0:
import matplotlib.pyplot as plt
hitsfile = os.path.join(madam_out, 'madam_hmap.fits')
print('Loading', hitsfile)
hits = hp.read_map(hitsfile, nest=True)
outfile = "{}.png".format(hitsfile)
hp.mollview(hits, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
toastfile = os.path.join(self.mapdir, 'hits.fits')
print('Loading', toastfile)
toasthits = hp.read_map(toastfile, nest=True)
nt.assert_equal(hits, toasthits)
tothits = np.sum(hits)
nt.assert_equal(self.validsamp, tothits)
covfile = os.path.join(madam_out, 'madam_wcov_inv.fits')
cov = hp.read_map(covfile, nest=True, field=None)
toastfile = os.path.join(self.mapdir, 'invnpp.fits')
toastcov = hp.read_map(toastfile, nest=True, field=None)
"""
for p in range(6):
print("elem {} madam min/max = ".format(p),
np.min(cov[p]), " / ", np.max(cov[p]))
print("elem {} toast min/max = ".format(p),
np.min(toastcov[p]), " / ", np.max(toastcov[p]))
print("elem {} invNpp max diff = ".format(p),
np.max(np.absolute(toastcov[p] - cov[p])))
nt.assert_almost_equal(cov[p], toastcov[p])
"""
covfile = os.path.join(madam_out, 'madam_wcov.fits')
cov = hp.read_map(covfile, nest=True, field=None)
toastfile = os.path.join(self.mapdir, 'npp.fits')
toastcov = hp.read_map(toastfile, nest=True, field=None)
"""
for p in range(6):
covdiff = toastcov[p] - cov[p]
print("elem {} madam min/max = ".format(p),
np.min(cov[p]), " / ", np.max(cov[p]))
print("elem {} toast min/max = ".format(p),
np.min(toastcov[p]), " / ", np.max(toastcov[p]))
print("elem {} Npp max diff = ".format(p),
np.max(np.absolute(covdiff[p])))
print("elem {} Npp mean / rms diff = ".format(p),
np.mean(covdiff[p]), " / ", np.std(covdiff[p]))
print("elem {} Npp relative diff mean / rms = ".format(p),
np.mean(np.absolute(covdiff[p]/cov[p])), " / ",
np.std(np.absolute(covdiff[p]/cov[p])))
nt.assert_almost_equal(cov[p], toastcov[p])
"""
binfile = os.path.join(madam_out, 'madam_bmap.fits')
bins = hp.read_map(binfile, nest=True, field=None)
mask = hp.mask_bad(bins[0])
bins[0][mask] = 0.0
mask = hp.mask_bad(bins[1])
bins[1][mask] = 0.0
mask = hp.mask_bad(bins[2])
bins[2][mask] = 0.0
outfile = "{}_I.png".format(binfile)
hp.mollview(bins[0], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_Q.png".format(binfile)
hp.mollview(bins[1], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_U.png".format(binfile)
hp.mollview(bins[2], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
toastfile = os.path.join(self.mapdir, 'binned.fits')
toastbins = hp.read_map(toastfile, nest=True, field=None)
outfile = "{}_I.png".format(toastfile)
hp.mollview(toastbins[0], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_Q.png".format(toastfile)
hp.mollview(toastbins[1], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_U.png".format(toastfile)
hp.mollview(toastbins[2], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
# compare binned map to madam output
diffmap = toastbins[0] - bins[0]
mask = (bins[0] != 0)
"""
print("toast/madam I diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("toast/madam I diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[0][mask]), " / ",
np.max(diffmap[mask]/bins[0][mask]))
"""
outfile = "{}_diff_madam_I.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
#nt.assert_almost_equal(bins[0][mask], binserial[0][mask],
# decimal=6)
diffmap = toastbins[1] - bins[1]
mask = (bins[1] != 0)
"""
print("toast/madam Q diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("toast/madam Q diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[1][mask]), " / ",
np.max(diffmap[mask]/bins[1][mask]))
"""
outfile = "{}_diff_madam_Q.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
#nt.assert_almost_equal(bins[1][mask], binserial[1][mask],
# decimal=6)
diffmap = toastbins[2] - bins[2]
mask = (bins[2] != 0)
"""
print("toast/madam U diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("toast/madam U diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[2][mask]), " / ",
np.max(diffmap[mask]/bins[2][mask]))
"""
outfile = "{}_diff_madam_U.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
#nt.assert_almost_equal(bins[2][mask], binserial[2][mask],
# decimal=6)
# compute the binned map serially as a check
zfile = os.path.join(self.mapdir, 'zmap.fits')
ztoast = hp.read_map(zfile, nest=True, field=None)
binserial = np.copy(ztoast)
for p in range(self.map_npix):
binserial[0][p] = toastcov[0][p] * ztoast[0][p] \
+ toastcov[1][p] * ztoast[1][p] \
+ toastcov[2][p] * ztoast[2][p]
binserial[1][p] = toastcov[1][p] * ztoast[0][p] \
+ toastcov[3][p] * ztoast[1][p] \
+ toastcov[4][p] * ztoast[2][p]
binserial[2][p] = toastcov[2][p] * ztoast[0][p] \
+ toastcov[4][p] * ztoast[1][p] \
+ toastcov[5][p] * ztoast[2][p]
toastfile = os.path.join(self.mapdir, 'binned_serial')
outfile = "{}_I.png".format(toastfile)
hp.mollview(binserial[0], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_Q.png".format(toastfile)
hp.mollview(binserial[1], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
outfile = "{}_U.png".format(toastfile)
hp.mollview(binserial[2], xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
# compare binned map to madam output
diffmap = binserial[0] - bins[0]
mask = (bins[0] != 0)
"""
print("serial/madam I diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("serial/madam I diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[0][mask]), " / ",
np.max(diffmap[mask]/bins[0][mask]))
"""
outfile = "{}_diff_madam_I.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
nt.assert_almost_equal(bins[0][mask],
binserial[0][mask],
decimal=3)
diffmap = binserial[1] - bins[1]
mask = (bins[1] != 0)
"""
print("serial/madam Q diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("serial/madam Q diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[1][mask]), " / ",
np.max(diffmap[mask]/bins[1][mask]))
"""
outfile = "{}_diff_madam_Q.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
nt.assert_almost_equal(bins[1][mask],
binserial[1][mask],
decimal=3)
diffmap = binserial[2] - bins[2]
mask = (bins[2] != 0)
"""
print("serial/madam U diff mean / std = ",
np.mean(diffmap[mask]), np.std(diffmap[mask]))
print("serial/madam U diff rel ratio min / max = ",
np.min(diffmap[mask]/bins[2][mask]), " / ",
np.max(diffmap[mask]/bins[2][mask]))
"""
outfile = "{}_diff_madam_U.png".format(toastfile)
hp.mollview(diffmap, xsize=1600, nest=True)
plt.savefig(outfile)
plt.close()
nt.assert_almost_equal(bins[2][mask],
binserial[2][mask],
decimal=3)
stop = MPI.Wtime()
elapsed = stop - start
self.print_in_turns("Madam test took {:.3f} s".format(elapsed))
else:
print("libmadam not available, skipping tests")
#self.assertTrue(False)
return
def execute(self, write_outputs=False):
"""Run map simulations
Execute simulations for all channels and write to disk the maps,
unless `write_outputs` is False, then return them.
"""
if self.run_pysm:
sky_config = []
preset_strings = []
if self.pysm_components_string is not None:
for model in self.pysm_components_string.split(","):
if model.startswith("SO"):
sky_config.append(get_so_models(model, self.nside))
else:
preset_strings.append(model)
if len(preset_strings) > 0:
input_reference_frame = "G"
assert (
len(sky_config) == 0
), "Cannot mix PySM and SO models, they are defined in G and C frames"
else:
input_reference_frame = "C"
self.pysm_sky = pysm.Sky(
nside=self.nside,
preset_strings=preset_strings,
component_objects=sky_config,
output_unit=u.Unit(self.unit),
)
if self.pysm_custom_components is not None:
for comp_name, comp in self.pysm_custom_components.items():
self.pysm_sky.components.append(comp)
if not write_outputs:
output = {}
# ch can be single channel or tuple of 2 channels (tube dichroic)
for ch in self.channels:
if not isinstance(ch, tuple):
ch = [ch]
output_map_shape = (len(ch), 3) + self.shape
output_map = np.zeros(output_map_shape, dtype=np.float64)
if self.run_pysm:
for each, channel_map in zip(ch, output_map):
bandpass_integrated_map = self.pysm_sky.get_emission(
*each.bandpass).value
beam_width_arcmin = each.beam
# smoothing and coordinate rotation with 1 spherical harmonics transform
smoothed_map = hp.ma(
pysm.apply_smoothing_and_coord_transform(
bandpass_integrated_map,
fwhm=beam_width_arcmin,
lmax=3 * self.nside - 1,
rot=None if input_reference_frame
== self.pysm_output_reference_frame else
hp.Rotator(coord=(
input_reference_frame,
self.pysm_output_reference_frame,
)),
map_dist=None
if COMM_WORLD is None else pysm.MapDistribution(
nside=self.nside,
smoothing_lmax=3 * self.nside - 1,
mpi_comm=COMM_WORLD,
),
))
if smoothed_map.shape[0] == 1:
channel_map[0] += smoothed_map
else:
channel_map += smoothed_map
output_map = output_map.reshape((len(ch), 1, 3) + self.shape)
if self.other_components is not None:
for comp in self.other_components.values():
kwargs = dict(tube=ch[0].tube, output_units=self.unit)
if function_accepts_argument(comp.simulate, "ch"):
kwargs.pop("tube")
kwargs["ch"] = ch
if function_accepts_argument(comp.simulate, "nsplits"):
kwargs["nsplits"] = self.nsplits
if function_accepts_argument(comp.simulate, "seed"):
kwargs["seed"] = self.num
component_map = comp.simulate(**kwargs)
if self.nsplits == 1:
component_map = component_map.reshape((len(ch), 1, 3) +
self.shape)
component_map[hp.mask_bad(component_map)] = np.nan
output_map = output_map + component_map
for each, channel_map in zip(ch, output_map):
if write_outputs:
for split, each_split_channel_map in enumerate(
channel_map):
filename = self.output_filename_template.format(
telescope=each.telescope
if each.tube is None else each.tube,
band=each.band,
nside=self.nside,
tag=self.tag,
num=self.num,
nsplits=self.nsplits,
split=split + 1,
)
warnings.warn("Writing output map " + filename)
if self.car:
pixell.enmap.write_map(
os.path.join(self.output_folder, filename),
each_split_channel_map,
extra=dict(units=self.unit),
)
else:
each_split_channel_map[np.isnan(
each_split_channel_map)] = hp.UNSEEN
each_split_channel_map = hp.reorder(
each_split_channel_map, r2n=True)
hp.write_map(
os.path.join(self.output_folder, filename),
each_split_channel_map,
coord=self.pysm_output_reference_frame,
column_units=self.unit,
dtype=np.float32,
overwrite=True,
nest=True,
)
else:
if self.nsplits == 1:
channel_map = channel_map[0]
if not self.car:
channel_map[np.isnan(channel_map)] = hp.UNSEEN
output[each.tag] = channel_map
if not write_outputs:
return output
