def test_homogeneous_noise(): seed = 1234 tube = "ST3" simulator = mapsims.SONoiseSimulator(nside=nside, homogeneous=False) output_map = simulator.simulate(tube, seed=seed) assert hp.mask_bad(output_map).sum() > 100 simulator = mapsims.SONoiseSimulator(nside=nside, homogeneous=True) output_map = simulator.simulate(tube, seed=seed) assert hp.mask_bad(output_map).sum() == 0 assert output_map[0][0][1].std() < 4
def __getMaskedPixels(self, pixels): pixels_masked = healpy.ma(pixels) if (self.Configuration.isConfigured(ChartHealpixOptions.MASK)): pixels_masked.mask = healpy.mask_bad(pixels, badval = self.__getMaskedValue()) return pixels_masked.filled(fill_value = numpy.nan)
def ivar_hp_to_cyl(hmap, shape, wcs, rot=False,do_mask=True,extensive=True): from . import mpi, utils import healpy as hp comm = mpi.COMM_WORLD rstep = 100 dtype = np.float32 nside = hp.npix2nside(hmap.size) dec, ra = enmap.posaxes(shape, wcs) pix = np.zeros(shape, np.int32) # Get the pixel area. We assume a rectangular pixelization, so this is just # a function of y ipixsize = 4 * np.pi / (12 * nside ** 2) opixsize = get_pixsize_rect(shape, wcs) nblock = (shape[-2] + rstep - 1) // rstep for bi in range(comm.rank, nblock, comm.size): if bi % comm.size != comm.rank: continue i = bi * rstep rdec = dec[i : i + rstep] opos = np.zeros((2, len(rdec), len(ra))) opos[0] = rdec[:, None] opos[1] = ra[None, :] if rot: # This is unreasonably slow ipos = coordinates.transform("equ", "gal", opos[::-1], pol=True) else: ipos = opos[::-1] pix[i : i + rstep, :] = hp.ang2pix(nside, np.pi / 2 - ipos[1], ipos[0]) del ipos, opos for i in range(0, shape[-2], rstep): pix[i : i + rstep] = utils.allreduce(pix[i : i + rstep], comm) omap = enmap.zeros((1,) + shape, wcs, dtype) imap = np.array(hmap).astype(dtype) imap = imap[None] if do_mask: bad = hp.mask_bad(imap) bad |= imap <= 0 imap[bad] = 0 del bad # Read off the nearest neighbor values omap[:] = imap[:, pix] if extensive: omap *= opixsize[:, None] / ipixsize # We ignore QU mixing during rotation for the noise level, so # it makes no sense to maintain distinct levels for them if do_mask: mask = omap[1:] > 0 omap[1:] = np.mean(omap[1:], 0) omap[1:] *= mask del mask return omap
runit *= factor_857 npol = 1 elif ("smica" in name) or ("WPR2" in name): npol = 1 elif ("COM_Mask_Lensing" in name): npol = 1 runit = 1 else: npol = 3 fields = range(npol) omap = enmap.zeros((npol,)+shape, wcs, dtype) progress("%s TQU read" % name) imap = np.array(healpy.read_map(ifile, fields)).astype(dtype) nside = healpy.npix2nside(imap.shape[-1]) progress("%s TQU mask" % name) imap[healpy.mask_bad(imap)] = 0 progress("%s TQU scale" % name) imap *= runit nside = healpy.npix2nside(imap.shape[-1]) lmax = args.lmax or 3*nside progress("%s TQU alm2map" % name) alm = curvedsky.map2alm_healpix(imap, lmax=lmax) del imap # work around healpix bug progress("%s TQU rotate_alm" % name) alm = alm.astype(np.complex128,copy=False) healpy.rotate_alm(alm, euler[0], euler[1], euler[2]) alm = alm.astype(ctype,copy=False) progress("%s TQU map2alm" % name) curvedsky.alm2map_cyl(alm, omap) del alm
# disk_plot(vec_l[i],dopp_vec_nilc[i],0.1,nside) for i in range(vec_l.size): disk_plot(vec_l[i], abdopp_vec_nilc[i], 0.1, nside) #for i in range(5): # disk_plot(vec_l[i],ab_vec_smica[i],0.1,nside) #for i in range(5): # disk_plot(vec_l[i],dopp_vec_smica[i],0.1,nside) for i in range(vec_l.size): disk_plot(vec_l[i], abdopp_vec_smica[i], 0.1, nside) m[m == 0] = hp.UNSEEN hp.mask_bad(m) hp.mollview(m, badcolor='white', unit=r'$\ell_{max}$', cbar=None, cmap=cmap_test, xsize=2048, title="") hp.graticule() ######################################## ######################################## phi = np.linspace(0, 2. * np.pi, 180) # #aberror=np.array([laterror_smica[0,0],longerror_smica[0,0]]); #dopperror=np.array([laterror_smica[1,1],longerror_smica[1,1]]);
def test_binned(self): rank = 0 if self.comm is not None: rank = self.comm.rank # flag data outside valid intervals gapflagger = OpFlagGaps() gapflagger.exec(self.data) # generate noise timestreams from the noise model nsig = OpSimNoise() nsig.exec(self.data) # make a simple pointing matrix pointing = OpPointingHpix(nside=self.map_nside, nest=True, mode="IQU") pointing.exec(self.data) handle = None if rank == 0: handle = open(os.path.join(self.outdir, "info.txt"), "w") self.data.info(handle=handle) if rank == 0: handle.close() # construct distributed maps to store the covariance, # noise weighted map, and hits invnpp = DistPixels(self.data, comm=self.data.comm.comm_world, nnz=6, dtype=np.float64) invnpp.data.fill(0.0) zmap = DistPixels(self.data, comm=self.data.comm.comm_world, nnz=3, dtype=np.float64) zmap.data.fill(0.0) hits = DistPixels(self.data, comm=self.data.comm.comm_world, nnz=1, dtype=np.int64) hits.data.fill(0) # accumulate the inverse covariance and noise weighted map. # Use detector weights based on the analytic NET. tod = self.data.obs[0]["tod"] nse = self.data.obs[0]["noise"] detweights = {} for d in tod.local_dets: detweights[d] = 1.0 / (self.rate * nse.NET(d)**2) build_invnpp = OpAccumDiag(detweights=detweights, invnpp=invnpp, hits=hits, zmap=zmap, name="noise") build_invnpp.exec(self.data) invnpp.allreduce() hits.allreduce() zmap.allreduce() hits.write_healpix_fits(os.path.join(self.outdir, "hits.fits")) invnpp.write_healpix_fits(os.path.join(self.outdir, "invnpp.fits")) zmap.write_healpix_fits(os.path.join(self.outdir, "zmap.fits")) # invert it covariance_invert(invnpp, 1.0e-3) invnpp.write_healpix_fits(os.path.join(self.outdir, "npp.fits")) # compute the binned map, N_pp x Z covariance_apply(invnpp, zmap) zmap.write_healpix_fits(os.path.join(self.outdir, "binned.fits")) # compare with MADAM madam_out = os.path.join(self.outdir, "madam") if rank == 0: if os.path.isdir(madam_out): shutil.rmtree(madam_out) os.mkdir(madam_out) pars = {} pars["temperature_only"] = "F" pars["force_pol"] = "T" pars["kfirst"] = "F" pars["base_first"] = 1.0 pars["fsample"] = self.rate pars["nside_map"] = self.map_nside pars["nside_cross"] = self.map_nside pars["nside_submap"] = self.map_nside pars["pixlim_cross"] = 1.0e-2 pars["pixlim_map"] = 1.0e-3 pars["write_map"] = "F" pars["write_binmap"] = "T" pars["write_matrix"] = "T" pars["write_wcov"] = "T" pars["write_hits"] = "T" pars["kfilter"] = "F" pars["path_output"] = madam_out pars["info"] = 0 madam = OpMadam(params=pars, detweights=detweights, name="noise") if madam.available: madam.exec(self.data) if rank == 0: import matplotlib.pyplot as plt hitsfile = os.path.join(madam_out, "madam_hmap.fits") hits = hp.read_map(hitsfile, nest=True, verbose=False) outfile = "{}.png".format(hitsfile) hp.mollview(hits, xsize=1600, nest=True) plt.savefig(outfile) plt.close() toastfile = os.path.join(self.outdir, "hits.fits") toasthits = hp.read_map(toastfile, nest=True, verbose=False) nt.assert_equal(hits, toasthits) tothits = np.sum(hits) nt.assert_equal(self.validsamp, tothits) covfile = os.path.join(madam_out, "madam_wcov_inv.fits") cov = hp.read_map(covfile, nest=True, field=None, verbose=False) toastfile = os.path.join(self.outdir, "invnpp.fits") toastcov = hp.read_map(toastfile, nest=True, field=None, verbose=False) """ for p in range(6): print("elem {} madam min/max = ".format(p), np.min(cov[p]), " / ", np.max(cov[p])) print("elem {} toast min/max = ".format(p), np.min(toastcov[p]), " / ", np.max(toastcov[p])) print("elem {} invNpp max diff = ".format(p), np.max(np.absolute(toastcov[p] - cov[p]))) nt.assert_almost_equal(cov[p], toastcov[p]) """ covfile = os.path.join(madam_out, "madam_wcov.fits") cov = hp.read_map(covfile, nest=True, field=None, verbose=False) toastfile = os.path.join(self.outdir, "npp.fits") toastcov = hp.read_map(toastfile, nest=True, field=None, verbose=False) """ for p in range(6): covdiff = toastcov[p] - cov[p] print("elem {} madam min/max = ".format(p), np.min(cov[p]), " / ", np.max(cov[p])) print("elem {} toast min/max = ".format(p), np.min(toastcov[p]), " / ", np.max(toastcov[p])) print("elem {} Npp max diff = ".format(p), np.max(np.absolute(covdiff[p]))) print("elem {} Npp mean / rms diff = ".format(p), np.mean(covdiff[p]), " / ", np.std(covdiff[p])) print("elem {} Npp relative diff mean / rms = ".format(p), np.mean(np.absolute(covdiff[p]/cov[p])), " / ", np.std(np.absolute(covdiff[p]/cov[p]))) nt.assert_almost_equal(cov[p], toastcov[p]) """ binfile = os.path.join(madam_out, "madam_bmap.fits") bins = hp.read_map(binfile, nest=True, field=None, verbose=False) mask = hp.mask_bad(bins[0]) bins[0][mask] = 0.0 mask = hp.mask_bad(bins[1]) bins[1][mask] = 0.0 mask = hp.mask_bad(bins[2]) bins[2][mask] = 0.0 outfile = "{}_I.png".format(binfile) hp.mollview(bins[0], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_Q.png".format(binfile) hp.mollview(bins[1], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_U.png".format(binfile) hp.mollview(bins[2], xsize=1600, nest=True) plt.savefig(outfile) plt.close() toastfile = os.path.join(self.outdir, "binned.fits") toastbins = hp.read_map(toastfile, nest=True, field=None, verbose=False) outfile = "{}_I.png".format(toastfile) hp.mollview(toastbins[0], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_Q.png".format(toastfile) hp.mollview(toastbins[1], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_U.png".format(toastfile) hp.mollview(toastbins[2], xsize=1600, nest=True) plt.savefig(outfile) plt.close() # compare binned map to madam output diffmap = toastbins[0] - bins[0] mask = bins[0] != 0 """ print("toast/madam I diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("toast/madam I diff rel ratio min / max = ", np.min(diffmap[mask]/bins[0][mask]), " / ", np.max(diffmap[mask]/bins[0][mask])) """ outfile = "{}_diff_madam_I.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() # nt.assert_almost_equal(bins[0][mask], binserial[0][mask], # decimal=6) diffmap = toastbins[1] - bins[1] mask = bins[1] != 0 """ print("toast/madam Q diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("toast/madam Q diff rel ratio min / max = ", np.min(diffmap[mask]/bins[1][mask]), " / ", np.max(diffmap[mask]/bins[1][mask])) """ outfile = "{}_diff_madam_Q.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() # nt.assert_almost_equal(bins[1][mask], binserial[1][mask], # decimal=6) diffmap = toastbins[2] - bins[2] mask = bins[2] != 0 """ print("toast/madam U diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("toast/madam U diff rel ratio min / max = ", np.min(diffmap[mask]/bins[2][mask]), " / ", np.max(diffmap[mask]/bins[2][mask])) """ outfile = "{}_diff_madam_U.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() # nt.assert_almost_equal(bins[2][mask], binserial[2][mask], # decimal=6) # compute the binned map serially as a check zfile = os.path.join(self.outdir, "zmap.fits") ztoast = hp.read_map(zfile, nest=True, field=None, verbose=False) binserial = np.copy(ztoast) for p in range(self.map_npix): binserial[0][p] = (toastcov[0][p] * ztoast[0][p] + toastcov[1][p] * ztoast[1][p] + toastcov[2][p] * ztoast[2][p]) binserial[1][p] = (toastcov[1][p] * ztoast[0][p] + toastcov[3][p] * ztoast[1][p] + toastcov[4][p] * ztoast[2][p]) binserial[2][p] = (toastcov[2][p] * ztoast[0][p] + toastcov[4][p] * ztoast[1][p] + toastcov[5][p] * ztoast[2][p]) toastfile = os.path.join(self.outdir, "binned_serial") outfile = "{}_I.png".format(toastfile) hp.mollview(binserial[0], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_Q.png".format(toastfile) hp.mollview(binserial[1], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_U.png".format(toastfile) hp.mollview(binserial[2], xsize=1600, nest=True) plt.savefig(outfile) plt.close() # compare binned map to madam output diffmap = binserial[0] - bins[0] mask = bins[0] != 0 """ print("serial/madam I diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("serial/madam I diff rel ratio min / max = ", np.min(diffmap[mask]/bins[0][mask]), " / ", np.max(diffmap[mask]/bins[0][mask])) """ outfile = "{}_diff_madam_I.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() nt.assert_almost_equal(bins[0][mask], binserial[0][mask], decimal=3) diffmap = binserial[1] - bins[1] mask = bins[1] != 0 """ print("serial/madam Q diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("serial/madam Q diff rel ratio min / max = ", np.min(diffmap[mask]/bins[1][mask]), " / ", np.max(diffmap[mask]/bins[1][mask])) """ outfile = "{}_diff_madam_Q.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() nt.assert_almost_equal(bins[1][mask], binserial[1][mask], decimal=3) diffmap = binserial[2] - bins[2] mask = bins[2] != 0 """ print("serial/madam U diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("serial/madam U diff rel ratio min / max = ", np.min(diffmap[mask]/bins[2][mask]), " / ", np.max(diffmap[mask]/bins[2][mask])) """ outfile = "{}_diff_madam_U.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() nt.assert_almost_equal(bins[2][mask], binserial[2][mask], decimal=3) else: if rank == 0: print("libmadam not available, skipping tests", flush=True) return
def test_binned(self): # flag data outside valid intervals gapflagger = OpFlagGaps() gapflagger.exec(self.data) # generate noise timestreams from the noise model nsig = OpSimNoise() nsig.exec(self.data) # make a simple pointing matrix pointing = OpPointingHpix( nside=self.map_nside, nest=True, mode="IQU", hwprpm=self.hwprpm ) pointing.exec(self.data) handle = None if self.comm.rank == 0: handle = open(os.path.join(self.outdir, "info.txt"), "w") self.data.info(handle) if self.comm.rank == 0: handle.close() # get locally hit pixels lc = OpLocalPixels() localpix = lc.exec(self.data) # find the locally hit submaps. localsm = np.unique(np.floor_divide(localpix, self.subnpix)) # construct distributed maps to store the covariance, # noise weighted map, and hits invnpp = DistPixels( comm=self.data.comm.comm_world, size=self.sim_npix, nnz=6, dtype=np.float64, submap=self.subnpix, local=localsm, ) invnpp.data.fill(0.0) zmap = DistPixels( comm=self.data.comm.comm_world, size=self.sim_npix, nnz=3, dtype=np.float64, submap=self.subnpix, local=localsm, ) zmap.data.fill(0.0) hits = DistPixels( comm=self.data.comm.comm_world, size=self.sim_npix, nnz=1, dtype=np.int64, submap=self.subnpix, local=localsm, ) hits.data.fill(0) # accumulate the inverse covariance and noise weighted map. # Use detector weights based on the analytic NET. tod = self.data.obs[0]["tod"] nse = self.data.obs[0]["noise"] detweights = {} for d in tod.local_dets: detweights[d] = 1.0 / (self.rate * nse.NET(d) ** 2) build_invnpp = OpAccumDiag( detweights=detweights, invnpp=invnpp, hits=hits, zmap=zmap, name="noise" ) build_invnpp.exec(self.data) invnpp.allreduce() hits.allreduce() zmap.allreduce() hits.write_healpix_fits(os.path.join(self.outdir, "hits.fits")) invnpp.write_healpix_fits(os.path.join(self.outdir, "invnpp.fits")) zmap.write_healpix_fits(os.path.join(self.outdir, "zmap.fits")) # invert it covariance_invert(invnpp, 1.0e-3) invnpp.write_healpix_fits(os.path.join(self.outdir, "npp.fits")) # compute the binned map, N_pp x Z covariance_apply(invnpp, zmap) zmap.write_healpix_fits(os.path.join(self.outdir, "binned.fits")) # compare with MADAM madam_out = os.path.join(self.outdir, "madam") if self.comm.rank == 0: if os.path.isdir(madam_out): shutil.rmtree(madam_out) os.mkdir(madam_out) pars = {} pars["temperature_only"] = "F" pars["force_pol"] = "T" pars["kfirst"] = "F" pars["base_first"] = 1.0 pars["fsample"] = self.rate pars["nside_map"] = self.map_nside pars["nside_cross"] = self.map_nside pars["nside_submap"] = self.map_nside pars["pixlim_cross"] = 1.0e-2 pars["pixlim_map"] = 1.0e-3 pars["write_map"] = "F" pars["write_binmap"] = "T" pars["write_matrix"] = "T" pars["write_wcov"] = "T" pars["write_hits"] = "T" pars["kfilter"] = "F" pars["path_output"] = madam_out pars["info"] = 0 madam = OpMadam(params=pars, detweights=detweights, name="noise") if madam.available: madam.exec(self.data) if self.comm.rank == 0: import matplotlib.pyplot as plt hitsfile = os.path.join(madam_out, "madam_hmap.fits") hits = hp.read_map(hitsfile, nest=True) outfile = "{}.png".format(hitsfile) hp.mollview(hits, xsize=1600, nest=True) plt.savefig(outfile) plt.close() toastfile = os.path.join(self.outdir, "hits.fits") toasthits = hp.read_map(toastfile, nest=True) nt.assert_equal(hits, toasthits) tothits = np.sum(hits) nt.assert_equal(self.validsamp, tothits) covfile = os.path.join(madam_out, "madam_wcov_inv.fits") cov = hp.read_map(covfile, nest=True, field=None) toastfile = os.path.join(self.outdir, "invnpp.fits") toastcov = hp.read_map(toastfile, nest=True, field=None) """ for p in range(6): print("elem {} madam min/max = ".format(p), np.min(cov[p]), " / ", np.max(cov[p])) print("elem {} toast min/max = ".format(p), np.min(toastcov[p]), " / ", np.max(toastcov[p])) print("elem {} invNpp max diff = ".format(p), np.max(np.absolute(toastcov[p] - cov[p]))) nt.assert_almost_equal(cov[p], toastcov[p]) """ covfile = os.path.join(madam_out, "madam_wcov.fits") cov = hp.read_map(covfile, nest=True, field=None) toastfile = os.path.join(self.outdir, "npp.fits") toastcov = hp.read_map(toastfile, nest=True, field=None) """ for p in range(6): covdiff = toastcov[p] - cov[p] print("elem {} madam min/max = ".format(p), np.min(cov[p]), " / ", np.max(cov[p])) print("elem {} toast min/max = ".format(p), np.min(toastcov[p]), " / ", np.max(toastcov[p])) print("elem {} Npp max diff = ".format(p), np.max(np.absolute(covdiff[p]))) print("elem {} Npp mean / rms diff = ".format(p), np.mean(covdiff[p]), " / ", np.std(covdiff[p])) print("elem {} Npp relative diff mean / rms = ".format(p), np.mean(np.absolute(covdiff[p]/cov[p])), " / ", np.std(np.absolute(covdiff[p]/cov[p]))) nt.assert_almost_equal(cov[p], toastcov[p]) """ binfile = os.path.join(madam_out, "madam_bmap.fits") bins = hp.read_map(binfile, nest=True, field=None) mask = hp.mask_bad(bins[0]) bins[0][mask] = 0.0 mask = hp.mask_bad(bins[1]) bins[1][mask] = 0.0 mask = hp.mask_bad(bins[2]) bins[2][mask] = 0.0 outfile = "{}_I.png".format(binfile) hp.mollview(bins[0], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_Q.png".format(binfile) hp.mollview(bins[1], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_U.png".format(binfile) hp.mollview(bins[2], xsize=1600, nest=True) plt.savefig(outfile) plt.close() toastfile = os.path.join(self.outdir, "binned.fits") toastbins = hp.read_map(toastfile, nest=True, field=None) outfile = "{}_I.png".format(toastfile) hp.mollview(toastbins[0], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_Q.png".format(toastfile) hp.mollview(toastbins[1], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_U.png".format(toastfile) hp.mollview(toastbins[2], xsize=1600, nest=True) plt.savefig(outfile) plt.close() # compare binned map to madam output diffmap = toastbins[0] - bins[0] mask = bins[0] != 0 """ print("toast/madam I diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("toast/madam I diff rel ratio min / max = ", np.min(diffmap[mask]/bins[0][mask]), " / ", np.max(diffmap[mask]/bins[0][mask])) """ outfile = "{}_diff_madam_I.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() # nt.assert_almost_equal(bins[0][mask], binserial[0][mask], # decimal=6) diffmap = toastbins[1] - bins[1] mask = bins[1] != 0 """ print("toast/madam Q diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("toast/madam Q diff rel ratio min / max = ", np.min(diffmap[mask]/bins[1][mask]), " / ", np.max(diffmap[mask]/bins[1][mask])) """ outfile = "{}_diff_madam_Q.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() # nt.assert_almost_equal(bins[1][mask], binserial[1][mask], # decimal=6) diffmap = toastbins[2] - bins[2] mask = bins[2] != 0 """ print("toast/madam U diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("toast/madam U diff rel ratio min / max = ", np.min(diffmap[mask]/bins[2][mask]), " / ", np.max(diffmap[mask]/bins[2][mask])) """ outfile = "{}_diff_madam_U.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() # nt.assert_almost_equal(bins[2][mask], binserial[2][mask], # decimal=6) # compute the binned map serially as a check zfile = os.path.join(self.outdir, "zmap.fits") ztoast = hp.read_map(zfile, nest=True, field=None) binserial = np.copy(ztoast) for p in range(self.map_npix): binserial[0][p] = ( toastcov[0][p] * ztoast[0][p] + toastcov[1][p] * ztoast[1][p] + toastcov[2][p] * ztoast[2][p] ) binserial[1][p] = ( toastcov[1][p] * ztoast[0][p] + toastcov[3][p] * ztoast[1][p] + toastcov[4][p] * ztoast[2][p] ) binserial[2][p] = ( toastcov[2][p] * ztoast[0][p] + toastcov[4][p] * ztoast[1][p] + toastcov[5][p] * ztoast[2][p] ) toastfile = os.path.join(self.outdir, "binned_serial") outfile = "{}_I.png".format(toastfile) hp.mollview(binserial[0], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_Q.png".format(toastfile) hp.mollview(binserial[1], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_U.png".format(toastfile) hp.mollview(binserial[2], xsize=1600, nest=True) plt.savefig(outfile) plt.close() # compare binned map to madam output diffmap = binserial[0] - bins[0] mask = bins[0] != 0 """ print("serial/madam I diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("serial/madam I diff rel ratio min / max = ", np.min(diffmap[mask]/bins[0][mask]), " / ", np.max(diffmap[mask]/bins[0][mask])) """ outfile = "{}_diff_madam_I.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() nt.assert_almost_equal(bins[0][mask], binserial[0][mask], decimal=3) diffmap = binserial[1] - bins[1] mask = bins[1] != 0 """ print("serial/madam Q diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("serial/madam Q diff rel ratio min / max = ", np.min(diffmap[mask]/bins[1][mask]), " / ", np.max(diffmap[mask]/bins[1][mask])) """ outfile = "{}_diff_madam_Q.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() nt.assert_almost_equal(bins[1][mask], binserial[1][mask], decimal=3) diffmap = binserial[2] - bins[2] mask = bins[2] != 0 """ print("serial/madam U diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("serial/madam U diff rel ratio min / max = ", np.min(diffmap[mask]/bins[2][mask]), " / ", np.max(diffmap[mask]/bins[2][mask])) """ outfile = "{}_diff_madam_U.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() nt.assert_almost_equal(bins[2][mask], binserial[2][mask], decimal=3) else: if self.comm.rank == 0: print("libmadam not available, skipping tests", flush=True) return
hp.gnomview(mapa2r, rot=[40,80,0], reso=8, title='CMB map', unit='K') pyplot.savefig('gnomonic_view_bad.png') mapa_save = [mapa2r,mapa[1]] hp.write_map('CMB_map_Ns1024.fits',mapa_save, coord='G', column_names=list(['I_CMB','Confidence-Mask']), column_units=list(['K',' '])) #%% """ STEP 2: Missing value """ m = np.arange(12.) m[2] = hp.UNSEEN m_bad = hp.mask_bad(m) m_good = hp.mask_good(m) print('m_bad ->', m_bad) print('m_good ->', m_good) hp.mollview(m_bad, title='mask_bad') hp.mollview(m_good, title='mask_good') #%% """ STEP 3: Treating missing pixels """
def test_binned(self): start = MPI.Wtime() # flag data outside valid intervals gapflagger = OpFlagGaps() gapflagger.exec(self.data) # generate noise timestreams from the noise model nsig = OpSimNoise() nsig.exec(self.data) # make a simple pointing matrix pointing = OpPointingHpix(nside=self.map_nside, nest=True, mode='IQU', hwprpm=self.hwprpm) pointing.exec(self.data) handle = None if self.comm.rank == 0: handle = open(os.path.join(self.mapdir, "info.txt"), "w") self.data.info(handle) if self.comm.rank == 0: handle.close() # get locally hit pixels lc = OpLocalPixels() localpix = lc.exec(self.data) # find the locally hit submaps. localsm = np.unique(np.floor_divide(localpix, self.subnpix)) # construct distributed maps to store the covariance, # noise weighted map, and hits invnpp = DistPixels(comm=self.toastcomm.comm_group, size=self.sim_npix, nnz=6, dtype=np.float64, submap=self.subnpix, local=localsm) invnpp.data.fill(0.0) zmap = DistPixels(comm=self.toastcomm.comm_group, size=self.sim_npix, nnz=3, dtype=np.float64, submap=self.subnpix, local=localsm) zmap.data.fill(0.0) hits = DistPixels(comm=self.toastcomm.comm_group, size=self.sim_npix, nnz=1, dtype=np.int64, submap=self.subnpix, local=localsm) hits.data.fill(0) # accumulate the inverse covariance and noise weighted map. # Use detector weights based on the analytic NET. tod = self.data.obs[0]['tod'] nse = self.data.obs[0]['noise'] detweights = {} for d in tod.local_dets: detweights[d] = 1.0 / (self.rate * nse.NET(d)**2) build_invnpp = OpAccumDiag(detweights=detweights, invnpp=invnpp, hits=hits, zmap=zmap, name="noise") build_invnpp.exec(self.data) invnpp.allreduce() hits.allreduce() zmap.allreduce() hits.write_healpix_fits(os.path.join(self.mapdir, "hits.fits")) invnpp.write_healpix_fits(os.path.join(self.mapdir, "invnpp.fits")) zmap.write_healpix_fits(os.path.join(self.mapdir, "zmap.fits")) # invert it covariance_invert(invnpp, 1.0e-3) invnpp.write_healpix_fits(os.path.join(self.mapdir, "npp.fits")) # compute the binned map, N_pp x Z covariance_apply(invnpp, zmap) zmap.write_healpix_fits(os.path.join(self.mapdir, "binned.fits")) # compare with MADAM madam_out = os.path.join(self.mapdir, "madam") if self.comm.rank == 0: if os.path.isdir(madam_out): shutil.rmtree(madam_out) os.mkdir(madam_out) pars = {} pars['temperature_only'] = 'F' pars['force_pol'] = 'T' pars['kfirst'] = 'F' pars['base_first'] = 1.0 pars['fsample'] = self.rate pars['nside_map'] = self.map_nside pars['nside_cross'] = self.map_nside pars['nside_submap'] = self.map_nside pars['pixlim_cross'] = 1.0e-2 pars['pixlim_map'] = 1.0e-3 pars['write_map'] = 'F' pars['write_binmap'] = 'T' pars['write_matrix'] = 'T' pars['write_wcov'] = 'T' pars['write_hits'] = 'T' pars['kfilter'] = 'F' pars['path_output'] = madam_out pars['info'] = 0 madam = OpMadam(params=pars, detweights=detweights, name="noise") if madam.available: madam.exec(self.data) if self.comm.rank == 0: import matplotlib.pyplot as plt hitsfile = os.path.join(madam_out, 'madam_hmap.fits') print('Loading', hitsfile) hits = hp.read_map(hitsfile, nest=True) outfile = "{}.png".format(hitsfile) hp.mollview(hits, xsize=1600, nest=True) plt.savefig(outfile) plt.close() toastfile = os.path.join(self.mapdir, 'hits.fits') print('Loading', toastfile) toasthits = hp.read_map(toastfile, nest=True) nt.assert_equal(hits, toasthits) tothits = np.sum(hits) nt.assert_equal(self.validsamp, tothits) covfile = os.path.join(madam_out, 'madam_wcov_inv.fits') cov = hp.read_map(covfile, nest=True, field=None) toastfile = os.path.join(self.mapdir, 'invnpp.fits') toastcov = hp.read_map(toastfile, nest=True, field=None) """ for p in range(6): print("elem {} madam min/max = ".format(p), np.min(cov[p]), " / ", np.max(cov[p])) print("elem {} toast min/max = ".format(p), np.min(toastcov[p]), " / ", np.max(toastcov[p])) print("elem {} invNpp max diff = ".format(p), np.max(np.absolute(toastcov[p] - cov[p]))) nt.assert_almost_equal(cov[p], toastcov[p]) """ covfile = os.path.join(madam_out, 'madam_wcov.fits') cov = hp.read_map(covfile, nest=True, field=None) toastfile = os.path.join(self.mapdir, 'npp.fits') toastcov = hp.read_map(toastfile, nest=True, field=None) """ for p in range(6): covdiff = toastcov[p] - cov[p] print("elem {} madam min/max = ".format(p), np.min(cov[p]), " / ", np.max(cov[p])) print("elem {} toast min/max = ".format(p), np.min(toastcov[p]), " / ", np.max(toastcov[p])) print("elem {} Npp max diff = ".format(p), np.max(np.absolute(covdiff[p]))) print("elem {} Npp mean / rms diff = ".format(p), np.mean(covdiff[p]), " / ", np.std(covdiff[p])) print("elem {} Npp relative diff mean / rms = ".format(p), np.mean(np.absolute(covdiff[p]/cov[p])), " / ", np.std(np.absolute(covdiff[p]/cov[p]))) nt.assert_almost_equal(cov[p], toastcov[p]) """ binfile = os.path.join(madam_out, 'madam_bmap.fits') bins = hp.read_map(binfile, nest=True, field=None) mask = hp.mask_bad(bins[0]) bins[0][mask] = 0.0 mask = hp.mask_bad(bins[1]) bins[1][mask] = 0.0 mask = hp.mask_bad(bins[2]) bins[2][mask] = 0.0 outfile = "{}_I.png".format(binfile) hp.mollview(bins[0], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_Q.png".format(binfile) hp.mollview(bins[1], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_U.png".format(binfile) hp.mollview(bins[2], xsize=1600, nest=True) plt.savefig(outfile) plt.close() toastfile = os.path.join(self.mapdir, 'binned.fits') toastbins = hp.read_map(toastfile, nest=True, field=None) outfile = "{}_I.png".format(toastfile) hp.mollview(toastbins[0], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_Q.png".format(toastfile) hp.mollview(toastbins[1], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_U.png".format(toastfile) hp.mollview(toastbins[2], xsize=1600, nest=True) plt.savefig(outfile) plt.close() # compare binned map to madam output diffmap = toastbins[0] - bins[0] mask = (bins[0] != 0) """ print("toast/madam I diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("toast/madam I diff rel ratio min / max = ", np.min(diffmap[mask]/bins[0][mask]), " / ", np.max(diffmap[mask]/bins[0][mask])) """ outfile = "{}_diff_madam_I.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() #nt.assert_almost_equal(bins[0][mask], binserial[0][mask], # decimal=6) diffmap = toastbins[1] - bins[1] mask = (bins[1] != 0) """ print("toast/madam Q diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("toast/madam Q diff rel ratio min / max = ", np.min(diffmap[mask]/bins[1][mask]), " / ", np.max(diffmap[mask]/bins[1][mask])) """ outfile = "{}_diff_madam_Q.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() #nt.assert_almost_equal(bins[1][mask], binserial[1][mask], # decimal=6) diffmap = toastbins[2] - bins[2] mask = (bins[2] != 0) """ print("toast/madam U diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("toast/madam U diff rel ratio min / max = ", np.min(diffmap[mask]/bins[2][mask]), " / ", np.max(diffmap[mask]/bins[2][mask])) """ outfile = "{}_diff_madam_U.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() #nt.assert_almost_equal(bins[2][mask], binserial[2][mask], # decimal=6) # compute the binned map serially as a check zfile = os.path.join(self.mapdir, 'zmap.fits') ztoast = hp.read_map(zfile, nest=True, field=None) binserial = np.copy(ztoast) for p in range(self.map_npix): binserial[0][p] = toastcov[0][p] * ztoast[0][p] \ + toastcov[1][p] * ztoast[1][p] \ + toastcov[2][p] * ztoast[2][p] binserial[1][p] = toastcov[1][p] * ztoast[0][p] \ + toastcov[3][p] * ztoast[1][p] \ + toastcov[4][p] * ztoast[2][p] binserial[2][p] = toastcov[2][p] * ztoast[0][p] \ + toastcov[4][p] * ztoast[1][p] \ + toastcov[5][p] * ztoast[2][p] toastfile = os.path.join(self.mapdir, 'binned_serial') outfile = "{}_I.png".format(toastfile) hp.mollview(binserial[0], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_Q.png".format(toastfile) hp.mollview(binserial[1], xsize=1600, nest=True) plt.savefig(outfile) plt.close() outfile = "{}_U.png".format(toastfile) hp.mollview(binserial[2], xsize=1600, nest=True) plt.savefig(outfile) plt.close() # compare binned map to madam output diffmap = binserial[0] - bins[0] mask = (bins[0] != 0) """ print("serial/madam I diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("serial/madam I diff rel ratio min / max = ", np.min(diffmap[mask]/bins[0][mask]), " / ", np.max(diffmap[mask]/bins[0][mask])) """ outfile = "{}_diff_madam_I.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() nt.assert_almost_equal(bins[0][mask], binserial[0][mask], decimal=3) diffmap = binserial[1] - bins[1] mask = (bins[1] != 0) """ print("serial/madam Q diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("serial/madam Q diff rel ratio min / max = ", np.min(diffmap[mask]/bins[1][mask]), " / ", np.max(diffmap[mask]/bins[1][mask])) """ outfile = "{}_diff_madam_Q.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() nt.assert_almost_equal(bins[1][mask], binserial[1][mask], decimal=3) diffmap = binserial[2] - bins[2] mask = (bins[2] != 0) """ print("serial/madam U diff mean / std = ", np.mean(diffmap[mask]), np.std(diffmap[mask])) print("serial/madam U diff rel ratio min / max = ", np.min(diffmap[mask]/bins[2][mask]), " / ", np.max(diffmap[mask]/bins[2][mask])) """ outfile = "{}_diff_madam_U.png".format(toastfile) hp.mollview(diffmap, xsize=1600, nest=True) plt.savefig(outfile) plt.close() nt.assert_almost_equal(bins[2][mask], binserial[2][mask], decimal=3) stop = MPI.Wtime() elapsed = stop - start self.print_in_turns("Madam test took {:.3f} s".format(elapsed)) else: print("libmadam not available, skipping tests") #self.assertTrue(False) return
def execute(self, write_outputs=False): """Run map simulations Execute simulations for all channels and write to disk the maps, unless `write_outputs` is False, then return them. """ if self.run_pysm: sky_config = [] preset_strings = [] if self.pysm_components_string is not None: for model in self.pysm_components_string.split(","): if model.startswith("SO"): sky_config.append(get_so_models(model, self.nside)) else: preset_strings.append(model) if len(preset_strings) > 0: input_reference_frame = "G" assert ( len(sky_config) == 0 ), "Cannot mix PySM and SO models, they are defined in G and C frames" else: input_reference_frame = "C" self.pysm_sky = pysm.Sky( nside=self.nside, preset_strings=preset_strings, component_objects=sky_config, output_unit=u.Unit(self.unit), ) if self.pysm_custom_components is not None: for comp_name, comp in self.pysm_custom_components.items(): self.pysm_sky.components.append(comp) if not write_outputs: output = {} # ch can be single channel or tuple of 2 channels (tube dichroic) for ch in self.channels: if not isinstance(ch, tuple): ch = [ch] output_map_shape = (len(ch), 3) + self.shape output_map = np.zeros(output_map_shape, dtype=np.float64) if self.run_pysm: for each, channel_map in zip(ch, output_map): bandpass_integrated_map = self.pysm_sky.get_emission( *each.bandpass).value beam_width_arcmin = each.beam # smoothing and coordinate rotation with 1 spherical harmonics transform smoothed_map = hp.ma( pysm.apply_smoothing_and_coord_transform( bandpass_integrated_map, fwhm=beam_width_arcmin, lmax=3 * self.nside - 1, rot=None if input_reference_frame == self.pysm_output_reference_frame else hp.Rotator(coord=( input_reference_frame, self.pysm_output_reference_frame, )), map_dist=None if COMM_WORLD is None else pysm.MapDistribution( nside=self.nside, smoothing_lmax=3 * self.nside - 1, mpi_comm=COMM_WORLD, ), )) if smoothed_map.shape[0] == 1: channel_map[0] += smoothed_map else: channel_map += smoothed_map output_map = output_map.reshape((len(ch), 1, 3) + self.shape) if self.other_components is not None: for comp in self.other_components.values(): kwargs = dict(tube=ch[0].tube, output_units=self.unit) if function_accepts_argument(comp.simulate, "ch"): kwargs.pop("tube") kwargs["ch"] = ch if function_accepts_argument(comp.simulate, "nsplits"): kwargs["nsplits"] = self.nsplits if function_accepts_argument(comp.simulate, "seed"): kwargs["seed"] = self.num component_map = comp.simulate(**kwargs) if self.nsplits == 1: component_map = component_map.reshape((len(ch), 1, 3) + self.shape) component_map[hp.mask_bad(component_map)] = np.nan output_map = output_map + component_map for each, channel_map in zip(ch, output_map): if write_outputs: for split, each_split_channel_map in enumerate( channel_map): filename = self.output_filename_template.format( telescope=each.telescope if each.tube is None else each.tube, band=each.band, nside=self.nside, tag=self.tag, num=self.num, nsplits=self.nsplits, split=split + 1, ) warnings.warn("Writing output map " + filename) if self.car: pixell.enmap.write_map( os.path.join(self.output_folder, filename), each_split_channel_map, extra=dict(units=self.unit), ) else: each_split_channel_map[np.isnan( each_split_channel_map)] = hp.UNSEEN each_split_channel_map = hp.reorder( each_split_channel_map, r2n=True) hp.write_map( os.path.join(self.output_folder, filename), each_split_channel_map, coord=self.pysm_output_reference_frame, column_units=self.unit, dtype=np.float32, overwrite=True, nest=True, ) else: if self.nsplits == 1: channel_map = channel_map[0] if not self.car: channel_map[np.isnan(channel_map)] = hp.UNSEEN output[each.tag] = channel_map if not write_outputs: return output