def noise_profile_fitting(residuals,
coverage,
nfsub,
config,
nbins,
dirsave=None):
plt.figure(figsize=(16, 10))
myfitcovs = []
for isub in range(nfsub):
sqn = np.int(np.sqrt(nfsub))
if (sqn**2) != nfsub:
sqn += 1
plt.subplot(sqn, sqn, isub + 1)
xx, yyfs, fitcov = qss.get_noise_invcov_profile(
residuals[isub, :, :],
coverage,
QUsep=True,
nbins=nbins,
label='End-To-End sub={}/{}'.format(isub + 1, nfsub),
fit=True,
norm=False,
allstokes=True,
doplot=True)
plt.legend(fontsize=9)
myfitcovs.append(fitcov)
if dirsave is None:
plt.show()
else:
plt.savefig(dirsave + f'NoiseProfileFitting_{nfsub}bands_' + config +
'.pdf',
format='pdf')
plt.close()
return myfitcovs
def noise_qss(dictionaries, sky_configuration, coverages, realizations, verbose = False):
"""
Assume all dictionaries have the same 'effective_duration' and same 'nf_recon'
"""
##### Getting FastSimulator output maps
noise = np.zeros((len(dictionaries), realizations, dictionaries[0]['nf_recon'], 12 * dictionaries[0]['nside']**2,3))
QubicSkyObject = []
for ic, idict in enumerate(dictionaries):
QubicSkyObject.append(qss.Qubic_sky(sky_configuration, idict))
if verbose: print(type(QubicSkyObject[ic]))
for i in range(realizations):
if verbose: print(type(coverages[ic]))
noise[ic, i, ...], _ = \
QubicSkyObject[ic].get_partial_sky_maps_withnoise(spatial_noise=False, coverage = coverages[ic],
noise_only = True, Nyears = idict['effective_duration'],
verbose = verbose)
if verbose: print('=== Done interation #{} ===='.format(i+1))
return noise
def get_nunu_covariance(residuals, coverage, nfsub, config, dirsave=None):
cI, cQ, cU, fitcov, noise_norm = qss.get_cov_nunu(residuals,
coverage,
QUsep=True)
corr_mats = [cI, cQ / 2, cU / 2]
plt.figure(figsize=(16, 6))
stn = ['I', 'Q/2', 'U/2']
bla = np.max(
[np.abs(np.min(np.array(corr_mats))),
np.max(np.array(corr_mats))])
mini = -bla
maxi = bla
for s in range(3):
plt.subplot(1, 3, 1 + s)
plt.imshow(corr_mats[s], vmin=mini, vmax=maxi, cmap='bwr')
plt.colorbar(orientation='horizontal')
plt.title('End-To-End Cov {} nsub={}'.format(stn[s], nfsub))
if dirsave is None:
plt.show()
else:
plt.savefig(dirsave + f'nunuCovariance_{nfsub}bands_' + config +
'.pdf',
format='pdf')
plt.close()
return cI, cQ, cU
def foreground_signal(dictionaries, sky_configuration, sky = 'T',
seed = None, verbose = False):
"""
Averaging manually the maps into a band if nf_sub != nfrecon, otherwise, foreground are computed
in nf_recon sub-bands
Assumes . dictionaries[:]['nf_sub'] == dictionaries[:]['nf_recon']
. all regions and frequencies uses same sky configuration.
Parameters:
dictionaries:
array of dictionaries. #dicts = #regions + #bands_needed
sky_configuration:
dictionary with PySM format to build foregrounds
sky:
'T' or 'P' for temperature or polarization study
seed:
fix seed for simulations
"""
# Generate foregrounds
##### QubicSkySim instanciation
seed = seed
QubicSkyObject = []
foreground_maps = np.zeros((len(dictionaries), dictionaries[0]['nf_recon'], 12 * dictionaries[0]['nside'] ** 2, 3))
for ic, idict in enumerate(dictionaries):
QubicSkyObject.append(qss.Qubic_sky(sky_configuration, idict))
if idict['nf_sub'] != idict['nf_recon']:
_, nus_edge_in, nus_in, _, _, _ = qubic.compute_freq(idict['filter_nu'] / 1e9,
idict['nf_sub'],
idict['filter_relative_bandwidth'])
_, nus_edge_out, nus_out, _, _, _ = qubic.compute_freq(idict['filter_nu'] / 1e9,
idict['nf_recon'],
idict['filter_relative_bandwidth'])
if verbose: print('Computing maps averaging')
# Generate convolved sky of dust without noise
dust_map_in = QubicSkyObject[ic].get_fullsky_convolved_maps(FWHMdeg = None, verbose = False)
if verbose: print('=== Done {} map ===='.format(idict['filter_nu'] / 1e9, regions[ic//len(regions)][0]))
for i in range(idict['nf_recon']):
inband = (nus_in > nus_edge_out[i]) & (nus_in < nus_edge_out[i + 1])
foreground_maps[ic, ...] = np.mean(dust_map_in[inband, ...], axis=0)
elif idict['nf_sub'] == idict['nf_recon']:
# Now averaging maps into reconstruction sub-bands maps
foreground_maps[ic] = QubicSkyObject[ic].get_fullsky_convolved_maps(FWHMdeg = None, verbose = False)
return foreground_maps
def run_mc(nbmc,
Namaster,
d,
signoise,
cov,
effective_variance_invcov,
verbose=False,
clnoise=None):
ell_bins, b = Namaster.get_binning(d['nside'])
mask_apo = Namaster.mask_apo
okpix = cov > (np.max(cov) * 0.1)
myd = d.copy()
myd['nf_sub'] = 1
seed = np.random.randint(1, 100000)
sky_config = {'cmb': seed}
Qubic_sky = qss.Qubic_sky(sky_config, myd)
w = None
cl_noise_qubic = np.zeros((nbmc, len(ell_bins), 4))
print(' Starting MC')
for imc in range(nbmc):
t0 = time.time()
qubicnoiseA = Qubic_sky.create_noise_maps(
signoise,
cov,
effective_variance_invcov=effective_variance_invcov,
clnoise=clnoise)
qubicnoiseB = Qubic_sky.create_noise_maps(
signoise,
cov,
effective_variance_invcov=effective_variance_invcov,
clnoise=clnoise)
### Compute Spectra:
# Noise Only
if verbose: print(' - QUBIC Noise maps')
leff, cl_noise_qubic[imc, :, :], w = Namaster.get_spectra(
qubicnoiseA.T,
map2=qubicnoiseB.T,
purify_e=False,
purify_b=True,
w=w,
verbose=False,
beam_correction=True)
t1 = time.time()
print(
' Monte-Carlo: Iteration {0:} over {1:} done in {2:5.2f} sec'
.format(imc, nbmc, t1 - t0))
# average MC results
mcl_noise_qubic = np.mean(cl_noise_qubic, axis=0)
scl_noise_qubic = np.std(cl_noise_qubic, axis=0)
return leff, mcl_noise_qubic, scl_noise_qubic
def get_RMS_profile(residuals, config, nfsub, dirsave=None):
xx, yyI, yyQ, yyU = qss.get_angular_profile(residuals[0, :, :],
nbins=30,
separate=True,
center=center)
pix_size = hp.nside2resol(256, arcmin=True)
meanvalI = np.mean(yyI[xx < 10]) * pix_size
meanvalQU = np.mean((yyQ[xx < 10] + yyQ[xx < 10]) / 2) * pix_size
plt.figure()
plt.plot(xx, yyI * pix_size, 'o', label='I')
plt.plot(xx, yyQ * pix_size, 'o', label='Q')
plt.plot(xx, yyU * pix_size, 'o', label='U')
plt.axhline(y=meanvalI,
label=r'I RMS = {0:5.1f} $\mu K.arcmin$'.format(meanvalI),
color='r',
ls=':')
plt.axhline(y=meanvalQU,
label=r'QU RMS = {0:5.1f} $\mu K.arcmin$'.format(meanvalQU),
color='m',
ls=':')
plt.xlabel('Degrees from center of the field')
plt.ylabel(r'Noise RMS $[\mu K.arcmin]$')
plt.title('QUBIC End-To-End - ' + config + ' - Nptg = {}'.format(nptg))
plt.legend(fontsize=11)
plt.xlim(0, 20)
plt.ylim(0, meanvalQU * 2)
if dirsave is None:
plt.show()
else:
plt.savefig(dirsave + f'RMSprofile_{nfsub}bands_' + config + '.pdf',
format='pdf')
plt.close()
return
def ctheta_measurement(residuals,
coverage,
myfitcovs,
nfsub,
config,
alpha,
dirsave=None):
plt.figure(figsize=(16, 6))
fct = lambda x, a, b, c: a * np.sin(x / b) * np.exp(-x / c)
thth = np.linspace(0, 180, 1000)
allcth = []
allclth = []
allresults = []
pixgood = coverage > 0.1
for i in range(nfsub):
corrected_qubicnoise = qss.correct_maps_rms(residuals[i, :, :],
coverage, myfitcovs[i])
th, thecth = qss.ctheta_parts(corrected_qubicnoise[:, 0],
pixgood,
0,
20,
20,
nsplit=5,
degrade_init=128,
verbose=False)
okfit = np.isfinite(thecth)
results = curve_fit(fct,
th[okfit][1:], (thecth[okfit][1:] / thecth[0]),
maxfev=100000,
ftol=1e-7,
p0=[0, 1, 1])
allcth.append(thecth)
allresults.append(results)
plt.subplot(1, 2, 1)
p = plt.plot(th,
allcth[i] / allcth[i][0],
'o',
label='End-To-End Sub {}'.format(i + 1))
plt.plot(thth, fct(thth, *allresults[i][0]), color=p[0].get_color())
plt.axhline(y=0, color='k', ls=':')
plt.xlim(0, 20)
plt.legend(fontsize=9)
plt.xlabel(r'$\theta$ [deg]')
plt.ylabel(r'$C(\theta$)')
# ### Convert to Cl and display
ctheta = fct(thth, *allresults[i][0])
ctheta[0] = 1
lll, clth = qc.ctheta_2_cell(thth, ctheta, lmax=1024)
clth = (clth - 1) * alpha + 1
allclth.append(clth)
plt.subplot(1, 2, 2)
plt.plot(lll,
clth,
label='End-To-End Sub {}'.format(i + 1),
color=p[0].get_color())
plt.axhline(y=1, color='k', ls=':')
plt.xlabel(r'$\ell$')
plt.ylabel(r'$C_\ell$')
if dirsave is None:
plt.show()
else:
plt.savefig(dirsave + f'Ctheta_{nfsub}bands_' + config + '.pdf',
format='pdf')
plt.close()
return allresults, allcth, allclth, lll, clth
if config not in ['FI150', 'FI220']:
raise ValueError('The config should be FI150 or FI220.')
d['filter_nu'] = int(config[-3:]) * 1e9
# Central frequencies and FWHM of each band
_, _, nus, _, _, _ = compute_freq(int(config[-3:]), nbands)
print('nus:', nus)
fwhms = [d['synthbeam_peak150_fwhm'] * 150 / nu for nu in nus]
print('fwhms', fwhms)
# Input sky
seed = 42
# sky_config = {'dust': 'd1', 'cmb':seed, 'synchrotron':'s1'}
sky_config = {'dust': 'd1'}
Qubic_sky = qss.Qubic_sky(sky_config, d)
inputmaps = Qubic_sky.get_fullsky_convolved_maps(FWHMdeg=None, verbose=True)
rnd_name = qss.random_string(10)
# ================== Make maps =============================
# Getting noise realisations with FastSimulator
nreals = 4
npix = 12 * d['nside']**2
noisemaps = np.zeros((nreals, nbands, npix, 3))
Nyears = 3.
print('Nyears:', Nyears)
# qubic_coverage = np.load('/pbs/home/l/lmousset/libs/qubic/qubic/scripts/Spectroimagery_paper/maps/'
# 'coverage_nfsub15_nptgs10000_qubicpatch.npy')
# out_dir = sys.argv[1]
# if out_dir[-1] != '/':
# out_dir = out_dir + '/'
# os.makedirs(out_dir, exist_ok=True)
# name = sys.argv[2]
nf_sub = [3] #[2, 4, 5, 10, 12, 14, 15, 16, 18, 20, 22, 24]
# ============= Sky config =====================
# CMB map
# ell, totDL, unlensedDL = qc.get_camb_Dl(r=0., lmax=3*d['nside']-1)
# cmb_dict = {'CAMBSpectra':totDL, 'ell':ell, 'seed':None}
# sky_config = {'cmb': cmb_dict}
# Dust map
sky_config = {'dust': 'd1'}
# Synchrotron map
# sky_config = {'synchrotron': 's1'}
# ============== Make the sky =======================
for nf in nf_sub:
print(nf)
d['nf_sub'] = nf
Qubic_sky = qss.Qubic_sky(sky_config, d)
x0 = Qubic_sky.get_simple_sky_map()
# FitsArray(x0).save(out_dir + name + '_nside{}_nfsub{}.fits'.format(d['nside'], nf))
def generate_cmb_dust_maps(dico_fast_simulator,
coverage,
n_years,
noise_profile,
nunu,
sc,
seed=None,
save_maps=False,
return_maps=True,
dust_only=False,
fwhm_gen=None,
iib=True,
noise_covcut=None):
"""
Save CMB+Dust maps to FITS image format for later use, and/or return them immediately.
:param dico_fast_simulator: dictionary for FastSimulator at the desired frequency (150 or 220)
:param coverage: the sky coverage
:param int n_years: number of integration years
:param bool noise_profile: include noise profile (inhomogeneity)
:param bool nunu: include noise frequency correlations
:param bool sc: include noise spatial correlations
:param seed: seed for the map generation (if None, a random seed is taken)
:param bool|None save_maps: save maps in the FITS format (warning: check code first!)
:param bool|None return_maps: whether the function has to return the generated maps
:param bool|None dust_only: generate sky maps containing only dust (no cmb)
:param float|None fwhm_gen: smooth maps to this fwhm during generation
:param bool iib: integrate simulated maps into output bands
:param float|None noise_covcut: coverage cut when generating noise maps
:return: cmb+dust maps with noise, cmb+dust noiseless, noise only maps
"""
if seed is None:
seed = np.random.randint(1000000)
if dust_only:
sky_config = {
'dust': 'd0'
} # see d0 in https://pysm3.readthedocs.io/en/latest/models.html
else:
sky_config = {'dust': 'd0', 'cmb': seed}
qubic_sky = Qss.Qubic_sky(sky_config, dico_fast_simulator)
if noise_covcut is None and coverage is not None:
x, n = find_mantissa_exponent(np.min(coverage[coverage > 0]), 10)
noise_covcut = np.floor(x * 10) / 10**(n + 1)
if coverage is None: # maps are full-sky
coverage = np.ones(hp.nside2npix(dico_fast_simulator['nside']))
noise_covcut = 0.1 # arbitrary but None would raise an error in the FastSimulator
cmb_dust, cmb_dust_noiseless, cmb_dust_noise_only, coverage_eff = \
qubic_sky.get_partial_sky_maps_withnoise(coverage=coverage,
Nyears=n_years,
noise_profile=noise_profile, # noise inhomogeneity
nunu_correlation=nunu, # noise frequency correlations
spatial_noise=sc, # noise spatial correlations
verbose=False,
seed=None,
FWHMdeg=fwhm_gen,
integrate_into_band=iib,
noise_covcut=noise_covcut,
)
if save_maps:
save_dir = "/media/simon/CLE32/qubic/maps/"
if noise_profile:
save_dir += "with_noise_profile/"
else:
save_dir += "no_noise_profile/"
save_dir += "{:d}ghz/".format(
int(dico_fast_simulator['filter_nu'] / 1e9))
save_dir += "{}/".format(seed)
try:
os.mkdir(save_dir)
except FileExistsError:
pass
common_fmt = "{}bands_{}y" # .format(band, nsub, nyears, seed)
common = common_fmt.format(dico_fast_simulator['nf_recon'], n_years)
hdu_cmb_dust = fits.PrimaryHDU(cmb_dust)
hdu_cmb_dust.writeto(save_dir + common + "_cmbdust.fits",
overwrite=True)
hdu_cmb_dust_noiseless = fits.PrimaryHDU(cmb_dust_noiseless)
hdu_cmb_dust_noiseless.writeto(save_dir + common +
"_cmbdust_noiseless.fits",
overwrite=True)
hdu_cmb_dust_noise_only = fits.PrimaryHDU(cmb_dust_noise_only)
hdu_cmb_dust_noise_only.writeto(save_dir + common + "_noise.fits",
overwrite=True)
if return_maps:
return cmb_dust, cmb_dust_noiseless, cmb_dust_noise_only, coverage_eff
else:
return
def run_mc(nbmc,
Namaster,
cov,
d,
configs,
verbose=False,
clnoise=None,
duration=4,
beam=0.39268176):
#### Dictionnary for 150 GHz
dA = d.copy()
dA['effective_duration'] = duration
dA['nside'] = 256
dA['nf_sub'] = 1
dA['filter_nu'] = int(configs[0][-3:]) * 1e9
print(configs[0], dA['filter_nu'] / 1e9, dA['effective_duration'], 'Years')
dB = d.copy()
dB['nside'] = 256
dB['effective_duration'] = duration
dB['nf_sub'] = 1
dB['filter_nu'] = int(configs[1][-3:]) * 1e9
print(configs[1], dB['filter_nu'] / 1e9, dB['effective_duration'], 'Years')
ell_bins, b = Namaster.get_binning(d['nside'])
mask_apo = Namaster.mask_apo
okpix = cov > (np.max(cov) * 0.1)
seed = np.random.randint(1, 100000)
sky_config = {'cmb': seed}
Qubic_sky_A = qss.Qubic_sky(sky_config, dA)
Qubic_sky_B = qss.Qubic_sky(sky_config, dB)
w = None
cl_noise_qubic = np.zeros((nbmc, 1, len(ell_bins), 4))
print(' Starting MC')
for imc in range(nbmc):
t0 = time.time()
qubicnoiseA = Qubic_sky_A.get_partial_sky_maps_withnoise(
spatial_noise=True,
noise_only=True,
Nyears=dA['effective_duration'],
old_config=old_config)[0][0, :, :]
qubicnoiseB = Qubic_sky_B.get_partial_sky_maps_withnoise(
spatial_noise=True,
noise_only=True,
Nyears=dB['effective_duration'],
old_config=old_config)[0][0, :, :]
print(qubicnoiseA.shape)
### Compute Spectra:
# Noise Only
if verbose: print(' - QUBIC Noise maps')
leff, cl_noise_qubic[imc, 0, :, :], w = Namaster.get_spectra(
qubicnoiseA.T * np.sqrt(2),
map2=qubicnoiseB.T * np.sqrt(2),
purify_e=False,
purify_b=True,
w=w,
verbose=False,
beam_correction=beam,
pixwin_correction=True)
t1 = time.time()
print(
' Monte-Carlo: Iteration {0:} over {1:} done in {2:5.2f} sec'
.format(imc, nbmc, t1 - t0))
# average MC results
mcl_noise_qubic = np.mean(cl_noise_qubic, axis=0)[0]
scl_noise_qubic = np.std(cl_noise_qubic, axis=0)[0]
# The shape of cl_noise_qubic is : (#reals, #bands, #bins, 4)
print('Old shape:', cl_noise_qubic.shape)
cl_noise_qubic_reshape = np.moveaxis(cl_noise_qubic, [1, 2, 3], [3, 1, 2])
print('New shape:', cl_noise_qubic_reshape.shape)
# Covariance and correlation matrices for TT EE BB TE
covbin, corrbin = amc.get_covcorr_patch(cl_noise_qubic_reshape,
stokesjoint=True,
doplot=False)
return leff, mcl_noise_qubic, scl_noise_qubic, covbin
d['nf_sub'] = 1 ### this is OK as we use noise-only simulations
### Bands
dA = d.copy()
dA['filter_nu'] = int(configs[0][-3:]) * 1e9
print('Frequency Band for A:', dA['filter_nu'] / 1e9)
dB = d.copy()
dB['filter_nu'] = int(configs[1][-3:]) * 1e9
print('Frequency Band for B', dB['filter_nu'] / 1e9)
### Sky
## Make a sky using PYSM: It will have the expected QUBIC beam, the coverage and noise according to this coverage
## This creates a realization of the sky (a realization of the CMB is there is CMB in sly_config)
seed = np.random.randint(1)
sky_config = {'cmb': seed}
Qubic_sky_A = qss.Qubic_sky(sky_config, dA)
Qubic_sky_B = qss.Qubic_sky(sky_config, dB)
maptest, coverageA = Qubic_sky_A.get_partial_sky_maps_withnoise(
spatial_noise=True,
noise_only=True,
Nyears=duration,
old_config=old_config)
################################### Flat Weighting #################################################################
### Create a Namaster object
cov = coverageA.copy()
lmax = 2 * d['nside'] - 1
okpix = cov > np.max(cov) * covcut
### We use Flat weighting