def cov_from_maps(maps0,maps1):
sh=np.shape(maps0)
npix=sh[1]
nbmc=sh[0]
covmc=np.zeros((npix,npix))
mm0=np.mean(maps0,axis=0)
mm1=np.mean(maps1,axis=0)
themaps0=np.zeros((nbmc,npix))
themaps1=np.zeros((nbmc,npix))
for i in np.arange(npix):
pyquad.progress_bar(i,npix)
themaps0[:,i]=maps0[:,i]-mm0[i]
themaps1[:,i]=maps1[:,i]-mm1[i]
for i in np.arange(npix):
pyquad.progress_bar(i,npix)
for j in np.arange(npix):
covmc[i,j]=np.mean(themaps0[:,i]*themaps1[:,j])
#covmc[j,i]=covmc[i,j]
return(covmc)
nbpixok=50
mask=(np.arange(12*nside**2) >= nbpixok)
#### maps simulation
nbmc=1000
npix=np.size(where(mask==False))
allmapsi1=np.zeros((nbmc,npix))
allmapsq1=np.zeros((nbmc,npix))
allmapsu1=np.zeros((nbmc,npix))
allmapsi2=np.zeros((nbmc,npix))
allmapsq2=np.zeros((nbmc,npix))
allmapsu2=np.zeros((nbmc,npix))
sigI=200.
sigQU=10.
for i in np.arange(nbmc):
pyquad.progress_bar(i,nbmc)
mapi,mapq,mapu=hp.synfast(spectra[1:],nside,fwhm=0,pixwin=True,new=True)
noisei1=np.random.randn(12*nside**2)*sigI
noiseq1=np.random.randn(12*nside**2)*sigQU
noiseu1=np.random.randn(12*nside**2)*sigQU
noisei2=np.random.randn(12*nside**2)*sigI
noiseq2=np.random.randn(12*nside**2)*sigQU
noiseu2=np.random.randn(12*nside**2)*sigQU
allmapsi1[i,:]=mapi[~mask]+noisei1[~mask]
allmapsq1[i,:]=mapq[~mask]+noiseq1[~mask]
allmapsu1[i,:]=mapu[~mask]+noiseu1[~mask]
allmapsi2[i,:]=mapi[~mask]+noisei2[~mask]
allmapsq2[i,:]=mapq[~mask]+noiseq2[~mask]
allmapsu2[i,:]=mapu[~mask]+noiseu2[~mask]
############ Auto
def qml(maps,mask,covmap,ellbins,fwhmrad,guess,ds_dcb,spectra,itmax=20,plot=False,cholesky=True,polar=True,temp=True,EBTB=False):
print('QML Estimator')
###### What will be calculated ?
print('Temp='+str(temp))
print('Polar='+str(polar))
print('EBTB='+str(EBTB))
all=['I','Q','U']
if EBTB:
xx=['TT','EE','BB','TE','EB','TB']
ind=[1,2,3,4,5,6]
else:
xx=['TT','EE','BB','TE']
ind=[1,2,3,4]
if not temp:
all=['Q','U']
if EBTB:
xx=['EE','BB','EB']
ind=[2,3,5]
else:
xx=['EE','BB']
ind=[2,3]
if not polar:
all=['I']
xx=['TT']
ind=[1]
print('Number of maps :',len(maps))
print('Stokes parameters :',all)
print('Spectra considered. :',xx)
nspec=len(xx)
if len(maps) != len(all):
print('Inconstitent number of maps and Stokes parameters: Exiting !')
return -1
### maps information
nside=hp.npix2nside(len(maps[0]))
ip=np.arange(12*nside**2)
ipok=ip[~mask]
npix=len(ipok)
npixtot=len(all)*npix
masktot=np.zeros(len(all)*len(mask),dtype=bool)
map=np.zeros(len(all)*npix)
for i in np.arange(len(all)):
map[i*npix:(i+1)*npix]=maps[i][~mask]
masktot[i*len(mask):(i+1)*len(mask)]=mask
### ell binning
nbins=len(ellbins)-1
ellmin=np.array(ellbins[0:nbins])
ellmax=np.array(ellbins[1:nbins+1])-1
ellval=(ellmin+ellmax)*1./2
ll=np.arange(int(np.max(ellbins))+1)
bl=np.exp(-0.5*ll**2*(fwhmrad/2.35)**2)
### Initial guess
nbinstot=nspec*nbins
specinit=np.zeros(nbinstot)
for s in np.arange(nspec):
specinit[s*nbins:(s+1)*nbins]=guess[ind[s]]
thespectrum=np.zeros((nbinstot,itmax+1))
thespectrum[:,0]=specinit
####################### P1 dS/dC Calculation #################################
if ds_dcb is 0:
ds_dcb=compute_ds_dcb_par(ellbins,nside,ipok,bl,polar=polar,temp=temp,EBTB=EBTB)
##############################################################################
stop
### Start iterations
num=0
convergence=0
lk=np.zeros(itmax)
while convergence==0:
##########################################################################
## P2 pixpix covariance matrix for maps and solving z=M-1.m and likelihood
print(' P2')
matsky=np.zeros((npixtot,npixtot))
for s in np.arange(nspec):
for i in np.arange(nbins):
matsky += thespectrum[s*nbins+i,num]*ds_dcb[s,i,:,:]
print(' - Done matsky')
matcov=covmap+matsky
if cholesky is True:
print(' Doing Cholesky decomposition')
U=scipy.linalg.cho_factor(matcov)
z=scipy.linalg.cho_solve(U,map)
else:
print(' Brute force inversion (No Cholesky)')
minv=scipy.linalg.inv(matcov)
z=np.dot(minv,map)
mapw=np.zeros(map.size)
mapw=z
lk[num]=-0.5*(np.dot(z,map)+np.sum(np.log(np.diag(matcov))))
##########################################################################
########################################################################
# P3 Solve the equations Wb=M^(-1).ds_dcb ##############################
print(' P3')
wb=np.zeros((nbinstot,npixtot,npixtot))
for s in np.arange(nspec):
pyquad.progress_bar(s,nspec)
for i in np.arange(nbins):
if cholesky is True:
wb[s*nbins+i,:,:]=scipy.linalg.cho_solve(U,ds_dcb[s,i,:,:])
else:
wb[s*nbins+i,:,:]=np.dot(minv,ds_dcb[s,i,:,:])
########################################################################
########################################################################
# P4 First derivatives of the likelihood ##############################
print(' P4')
dldcb=np.zeros(nspec*nbins)
for i in np.arange(nbinstot):
pyquad.progress_bar(i,nbinstot)
dldcb[i]=0.5*(np.dot(map,np.dot(wb[i,:,:],z))-np.trace(wb[i,:,:]))
#######################################################################
##########################################################################
# P5 second derivatives of the likelihood ################################
print(' P5')
d2ldcbdcb=np.zeros((nspec*nbins,nspec*nbins))
fisher=np.zeros((nspec*nbins,nspec*nbins))
kk=0
for i in np.arange(nbinstot):
for j in np.arange(i,nbinstot):
pyquad.progress_bar(kk,(nbinstot+1)*nbinstot/2)
wbwb=np.dot(wb[i,:,:],wb[j,:,:])
fisher[i,j]=np.trace(wbwb)
d2ldcbdcb[i,j]=-np.dot(map,np.dot(wbwb,z))+0.5*fisher[i,j]
fisher[j,i]=fisher[i,j]
d2ldcbdcb[j,i]=d2ldcbdcb[i,j]
kk+=1
###########################################################################
###########################################################################
# P6 Correction to be applied to the spectrum #############################
print(' P6')
invfisher=(np.linalg.inv(fisher))
err=np.sqrt(np.diag(invfisher))
deltac=-np.dot(np.linalg.inv(d2ldcbdcb),dldcb)
print(' Result for iteration',num)
print(' C : ',min(thespectrum[:,num]),max(thespectrum[:,num]))
print(' dC : ',min(deltac),max(deltac))
newspectrum=(thespectrum[:,num]+deltac)
deltac[newspectrum < 0]=0
thespectrum[:,num+1]=(thespectrum[:,num]+deltac)
############################################################################
spec=thespectrum[:,num]
finalspectrum=[ellval,0,0,0,0,0,0]
error=[ellval,0,0,0,0,0,0]
for s in np.arange(nspec):
finalspectrum[ind[s]]=spec[s*nbins:(s+1)*nbins]
error[ind[s]]=err[s*nbins:(s+1)*nbins]
############################################################################
# Critere de convergence ###################################################
conv=deltac/np.sqrt(np.abs(np.diag(invfisher)))
print(' Conv : ',max(abs(conv)))
print(' Likelihood : ',lk[num])
nplot_tot=1+nspec
sqrn=int(np.sqrt(nplot_tot))
if sqrn**2==nplot_tot:
nr=sqrn
nc=sqrn
else:
nr=sqrn+1
nc=sqrn
iplot=0
if plot is True:
mp.clf()
mp.subplot(nr,nc,iplot+1)
for s in np.arange(nspec):
mp.xlim(0,np.max(ellmax)*1.2)
ell=spectra[0]
cl=spectra[ind[s]]
ellb=finalspectrum[0]
mp.plot(ell,cl*(ell*(ell+1))/(2*np.pi),lw=3)
mp.plot(ellb,guess[ind[s]]*ellb*(ellb+1)/(2*np.pi),'go',alpha=0.5)
mp.xlabel('$\ell$')
mp.ylabel('$\ell(\ell+1)C_\ell/(2\pi)$')
mp.errorbar(ellb,finalspectrum[ind[s]]*ellb*(ellb+1)/(2*np.pi),error[ind[s]]*ellb*(ellb+1)/(2*np.pi),xerr=(ellmax+1-ellmin)/2,label=str(i),fmt='ro')
mp.draw()
mp.title(xx[s])
iplot+=1
mp.subplot(nr,nc,iplot+1)
if num>0: mp.plot(lk[:num],'o-')
mp.xlabel('Iteration')
mp.ylabel('Likelihood')
mp.draw()
if np.max(np.abs(conv)) <= 0.01 or num==itmax-1:
convergence=1
else:
num=num+1
###########################################################################
return finalspectrum,error,invfisher,lk,num,ds_dcb
def covth(nside,ipok,lmax,bl,spectra,polar=True,temp=True,allinone=True):
all=['I','Q','U']
if not temp:
all=['Q','U']
if not polar:
all=['I']
rpix=np.array(hp.pix2vec(nside,ipok))
allcosang=np.dot(np.transpose(rpix),rpix)
pixwin=hp.pixwin(nside)[0:lmax+1]
nspec,nell=np.shape(spectra)
ell=spectra[0]
maskl=ell<(lmax+1)
ell=ell[maskl]
ctt=spectra[1][maskl]
cee=spectra[2][maskl]
cbb=spectra[3][maskl]
cte=spectra[4][maskl]
if nspec==5:
print('Assuming EB and TB are zero')
ceb=0
ctb=0
else:
ceb=spectra[5][maskl]
ctb=spectra[6][maskl]
norm=(2*ell+1)/(4*np.pi)*(pixwin**2)*(bl[maskl]**2)
effctt=norm*ctt
effcte=norm*cte
effcee=norm*cee
effcbb=norm*cbb
effceb=norm*ceb
effctb=norm*ctb
nbpixok=ipok.size
nstokes=np.size(all)
print(nstokes)
cov=np.zeros((nstokes,nstokes,nbpixok,nbpixok))
for i in np.arange(nbpixok):
pyquad.progress_bar(i,nbpixok)
for j in np.arange(i,nbpixok):
if nstokes==1:
pl=pl0(allcosang[i,j],lmax)
TT=effctt*pl
cov[0,0,i,j]=np.sum(TT)
cov[0,0,j,i]=cov[0,0,i,j]
if nstokes==2:
cij,sij=polrotangle(rpix[:,i],rpix[:,j])
cji,sji=polrotangle(rpix[:,j],rpix[:,i])
f12=F1l2(allcosang[i,j],lmax)
f22=F2l2(allcosang[i,j],lmax)
QQ=f12*effcee-f22*effcbb
UU=f12*effcbb-f22*effcee
QU=(f12+f22)*effceb
cQQpsQU = ( cij*QQ + sij*QU )
cQUpsUU = ( cij*QU + sij*UU )
cQUmsQQ = ( cij*QU - sij*QQ )
cUUmsQU = ( cij*UU - sij*QU )
cov[0,0,i,j] = np.sum( cji*cQQpsQU + sji*cQUpsUU )
cov[0,0,j,i] = cov[0,0,i,j]
cov[0,1,i,j] = np.sum(-sji*cQQpsQU + cji*cQUpsUU )
cov[1,0,j,i] = cov[0,1,i,j]
cov[1,1,i,j] = np.sum(-sji*cQUmsQQ + cji*cUUmsQU )
cov[1,1,j,i] = cov[1,1,i,j]
cov[0,1,j,i] = np.sum( cji*cQUmsQQ + sji*cUUmsQU )
cov[1,0,i,j] = cov[0,1,j,i]
if nstokes==3:
cij,sij=polrotangle(rpix[:,i],rpix[:,j])
cji,sji=polrotangle(rpix[:,j],rpix[:,i])
pl=pl0(allcosang[i,j],lmax)
f10=F1l0(allcosang[i,j],lmax)
f12=F1l2(allcosang[i,j],lmax)
f22=F2l2(allcosang[i,j],lmax)
TT=effctt*pl
QQ=f12*effcee-f22*effcbb
UU=f12*effcbb-f22*effcee
TQ=-f10*effcte
TU=-f10*effctb
QU=(f12+f22)*effceb
cQQpsQU = ( cij*QQ + sij*QU )
cQUpsUU = ( cij*QU + sij*UU )
cQUmsQQ = ( cij*QU - sij*QQ )
cUUmsQU = ( cij*UU - sij*QU )
cov[0,0,i,j] = np.sum( TT )
cov[0,0,j,i] = cov[0,0,i,j]
cov[0,1,i,j] = np.sum( cji*TQ + sji*TU )
cov[1,0,j,i] = cov[0,1,i,j]
cov[1,0,i,j] = np.sum( cij*TQ + sij*TU )
cov[0,1,j,i] = cov[1,0,i,j]
cov[0,2,i,j] = np.sum(-sji*TQ + cij*TU )
cov[2,0,j,i] = cov[0,2,i,j]
cov[2,0,i,j] = np.sum(-sij*TQ + cij*TU )
cov[0,2,j,i] = cov[2,0,i,j]
cov[1,1,i,j] = np.sum( cji*cQQpsQU + sji*cQUpsUU )
cov[1,1,j,i] = cov[1,1,i,j]
cov[1,2,i,j] = np.sum(-sji*cQQpsQU + cji*cQUpsUU )
cov[2,1,j,i] = cov[1,2,i,j]
cov[2,1,i,j] = np.sum( cji*cQUmsQQ + sji*cUUmsQU )
cov[1,2,j,i] = cov[2,1,i,j]
cov[2,2,i,j] = np.sum(-sji*cQUmsQQ + cji*cUUmsQU )
cov[2,2,j,i] = cov[2,2,i,j]
if allinone==True:
return(allmat2bigmat(cov))
else:
return cov
def covth_bins(ellbins,nside,ipok,bl,spectra,polar=True,temp=True,allinone=True):
#### define bins in ell
nbins=len(ellbins)-1
minell=np.array(ellbins[0:nbins])
maxell=np.array(ellbins[1:nbins+1])-1
ellval=(minell+maxell)*0.5
lmax=np.max(ellbins)
#maxell[nbins-1]=lmax
print('minell:',minell)
print('maxell:',maxell)
#### define Stokes
all=['I','Q','U']
if not temp:
all=['Q','U']
if not polar:
all=['I']
#### define pixels
print('rpix calculation')
rpix=np.array(hp.pix2vec(nside,ipok))
print('rpix done: ',rpix.shape)
allcosang=np.dot(np.transpose(rpix),rpix)
print('cosang done')
#### define Pixel window function
pixwin=hp.pixwin(nside)[0:lmax+1]
#### get ell values from spectra and restrict spectra to good values
nspec,nell=np.shape(spectra)
ell=spectra[0]
maskl=ell<(lmax+1)
ell=ell[maskl]
ctt=spectra[1][maskl]
cee=spectra[2][maskl]
cbb=spectra[3][maskl]
cte=spectra[4][maskl]
if nspec==5:
print('Assuming EB and TB are zero')
ceb=0
ctb=0
else:
ceb=spectra[5][maskl]
ctb=spectra[6][maskl]
# Effective spectra
print('Calculating effective spectra')
norm=(2*ell+1)/(4*np.pi)*(pixwin**2)*(bl[maskl]**2)
norm[1:]=norm[1:]/(ell[1:]*(ell[1:]+1))
effctt=norm*ctt
effcte=norm*cte
effcee=norm*cee
effcbb=norm*cbb
effceb=norm*ceb
effctb=norm*ctb
#### define masks for ell bins
masks=[]
for i in np.arange(nbins):
masks.append((ell>=minell[i]) & (ell<=maxell[i]))
### Create array for covariances matrices per bin
nbpixok=ipok.size
nstokes=np.size(all)
print('creating array')
cov=np.zeros((nbins,nstokes,nstokes,nbpixok,nbpixok))
for i in np.arange(nbpixok):
print(i)
pyquad.progress_bar(i,nbpixok)
for j in np.arange(i,nbpixok):
if nstokes==1:
pl=pl0(allcosang[i,j],lmax)
TT=effctt*pl
for b in np.arange(nbins):
cov[b,0,0,i,j]=np.sum(TT[masks[b]])*ellval[b]*(ellval[b]+1)
cov[b,0,0,j,i]=cov[b,0,0,i,j]
if nstokes==2:
cij,sij=polrotangle(rpix[:,i],rpix[:,j])
cji,sji=polrotangle(rpix[:,j],rpix[:,i])
f12=F1l2(allcosang[i,j],lmax)
f22=F2l2(allcosang[i,j],lmax)
QQ=f12*effcee-f22*effcbb
UU=f12*effcbb-f22*effcee
QU=(f12+f22)*effceb
cQQpsQU = ( cij*QQ + sij*QU )
cQUpsUU = ( cij*QU + sij*UU )
cQUmsQQ = ( cij*QU - sij*QQ )
cUUmsQU = ( cij*UU - sij*QU )
for b in np.arange(nbins):
cov[b,0,0,i,j] = np.sum( cji*cQQpsQU[masks[b]] + sji*cQUpsUU[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,0,0,j,i] = cov[b,0,0,i,j]
cov[b,0,1,i,j] = np.sum(-sji*cQQpsQU[masks[b]] + cji*cQUpsUU[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,1,0,j,i] = cov[b,0,1,i,j]
cov[b,1,1,i,j] = np.sum(-sji*cQUmsQQ[masks[b]] + cji*cUUmsQU[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,1,1,j,i] = cov[b,1,1,i,j]
cov[b,0,1,j,i] = np.sum( cji*cQUmsQQ[masks[b]] + sji*cUUmsQU[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,1,0,i,j] = cov[b,0,1,j,i]
if nstokes==3:
cij,sij=polrotangle(rpix[:,i],rpix[:,j])
cji,sji=polrotangle(rpix[:,j],rpix[:,i])
pl=pl0(allcosang[i,j],lmax)
f10=F1l0(allcosang[i,j],lmax)
f12=F1l2(allcosang[i,j],lmax)
f22=F2l2(allcosang[i,j],lmax)
TT=effctt*pl
QQ=f12*effcee-f22*effcbb
UU=f12*effcbb-f22*effcee
TQ=-f10*effcte
TU=-f10*effctb
QU=(f12+f22)*effceb
cQQpsQU = ( cij*QQ + sij*QU )
cQUpsUU = ( cij*QU + sij*UU )
cQUmsQQ = ( cij*QU - sij*QQ )
cUUmsQU = ( cij*UU - sij*QU )
for b in np.arange(nbins):
cov[b,0,0,i,j] = np.sum( TT[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,0,0,j,i] = cov[b,0,0,i,j]
cov[b,0,1,i,j] = np.sum( cji*TQ[masks[b]] + sji*TU[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,1,0,j,i] = cov[b,0,1,i,j]
cov[b,1,0,i,j] = np.sum( cij*TQ[masks[b]] + sij*TU[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,0,1,j,i] = cov[b,1,0,i,j]
cov[b,0,2,i,j] = np.sum(-sji*TQ[masks[b]] + cij*TU[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,2,0,j,i] = cov[b,0,2,i,j]
cov[b,2,0,i,j] = np.sum(-sij*TQ[masks[b]] + cij*TU[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,0,2,j,i] = cov[b,2,0,i,j]
cov[b,1,1,i,j] = np.sum( cji*cQQpsQU[masks[b]] + sji*cQUpsUU[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,1,1,j,i] = cov[b,1,1,i,j]
cov[b,1,2,i,j] = np.sum(-sji*cQQpsQU[masks[b]] + cji*cQUpsUU[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,2,1,j,i] = cov[b,1,2,i,j]
cov[b,2,1,i,j] = np.sum( cji*cQUmsQQ[masks[b]] + sji*cUUmsQU[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,1,2,j,i] = cov[b,2,1,i,j]
cov[b,2,2,i,j] = np.sum(-sji*cQUmsQQ[masks[b]] + cji*cUUmsQU[masks[b]] )*ellval[b]*(ellval[b]+1)
cov[b,2,2,j,i] = cov[b,2,2,i,j]
if allinone:
newcov=np.zeros((nbins,nstokes*nbpixok,nstokes*nbpixok))
for i in np.arange(nbins):
for si in np.arange(nstokes):
for sj in np.arange(nstokes):
newcov[i,si*nbpixok:(si+1)*nbpixok,sj*nbpixok:(sj+1)*nbpixok]=cov[i,si,sj,:,:]
return(newcov)
else:
return(cov)
print(npix)
#### noise covariance matrix
# get it through MC
nbmc=10000
allnoisemaps=np.zeros((npix,nbmc))
for i in np.arange(nbmc):
print(i,nbmc)
todnoise = tod*0 + np.random.randn(ndet,nsamples)*signoise
solution = pcg(P_packed.T * P_packed, P_packed.T(todnoise), M=DiagonalOperator(1/coverage[~mask]))
allnoisemaps[:,i]=unpack(solution['x'])[~mask]
covmc=np.zeros((npix,npix))
mm=np.mean(allnoisemaps,axis=-1)
for i in np.arange(npix):
pyquad.progress_bar(i,npix)
mm0=allnoisemaps[i,:]-mm[i]
for j in np.arange(i,npix):
covmc[i,j]=np.mean( mm0*(allnoisemaps[j,:]-mm[j]))
covmc[j,i]=covmc[i,j]
cormc=pyquad.cov2cor(covmc)
from pysimulators import FitsArray
FitsArray(covmc,copy=False).save('covmc'+str(signoise)+'_'+str(nbmc)+'.dat')
FitsArray(cormc,copy=False).save('cormc'+str(signoise)+'_'+str(nbmc)+'.dat')
