def window_fn_matrix(Q,N,num_remov=None,save_tag=None,lms=None):
Q = n.matrix(Q); N = n.matrix(N)
Ninv = uf.pseudo_inverse(N,num_remov=None) # XXX want to remove dynamically
#print Ninv
info = n.dot(Q.H,n.dot(Ninv,Q))
M = uf.pseudo_inverse(info,num_remov=num_remov)
W = n.dot(M,info)
if save_tag!=None:
foo = W[0,:]
foo = n.real(n.array(foo))
foo.shape = (foo.shape[1]),
print foo.shape
p.scatter(lms[:,0],foo,c=lms[:,1],cmap=mpl.cm.PiYG,s=50)
p.xlabel('l (color is m)')
p.ylabel('W_0,lm')
p.title('First Row of Window Function Matrix')
p.colorbar()
p.savefig('{0}/{1}_W.pdf'.format(fig_loc,save_tag))
p.clf()
print 'W ',W.shape
p.imshow(n.real(W))
p.title('Window Function Matrix')
p.colorbar()
p.savefig('{0}/{1}_W_im.pdf'.format(fig_loc,save_tag))
p.clf()
return W
def error_covariance(Q,N):
Q = n.matrix(Q)
N = n.matrix(N)
Ninv = uf.pseudo_inverse(N,num_remov=None)#N.shape[0]-5)
info = n.dot(Q.H,n.dot(Ninv,Q))
err_cov = uf.pseudo_inverse(info)
err_cov = n.array(err_cov)
return err_cov
def test_invertible(self):
M = n.diag(n.arange(1,9,dtype='float'))
Minv = uf.pseudo_inverse(M,num_remov=0)
self.assertTrue(n.all(Minv==n.where(M>0,1/M,0)))
Minv = uf.pseudo_inverse(M,num_remov=2)
#print 'remov 2 ', Minv
self.assertTrue(n.all(Minv==n.diag(n.array([0,0,1/3.,1/4.,1/5.,1/6.,1/7.,1/8.]))))
def return_MQdagNinv(Q,N,num_remov=None):
Q = n.matrix(Q); N = n.matrix(N)
Ninv = uf.pseudo_inverse(N,num_remov=None) # XXX want to remove dynamically
info = n.dot(Q.H,n.dot(Ninv,Q))
M = uf.pseudo_inverse(info,num_remov=num_remov)
MQN = n.dot(M,n.dot(Q.H,Ninv))
#print n.diag(N)
return MQN
def test_pseudo_remov(self):
#M = n.diag(n.array([100,10,1,0.1,0.01,0.001]))
M = n.diag(n.array([0.00001,0.0001,0.001,0.01,0.1,1])[::-1])
Minv = uf.pseudo_inverse(M,num_remov=None)
print M
print Minv
Minvp = uf.pseudo_inverse(M,num_remov=2)
print Minvp
self.assertTrue(n.allclose(Minv,Minvp))
def test_pseudo_remov(self):
#M = n.diag(n.array([100,10,1,0.1,0.01,0.001]))
M = n.diag(n.array([0.00001, 0.0001, 0.001, 0.01, 0.1, 1])[::-1])
Minv = uf.pseudo_inverse(M, num_remov=None)
print M
print Minv
Minvp = uf.pseudo_inverse(M, num_remov=2)
print Minvp
self.assertTrue(n.allclose(Minv, Minvp))
def return_ahat(y,Q,N,num_remov=None):
assert len(y.shape)==1
Q = n.matrix(Q); N = n.matrix(N)
Ninv = uf.pseudo_inverse(N,num_remov=None) # XXX want to remove dynamically
info = n.dot(Q.H,n.dot(Ninv,Q))
M = uf.pseudo_inverse(info,num_remov=num_remov)
#print Ninv.shape, y.shape
ahat = uf.vdot(M,uf.vdot(Q.H,uf.vdot(Ninv,y)))
assert len(ahat.shape)==1
return ahat
def return_MQdagNinv(Q, N, num_remov=None):
Q = n.matrix(Q)
N = n.matrix(N)
Ninv = uf.pseudo_inverse(N,
num_remov=None) # XXX want to remove dynamically
info = n.dot(Q.H, n.dot(Ninv, Q))
M = uf.pseudo_inverse(info, num_remov=num_remov)
MQN = n.dot(M, n.dot(Q.H, Ninv))
#print n.diag(N)
return MQN
def test_invertible(self):
M = n.diag(n.arange(1, 9, dtype='float'))
Minv = uf.pseudo_inverse(M, num_remov=0)
self.assertTrue(n.all(Minv == n.where(M > 0, 1 / M, 0)))
Minv = uf.pseudo_inverse(M, num_remov=2)
#print 'remov 2 ', Minv
self.assertTrue(
n.all(Minv == n.diag(
n.array([0, 0, 1 / 3., 1 / 4., 1 / 5., 1 / 6., 1 / 7., 1 /
8.]))))
def test_pseudo_inverse(self):
M = n.diag(n.arange(1,9))
Z = n.zeros((8,2))
Z[0,0] = 1; Z[1,1]=1
PP = uf.projection_matrix(Z)
M_tilde = n.dot(PP.H,n.dot(M,PP))
Minv = uf.pseudo_inverse(M,num_remov=2)
M_id = n.dot(Minv,M_tilde)
proj_id = n.dot(PP.H,n.dot(n.identity(8),PP))
self.assertTrue(n.all(M_id==proj_id))
def window_function_matrix(Q, N, lms, save_tag=None):
ls = n.arange(max(lms[:, 0]) + 1)
l_locs = ls * ls
M = n.matrix(n.zeros_like(Q))
Q = n.matrix(Q)
N = n.matrix(N)
Ninv = N.I
info = Q.H * Ninv * Q
p.imshow(n.log(n.absolute(info)))
p.title('Info Matrix')
p.xticks(l_locs, ls)
p.yticks(l_locs, ls)
p.xlabel('l')
p.ylabel('l')
p.savefig('./figures/{0}_info_matrix_ufpseudo.pdf'.format(save_tag))
#p.show()
p.clf()
# for ii in range(M.shape[0]):
# M[ii,ii] = 1/info[ii,ii]
#M = n.linalg.pinv(info)
num_remov = 10
M = uf.pseudo_inverse(info, num_remov=num_remov)
#n.set_printoptions(threshold='nan')
#print M*info
#BB = n.absolute(n.absolute(M*info)-n.identity(M.shape[0]))
#print n.amax(BB)
W = M * info
#print M[0:4,0:4]
#print info[0:4,0:4]
#print W[0:4,0:4]
p.imshow(n.log(n.absolute(W)))
p.title('Window Function Matrix')
p.xticks(l_locs, ls)
p.yticks(l_locs, ls)
p.xlabel('l')
p.ylabel('l')
p.savefig('./figures/{0}_W_matrix_ufpseudo_remov_{1}.pdf'.format(
save_tag, num_remov))
#p.show()
p.clf()
foo = n.array(n.absolute(W[0, :]))
p.scatter(lms[:, 0], foo, c=lms[:, 1], cmap=mpl.cm.PiYG, s=50)
#p.yscale('log')
#p.ylim([10**-3,10**0.2])
p.xlabel('l (color is m)')
p.ylabel('first row of Window Function Matrix')
p.colorbar()
p.savefig('./figures/{0}_W_pinv_matrix_elements_remov_{1}.pdf'.format(
save_tag, num_remov))
#p.show()
p.clf()
def test_pseudo_inverse(self):
M = n.diag(n.arange(1, 9))
Z = n.zeros((8, 2))
Z[0, 0] = 1
Z[1, 1] = 1
PP = uf.projection_matrix(Z)
M_tilde = n.dot(PP.H, n.dot(M, PP))
Minv = uf.pseudo_inverse(M, num_remov=2)
M_id = n.dot(Minv, M_tilde)
proj_id = n.dot(PP.H, n.dot(n.identity(8), PP))
self.assertTrue(n.all(M_id == proj_id))
def window_function_matrix(Q,N,lms,save_tag=None):
ls = n.arange(max(lms[:,0])+1)
l_locs = ls*ls
M = n.matrix(n.zeros_like(Q))
Q = n.matrix(Q); N = n.matrix(N)
Ninv = N.I
info = Q.H*Ninv*Q
p.imshow(n.log(n.absolute(info)))
p.title('Info Matrix')
p.xticks(l_locs,ls)
p.yticks(l_locs,ls)
p.xlabel('l')
p.ylabel('l')
p.savefig('./figures/{0}_info_matrix_ufpseudo.pdf'.format(save_tag))
#p.show()
p.clf()
# for ii in range(M.shape[0]):
# M[ii,ii] = 1/info[ii,ii]
#M = n.linalg.pinv(info)
num_remov = 10
M = uf.pseudo_inverse(info,num_remov=num_remov)
#n.set_printoptions(threshold='nan')
#print M*info
#BB = n.absolute(n.absolute(M*info)-n.identity(M.shape[0]))
#print n.amax(BB)
W = M*info
#print M[0:4,0:4]
#print info[0:4,0:4]
#print W[0:4,0:4]
p.imshow(n.log(n.absolute(W)))
p.title('Window Function Matrix')
p.xticks(l_locs,ls)
p.yticks(l_locs,ls)
p.xlabel('l')
p.ylabel('l')
p.savefig('./figures/{0}_W_matrix_ufpseudo_remov_{1}.pdf'.format(save_tag,num_remov))
#p.show()
p.clf()
foo = n.array(n.absolute(W[0,:]))
p.scatter(lms[:,0],foo,c=lms[:,1],cmap=mpl.cm.PiYG,s=50)
#p.yscale('log')
#p.ylim([10**-3,10**0.2])
p.xlabel('l (color is m)')
p.ylabel('first row of Window Function Matrix')
p.colorbar()
p.savefig('./figures/{0}_W_pinv_matrix_elements_remov_{1}.pdf'.format(save_tag,num_remov))
#p.show()
p.clf()
def return_MQdagNinv(Q,N,mode,savekey,fq,Nfg_type,eps=10**-4,num_remov=None):
#Q = n.matrix(Q); N = n.matrix(N)
#Ninv = uf.pseudo_inverse(N,num_remov=None) # XXX want to remove dynamically
Ninv = n.linalg.inv(N)
info = n.dot(n.conjugate(Q.T),n.dot(Ninv,Q))
#info = n.dot(Q.H,n.dot(Ninv,Q))
eigens = n.linalg.eigvalsh(info).real
diags = n.diagonal(info).real
plotTitleBase = '{0}_fq_{1:.3f}'.format(savekey,fq)
f, ax = p.subplots(figsize=(8,6), dpi=400)
ax.plot(eigens)
ax.set_ylabel("Information matrix eigenvalues", fontsize=20, fontname="Times New Roman")
ax.set_xlabel("Eigenvalue number", fontsize=20, fontname="Times New Roman")
ax.set_title('Info_scree_\nN_fg_{1}_\n{0}'.format(plotTitleBase,Nfg_type))
f.savefig('{0}/plots/{1}/Info_scree_Nfg_{3}_{2}.pdf'.format(data_loc,savekey,plotTitleBase,Nfg_type))
f, ax = p.subplots(figsize=(8,6), dpi=400)
ax.plot(diags)
ax.set_ylabel("Information matrix diagonals", fontsize=20, fontname="Times New Roman")
ax.set_xlabel("Element number", fontsize=20, fontname="Times New Roman")
ax.set_title('Info_diagonal_\nN_fg_{1}_\n{0}'.format(plotTitleBase,Nfg_type))
f.savefig('{0}/plots/{1}/Info_diagonal_Nfg_{3}_{2}.pdf'.format(data_loc,savekey,plotTitleBase,Nfg_type))
if mode == "pseudo":
M = uf.pseudo_inverse(info,num_remov=num_remov)
MQN = n.dot(M,n.dot(n.conjugate(Q.T),Ninv))
MQN = n.array(MQN)
elif mode == "regularization":
info += eps**2 * n.identity(info.shape[0])
MQN = n.dot(n.linalg.inv(info),n.dot(n.conjugate(Q.T),Ninv))
elif mode == "diagonal":
diag_info = n.diag(n.diag(info))
MQN = n.dot(n.linalg.inv(diag_info),n.dot(n.conjugate(Q.T),Ninv))
WindMatrix = n.dot(MQN,Q)
WindNorm = WindMatrix[0,0]
WindMatrixRow = WindMatrix[0] / WindNorm
MQN_firstRowNormed = MQN[0] / WindNorm
return MQN_firstRowNormed, WindMatrixRow
def Q_N_plots(Q,N,lms,save_tag):
ls = n.arange(max(lms[:,0])+1)
l_locs = ls*ls
Q = n.matrix(Q)
N = n.matrix(N)
Ninv = uf.pseudo_inverse(N,num_remov=None)
foo = n.dot(Q.H,Q)
foo = n.array(uf.pseudo_inverse(foo))
bar = n.dot(Q.H,n.dot(Ninv,Q))
bar_inv = uf.pseudo_inverse(bar)
sigma = n.dot(bar_inv,n.dot(bar,bar_inv.H))
bar_inv = n.array(bar_inv); sigma = n.array(sigma)
p.imshow(n.log(n.absolute(foo)))
p.title('(Qdag Q)^-1')
p.xticks(l_locs,ls)
p.yticks(l_locs,ls)
p.xlabel('l')
p.ylabel('l')
p.savefig('./figures/{0}_QdagQ_inv.pdf'.format(save_tag))
p.clf()
p.scatter(lms[:,0],n.diag(foo),c=lms[:,1],cmap=mpl.cm.PiYG,s=50)
p.xlabel('l (color is m)')
p.ylabel('diagonal elements of (Qdag Q)^-1')
p.colorbar()
p.savefig('./figures/{0}_QdagQ_inv_diagonal.pdf'.format(save_tag))
p.clf()
p.imshow(n.log(n.absolute(bar_inv)))
p.title('(Qdag Ninv Q)^-1')
p.xticks(l_locs,ls)
p.yticks(l_locs,ls)
p.xlabel('l')
p.ylabel('l')
p.savefig('./figures/{0}_QdagNinvQ_inv.pdf'.format(save_tag))
p.clf()
p.scatter(lms[:,0],n.diag(bar_inv),c=lms[:,1],cmap=mpl.cm.PiYG,s=50)
p.xlabel('l (color is m)')
p.ylabel('diagonal elements of (Qdag Ninv Q)^-1')
p.colorbar()
p.savefig('./figures/{0}_QdagNinvQ_inv_diagonal.pdf'.format(save_tag))
p.clf()
p.imshow(n.log(n.absolute(sigma)))
p.title('Sigma Matrix')
p.xticks(l_locs,ls)
p.yticks(l_locs,ls)
p.xlabel('l')
p.ylabel('l')
p.savefig('./figures/{0}_sigma.pdf'.format(save_tag))
p.clf()
p.scatter(lms[:,0],n.diag(sigma),c=lms[:,1],cmap=mpl.cm.PiYG,s=50)
p.xlabel('l (color is m)')
p.ylabel('diagonal elements of Sigma Matrix')
p.colorbar()
p.savefig('./figures/{0}_sigma_diagonal.pdf'.format(save_tag))
p.clf()
def compare_grids(lmax=3,num_remov=None):
fits_file = '/Users/mpresley/soft/gsm/haslam408_32.fits'
beam_sigs = n.array([0.09,0.17,0.35,0.69,1.05])
del_bls = n.array([4,6,8,10,20])
param_grid = n.zeros([len(beam_sigs),len(del_bls)])#n.meshgrid(beam_sigs,del_bls)
for ii,beam_sig in enumerate(beam_sigs):
for jj,del_bl in enumerate(del_bls):
save_tag = 'grid_del_bl_{0:.2f}_num_bl_10_beam_sig_{1:.2f}_'.format(del_bl,beam_sig)
Q_file = './Q_matrices/{0}Q_max_l_10.npz'.format(save_tag)
gsm_file = './gsm_matrices/{0}gsm_max_l_10.npz'.format(save_tag)
Qstuff = n.load(Q_file)
Q = Qstuff['Q']
Q = Q[:,0:(lmax+1)**2]
lms = Qstuff['lms']
lms = lms[0:(lmax+1)**2,:]
baselines = Qstuff['baselines']
num_bl = len(baselines)
#print Q.shape, lms.shape, baselines.shape
N = total_noise_covar(0.1,num_bl,gsm_file)
#N = (1.0**2)*n.identity(num_bl)
Ninv = uf.pseudo_inverse(N,num_remov=None)
Ninvp = n.array(n.log10(n.abs(Ninv)))
p.imshow(Ninvp)
p.title('Log10 of N^-1')
p.colorbar()
p.savefig('./figures/compare_grids/{0}_Ninv_im.pdf'.format(save_tag))
p.clf()
p.scatter(baselines[:,0],Ninvp[0,:],c=baselines[:,1],cmap=mpl.cm.PiYG,s=50)
p.xlabel('baseline in x direction (color is in y direction)')
p.ylabel('Log10 of First Row of N^-1')
p.colorbar()
p.savefig('./figures/compare_grids/{0}_Ninv.pdf'.format(save_tag))
p.clf()
alms = generate_sky_model_alms(fits_file,lmax=lmax)
alms = alms[:,2]
#alms = n.zeros((lmax+1)**2,dtype='complex')
#alms[0] = 100.; alms[1] = 50.
y = generate_sky_model_y(baselines,beam_sig,fits_file)
a,ahat,err = test_recover_alms(y,Q,N,alms,num_remov=num_remov)
gs_true, gs_recov = a[0],ahat[0]
param_grid[ii,jj] = n.log10(n.abs(2*n.real(ahat[0])))#n.sum(ahat[1:]) #n.abs((gs_true-n.real(gs_recov))/gs_true)
#print param_grid.shape
p.imshow(param_grid,interpolation='nearest',aspect='auto',extent=[0,len(del_bls),0,len(beam_sigs)],cmap='RdBu') #extent=[4,7,0.175,1.1]
p.title('Log10 of Recovered Global Signal from only GSM')
p.yticks(range(len(beam_sigs)),beam_sigs)
p.xticks(range(len(del_bls)),del_bls)
p.ylabel('beam sigmas')
p.xlabel('del baselines')
p.colorbar()
#p.show()
p.savefig('./figures/compare_grids_log10.pdf')
p.clf()