def return_ahat(y,Q,N,num_remov=None):
assert len(y.shape)==1
Q = n.matrix(Q); N = n.matrix(N)
Ninv = uf.pseudo_inverse(N,num_remov=None) # XXX want to remove dynamically
info = n.dot(Q.H,n.dot(Ninv,Q))
M = uf.pseudo_inverse(info,num_remov=num_remov)
#print Ninv.shape, y.shape
ahat = uf.vdot(M,uf.vdot(Q.H,uf.vdot(Ninv,y)))
assert len(ahat.shape)==1
return ahat
def construct_gs_hist(del_bl=8.,num_bl=10,beam_sig=0.09,fq=0.1):
save_tag = 'grid_del_bl_{0:.2f}_num_bl_{1}_beam_sig_{2:.2f}_fq_{3:.3f}'.format(del_bl,num_bl,beam_sig,fq)
save_tag_mc = 'grid_del_bl_{0:.2f}_num_bl_{1}_beam_sig_{2:.2f}_fq_{3}'.format(del_bl,num_bl,beam_sig,fq)
ys = load_mc_data('{0}/monte_carlo/{1}'.format(data_loc,save_tag_mc))
print 'ys ',ys.shape
alms_fg = qgea.generate_sky_model_alms(gsm_fits_file,lmax=3)
alms_fg = alms_fg[:,2]
baselines,Q,lms = load_Q_file(gh='grid',del_bl=del_bl,num_bl=num_bl,beam_sig=beam_sig,fq=fq,lmax=3)
N = total_noise_covar(0.1,baselines.shape[0],'{0}/gsm_matrices/gsm_{1}.npz'.format(data_loc,save_tag))
MQN = return_MQdagNinv(Q,N,num_remov=None)
print MQN
ahat00s = n.array([])
for ii in xrange(ys.shape[1]):
#_,ahat,_ = qgea.test_recover_alms(ys[:,ii],Q,N,alms_fg,num_remov=None)
ahat = uf.vdot(MQN,ys[:,ii])
ahat00s = n.append(n.real(ahat[0]),ahat00s)
#print ahat00s
print ahat00s.shape
_,bins,_ = p.hist(ahat00s,bins=36,normed=True)
# plot best fit line
mu,sigma = norm.fit(ahat00s)
print "mu, sigma = ",mu,', ',sigma
y_fit = mpl.mlab.normpdf(bins,mu,sigma)
p.plot(bins, y_fit, 'r--', linewidth=2)
p.xlabel('ahat_00')
p.ylabel('Probability')
p.title(save_tag)
p.annotate('mu = {0:.2f}\nsigma = {1:.2f}'.format(mu,sigma), xy=(0.05, 0.5), xycoords='axes fraction')
p.savefig('./figures/monte_carlo/{0}.pdf'.format(save_tag))
p.clf()
def test_sum(self):
N = n.identity(8)
Q = n.identity(8)
Q[0,3] = 1; Q[3,0] = 1
a = n.arange(8); a[0] = 10
y = uf.vdot(Q,a)
a,ahat,err = qgea.test_recover_alms(y,Q,N,a,num_remov=1)
W = qgea.window_fn_matrix(Q,N,num_remov=1)
Wa = uf.vdot(W,a)
print 'Wa ',Wa
print 'ahat ',ahat
print Wa[0]/Wa[1]
print a[0]/a[1]
self.assertAlmostEqual(ahat[0],(a[0]+a[3])/2.)
self.assertAlmostEqual(ahat[3],(a[0]+a[3])/2.)
self.assertTrue(n.allclose(Wa,ahat))
def compute_element_mult_fqs(bli,blj,amp):
bix,biy,biz = bli; bjx,bjy,bjz = blj
rx,ry,rz = crd_array
rb_grid,fqs_grid = n.meshgrid((bix*rx+biy*ry+biz*rz),fqs)
Gi = amp*n.exp(-2j*n.pi*fqs_grid*rb_grid)*dOmega
fqs_grid, rb_grid = n.meshgrid((bjx*rx+bjy*ry+bjz*rz),fqs)
Gj_star = n.conj(amp*n.exp(-2j*n.pi*fqs_grid*rb_grid))*dOmega
elements = uf.vdot(Gi*Gj_star,Rsq)
return elements
def compute_element_mult_fqs(bli, blj, amp):
bix, biy, biz = bli
bjx, bjy, bjz = blj
rx, ry, rz = crd_array
rb_grid, fqs_grid = n.meshgrid((bix * rx + biy * ry + biz * rz), fqs)
Gi = amp * n.exp(-2j * n.pi * fqs_grid * rb_grid) * dOmega
fqs_grid, rb_grid = n.meshgrid((bjx * rx + bjy * ry + bjz * rz), fqs)
Gj_star = n.conj(amp * n.exp(-2j * n.pi * fqs_grid * rb_grid)) * dOmega
elements = uf.vdot(Gi * Gj_star, Rsq)
return elements
def test_recover_gs_ratio(self):
print "GS Ratio"
Q = n.diag(n.arange(1,9))
N = n.identity(8)*50
#N[0,0]=1
a = n.array([16,4,0,0,0,0,0,0])#n.arange(16,1,-2)
print 'a ',a
n_vec = uf.rand_from_covar(N)
Qa = uf.vdot(Q,a)
y = Qa+n_vec
self.assertTrue(y.shape==(8,))
a,ahat,err = qgea.test_recover_alms(y,Q,N,a,num_remov=0)
W = qgea.window_fn_matrix(Q,N,num_remov=0)
Wa = uf.vdot(W,a)
print 'Wa ',Wa
print 'ahat ',ahat
print Wa[0]/Wa[1]
print a[0]/a[1]
self.assertTrue(n.abs(Wa[0]/Wa[1]-a[0]/a[1])<0.1)
def test_subtract_noise(self):
Q = n.diag(n.arange(1,9))
N = n.identity(8)
a = n.arange(2,17,2)
n_vec = uf.rand_from_covar(N)
Qa = uf.vdot(Q,a)
y = Qa+n_vec
a,ahat,err = qgea.test_recover_alms(y,Q,N,a,num_remov=-4)
_,nhat,err = qgea.test_recover_alms(n_vec,Q,N,a,num_remov=-4)
ahat_better = ahat - nhat
W = qgea.window_fn_matrix(Q,N,num_remov=-4)
self.assertTrue(n.allclose(n.dot(W,a),ahat_better))
def test_recover_wind(self):
print "Window"
num=100
Q = n.diag(n.arange(1,9))
N = n.identity(8)
a = n.arange(2,17,2)
a1 = n.arange(2,17,2)
ahat_avg = 0
for ii in range(num):
n_vec = uf.rand_from_covar(N)
Qa = uf.vdot(Q,a)
y = Qa+n_vec
self.assertTrue(y.shape==(8,))
a,ahat,err = qgea.test_recover_alms(y,Q,N,a,num_remov=-4)
ahat_avg += ahat
self.assertTrue(n.all(y==n.dot(Q,a1)+n_vec))
ahat_avg = ahat_avg/num
W = qgea.window_fn_matrix(Q,N,num_remov=-4)
print uf.vdot(W,a)
print ahat_avg
self.assertTrue(n.all(n.abs(n.dot(W,a)-ahat_avg)<0.1))
def test_recover_alms(y,Q,N,a,num_remov=None):
# a is the alms from generate_sky_model_alms
assert len(y.shape)==1
assert len(a.shape)==1
# XXX num_removs shouldn't be the same
W = window_fn_matrix(Q,N,num_remov=num_remov) # W a = < a-hat >
ahat = return_ahat(y,Q,N,num_remov=num_remov)
#print "true gs = {0}\nrecovered gs = {1}".format(a[0],ahat[0])
err = n.abs(uf.vdot(W,a)-ahat)
#print 'err = ',err[0]
assert len(ahat.shape)==1
assert len(err.shape)==1
return a,ahat,err
def test_recover_gs(self):
print "GS"
num=500
Q = n.diag(n.arange(1,9))
N = n.identity(8)*50
N[0,0]=1
a = n.array([16,0,0,0,0,0,0,0])#n.arange(16,1,-2)
print 'a ',a
ahat_avg = 0
for ii in range(num):
n_vec = uf.rand_from_covar(N)
Qa = uf.vdot(Q,a)
y = Qa+n_vec
self.assertTrue(y.shape==(8,))
a,ahat,err = qgea.test_recover_alms(y,Q,N,a,num_remov=0)
ahat_avg += ahat
ahat_avg = ahat_avg/num
W = qgea.window_fn_matrix(Q,N,num_remov=0)
print uf.vdot(W,a)
print ahat_avg
self.assertTrue(n.abs(uf.vdot(W,a)[0]-ahat_avg[0])<0.1)
def plot_haslam_spectrum(del_bl=8.,num_bl=10,beam_sig=0.09,savea00=True,lmax=3):
save_tag_base = 'grid_del_bl_{0:.2f}_sqGridSideLen_{1}_beam_sig_{2:.2f}'.format(del_bl,num_bl,beam_sig)
print save_tag_base
save_tag_base_old = 'grid_del_bl_{0:.2f}_num_bl_{1}_beam_sig_{2:.2f}'.format(del_bl,num_bl,beam_sig)
mu = n.array([]); sigma = n.array([])
fqs = n.array([]); nums = n.array([])
for mc_fold in os.listdir('{0}/monte_carlo'.format(data_loc)):
if save_tag_base_old in mc_fold:
fq = float(mc_fold.split('_')[-1])
#for fq in (0.050,0.064,0.076,0.088,0.090):
mc_fold = '{0}_fq_{1}'.format(save_tag_base,fq)
ys = load_mc_data('{0}/monte_carlo_haslam'.format(data_loc,del_bl=del_bl,num_bl=num_bl,beam_sig=beam_sig,fq=fq))
baselines,Q,lms = load_Q_file(gh='grid',del_bl=del_bl,num_bl=num_bl,beam_sig=beam_sig,fq=fq,lmax=lmax)
#N = total_noise_covar(0.0,del_bl=del_bl,num_bl=num_bl,beam_sig=beam_sig,fq=fq)
N = n.eye(baselines.shape[0])
MQN = return_MQdagNinv(Q,N,num_remov=None)
#print 'MQN \n',MQN
ahat00s = n.array([])
for ii in xrange(ys.shape[1]):
#print MQN.shape
#print 'ys shape ',ys.shape
ahat = uf.vdot(MQN,ys[:,ii])
#print ahat[0]
ahat00s = n.append(n.real(ahat[0]),ahat00s)
mu0,sigma0 = norm.fit(ahat00s)
mu = n.append(mu,mu0); sigma = n.append(sigma,sigma0)
fqs = n.append(fqs,fq); nums = n.append(nums,ys.shape[1])
print fq
print mu0
#if fq==0.06: print ahat00s
if savea00: n.savez_compressed('{0}/monte_carlo_haslam/{1}/ahat00s'.format(data_loc,mc_fold),ahat00s=ahat00s)
print fqs
print mu
print sigma
p.errorbar(fqs, mu, yerr=sigma, fmt='o',ecolor='Black')#,c=nums,cmap=mpl.cm.copper_r)
#p.scatter(fqs,mu,color='g')
p.title('Mean and sigma of recovered global signal for\n{0}'.format(save_tag_base))
p.xlim([0.045,0.100])
#p.ylim([-1500.,0.0])
p.xlabel('Freq (GHz)')
p.ylabel('Recovered a00')
p.savefig('{0}/mc_spec_figs/{1}_haslam.pdf'.format(fig_loc,save_tag_base))
p.clf()
def construct_gs_hist(del_bl=8., num_bl=10, beam_sig=0.09, fq=0.1):
save_tag = 'grid_del_bl_{0:.2f}_num_bl_{1}_beam_sig_{2:.2f}_fq_{3:.3f}'.format(
del_bl, num_bl, beam_sig, fq)
save_tag_mc = 'grid_del_bl_{0:.2f}_num_bl_{1}_beam_sig_{2:.2f}_fq_{3}'.format(
del_bl, num_bl, beam_sig, fq)
ys = load_mc_data('{0}/monte_carlo/{1}'.format(data_loc, save_tag_mc))
print 'ys ', ys.shape
alms_fg = qgea.generate_sky_model_alms(gsm_fits_file, lmax=3)
alms_fg = alms_fg[:, 2]
baselines, Q, lms = load_Q_file(gh='grid',
del_bl=del_bl,
num_bl=num_bl,
beam_sig=beam_sig,
fq=fq,
lmax=3)
N = total_noise_covar(
0.1, baselines.shape[0],
'{0}/gsm_matrices/gsm_{1}.npz'.format(data_loc, save_tag))
MQN = return_MQdagNinv(Q, N, num_remov=None)
print MQN
ahat00s = n.array([])
for ii in xrange(ys.shape[1]):
#_,ahat,_ = qgea.test_recover_alms(ys[:,ii],Q,N,alms_fg,num_remov=None)
ahat = uf.vdot(MQN, ys[:, ii])
ahat00s = n.append(n.real(ahat[0]), ahat00s)
#print ahat00s
print ahat00s.shape
_, bins, _ = p.hist(ahat00s, bins=36, normed=True)
# plot best fit line
mu, sigma = norm.fit(ahat00s)
print "mu, sigma = ", mu, ', ', sigma
y_fit = mpl.mlab.normpdf(bins, mu, sigma)
p.plot(bins, y_fit, 'r--', linewidth=2)
p.xlabel('ahat_00')
p.ylabel('Probability')
p.title(save_tag)
p.annotate('mu = {0:.2f}\nsigma = {1:.2f}'.format(mu, sigma),
xy=(0.05, 0.5),
xycoords='axes fraction')
p.savefig('./figures/monte_carlo/{0}.pdf'.format(save_tag))
p.clf()
def get_Q_element_mult_fqs(tx,ty,tz,dOmega,amp,baseline,l,m):
"""
This returns a vector of Q elements for multiple frequencies, so the
vector is the same length as fqs.
"""
bx,by,bz = baseline
# compute spherical harmonic
Y = n.array(special.sph_harm(m,l,theta,phi)) #using math convention of theta=[0,2pi], phi=[0,pi]
#fringe pattern
tb_grid,fqs_grid = n.meshgrid((bx*tx+by*ty+bz*tz),fqs)
phs = n.exp(-2j*n.pi*fqs_grid*tb_grid)
# bl in ns, fq in GHz => bl*fq = 1
valid = n.logical_not(tx.mask)
amp = n.where(valid, amp, n.zeros_like(amp))
phs = n.where(valid, phs, n.zeros_like(phs))
Y = n.where(valid, Y, n.zeros_like(Y))
Q_elements = uf.vdot(phs*amp,Y*dOmega) #n.sum(amp*Y*phs*dOmega)
return Q_elements
def get_Q_element_mult_fqs(tx, ty, tz, dOmega, amp, baseline, l, m):
"""
This returns a vector of Q elements for multiple frequencies, so the
vector is the same length as fqs.
"""
bx, by, bz = baseline
# compute spherical harmonic
Y = n.array(special.sph_harm(
m, l, theta, phi)) #using math convention of theta=[0,2pi], phi=[0,pi]
#fringe pattern
tb_grid, fqs_grid = n.meshgrid((bx * tx + by * ty + bz * tz), fqs)
phs = n.exp(-2j * n.pi * fqs_grid * tb_grid)
# bl in ns, fq in GHz => bl*fq = 1
valid = n.logical_not(tx.mask)
amp = n.where(valid, amp, n.zeros_like(amp))
phs = n.where(valid, phs, n.zeros_like(phs))
Y = n.where(valid, Y, n.zeros_like(Y))
Q_elements = uf.vdot(phs * amp, Y * dOmega) #n.sum(amp*Y*phs*dOmega)
return Q_elements
def gaussian_model((A,nu0,sigma),nuvec):
return -A*n.exp(-(nuvec-nu0)**2/sigma**2/2.)
def gaussian_model_derivs((A,nu0,sigma),nuvec):
Aderiv = -n.exp(-0.5*(nuvec-nu0)**2/sigma**2)
nu0deriv = (A/sigma**2)*(nuvec-nu0)*n.exp(-0.5*(nuvec-nu0)**2/sigma**2)
sigderiv = (A/sigma**3)*(nuvec-nu0)**2*n.exp(-0.5*(nuvec-nu0)**2/sigma**2)
return Aderiv, nu0deriv, sigderiv
def fisher_matrix((A,nu0,sigma),nuvec,Cinv):
fisher = n.zeros((3,3))
derivs = gaussian_model_derivs((A,nu0,sigma),nuvec)
for ii in range(3):
for jj in range(3):
fisher[ii,jj] = n.dot(n.transpose(derivs[ii]),uf.vdot(Cinv,derivs[jj]))
print 'fisher ',fisher
return fisher
def fisher_select_pair(F,si,sj):
Finv = n.linalg.inv(F) #uf.pseudo_inverse(F)
for kk in range(Finv.shape[0])[::-1]:
if kk==si or kk==sj:
continue
Finv = n.delete(Finv,kk,axis=0)
Finv = n.delete(Finv,kk,axis=1)
#print Finv
Fnew = n.linalg.inv(Finv) #uf.pseudo_inverse(Finv)
return Fnew
def plot_pairwise_contours(theta,nuvec,Cinv,lvls=(2.291,6.158,11.618)):
def plot_haslam_spectrum(del_bl=8.,
num_bl=10,
beam_sig=0.09,
savea00=True,
lmax=3):
save_tag_base = 'grid_del_bl_{0:.2f}_sqGridSideLen_{1}_beam_sig_{2:.2f}'.format(
del_bl, num_bl, beam_sig)
print save_tag_base
save_tag_base_old = 'grid_del_bl_{0:.2f}_num_bl_{1}_beam_sig_{2:.2f}'.format(
del_bl, num_bl, beam_sig)
mu = n.array([])
sigma = n.array([])
fqs = n.array([])
nums = n.array([])
for mc_fold in os.listdir('{0}/monte_carlo'.format(data_loc)):
if save_tag_base_old in mc_fold:
fq = float(mc_fold.split('_')[-1])
#for fq in (0.050,0.064,0.076,0.088,0.090):
mc_fold = '{0}_fq_{1}'.format(save_tag_base, fq)
ys = load_mc_data('{0}/monte_carlo_haslam'.format(
data_loc,
del_bl=del_bl,
num_bl=num_bl,
beam_sig=beam_sig,
fq=fq))
baselines, Q, lms = load_Q_file(gh='grid',
del_bl=del_bl,
num_bl=num_bl,
beam_sig=beam_sig,
fq=fq,
lmax=lmax)
#N = total_noise_covar(0.0,del_bl=del_bl,num_bl=num_bl,beam_sig=beam_sig,fq=fq)
N = n.eye(baselines.shape[0])
MQN = return_MQdagNinv(Q, N, num_remov=None)
#print 'MQN \n',MQN
ahat00s = n.array([])
for ii in xrange(ys.shape[1]):
#print MQN.shape
#print 'ys shape ',ys.shape
ahat = uf.vdot(MQN, ys[:, ii])
#print ahat[0]
ahat00s = n.append(n.real(ahat[0]), ahat00s)
mu0, sigma0 = norm.fit(ahat00s)
mu = n.append(mu, mu0)
sigma = n.append(sigma, sigma0)
fqs = n.append(fqs, fq)
nums = n.append(nums, ys.shape[1])
print fq
print mu0
#if fq==0.06: print ahat00s
if savea00:
n.savez_compressed('{0}/monte_carlo_haslam/{1}/ahat00s'.format(
data_loc, mc_fold),
ahat00s=ahat00s)
print fqs
print mu
print sigma
p.errorbar(fqs, mu, yerr=sigma, fmt='o',
ecolor='Black') #,c=nums,cmap=mpl.cm.copper_r)
#p.scatter(fqs,mu,color='g')
p.title('Mean and sigma of recovered global signal for\n{0}'.format(
save_tag_base))
p.xlim([0.045, 0.100])
#p.ylim([-1500.,0.0])
p.xlabel('Freq (GHz)')
p.ylabel('Recovered a00')
p.savefig('{0}/mc_spec_figs/{1}_haslam.pdf'.format(fig_loc, save_tag_base))
p.clf()
Finv = n.linalg.inv(fisher_matrix((A, nu0, sigma), nuvec, Cinv, model))
for ii in range(3):
bias[ii] = n.dot(derivs[ii], n.dot(Cinv, expBias))
bias = n.dot(Finv, bias)
return bias
def fisher_matrix((A, nu0, sigma), nuvec, Cinv, model):
fisher = n.zeros((3, 3))
command = "derivs = " + str(model) + "_model_derivs((A,nu0,sigma),nuvec)"
exec command
#derivs = gaussian_model_derivs((A,nu0,sigma),nuvec)
for ii in range(3):
for jj in range(3):
fisher[ii, jj] = n.dot(n.transpose(derivs[ii]),
uf.vdot(Cinv, derivs[jj]))
print 'fisher ', fisher
return fisher
def fisher_select_pair(F, si, sj):
Finv = n.linalg.inv(F) #uf.pseudo_inverse(F)
for kk in range(Finv.shape[0])[::-1]:
if kk == si or kk == sj:
continue
Finv = n.delete(Finv, kk, axis=0)
Finv = n.delete(Finv, kk, axis=1)
#print Finv
Fnew = n.linalg.inv(Finv) #uf.pseudo_inverse(Finv)
return Fnew
