def xicorrect(xiNNin, xiangin, splitxi0=5, splitxi2=6):
xicorrxi = copy.deepcopy(xiangin.xi)
xicorrxi[0, splitxi0:] = xiNNin.xi[0, splitxi0:]
xicorrxi[1, splitxi2:] = xiNNin.xi[1, splitxi2:]
xicorr = xiell.xiell(sxilist=[xiangin.svec, xicorrxi])
## need to fix xi0, xi2, xilong, xi. do those go through?
return xicorr
def simplexiell(self):
"""
Computes multipoles in the simplest way --
Direct summation over the bins times Legendre polynomials.
No interpolation or correction for the fact that xi_{2,4,...} may not
be 0 for xi=0.
"""
myxiell = np.zeros([3,self.ns],dtype=float)
mysvec = np.zeros([3,self.ns],dtype=float)
for i in range(self.ns):
i1 = i*self.nmu
i2 = (i+1)*self.nmu
mymu = self.mu[i1:i2]
## sanity check that you picked out a correct and full range of mu.
assert (mymu[1:] - mymu[:-1] > 0.).all()
dmu = (mymu[1:] - mymu[:-1]).mean()
assert np.fabs(mymu[0]-dmu*0.5) < 2.0e-4
assert np.fabs(mymu[-1]+dmu*0.5-1.) < 2.0e-4
for ell in [0,2,4]:
myxiell[ell/2,i] = (self.xi[i1:i2]*ximisc.legendre(ell,mymu)).sum()*(2.*ell+1.)/float(self.nmu)
mysvec[ell/2,i] = self.s1d[i]
## tmp!
#print '%e %e\n' % (mysvec[ell/2,i], myxiell[ell/2,i])
## create a xiell object.
return xiell.xiell(ellmax=4,sxilist=[mysvec,myxiell])
def xicorrect(xiNNin, xiangin, splitxi0=5, splitxi2=6):
xicorrxi = copy.deepcopy(xiangin.xi)
xicorrxi[0, splitxi0:] = xiNNin.xi[0, splitxi0:]
xicorrxi[1, splitxi2:] = xiNNin.xi[1, splitxi2:]
xicorr = xiell.xiell(sxilist=[xiangin.svec, xicorrxi])
## need to fix xi0, xi2, xilong, xi. do those go through?
return xicorr
def simplexiell(self):
"""
Computes multipoles in the simplest way --
Direct summation over the bins times Legendre polynomials.
No interpolation or correction for the fact that xi_{2,4,...} may not
be 0 for xi=0.
"""
myxiell = np.zeros([3, self.ns], dtype=float)
mysvec = np.zeros([3, self.ns], dtype=float)
for i in range(self.ns):
i1 = i * self.nmu
i2 = (i + 1) * self.nmu
mymu = self.mu[i1:i2]
## sanity check that you picked out a correct and full range of mu.
assert (mymu[1:] - mymu[:-1] > 0.).all()
dmu = (mymu[1:] - mymu[:-1]).mean()
assert np.fabs(mymu[0] - dmu * 0.5) < 2.0e-4
assert np.fabs(mymu[-1] + dmu * 0.5 - 1.) < 2.0e-4
for ell in [0, 2, 4]:
myxiell[ell / 2, i] = (self.xi[i1:i2] * ximisc.legendre(
ell, mymu)).sum() * (2. * ell + 1.) / float(self.nmu)
mysvec[ell / 2, i] = self.s1d[i]
## tmp!
#print '%e %e\n' % (mysvec[ell/2,i], myxiell[ell/2,i])
## create a xiell object.
return xiell.xiell(ellmax=4, sxilist=[mysvec, myxiell])
def dsetNS(dsetN, dsetS, tasklist = [0,1,3], icovlist=[None,None,None]):
x = dset(fbase=None,wdir=None,tasklist = []) ## do nothing.
x.fbase = dsetN.fbase
x.wdir = dsetN.wdir
if 0 in tasklist:
cut=0.533655
## remove the hard-coding here soon!
try:
indx = np.where(np.array(tasklist) == 0)[0]
icovfname = icovlist[indx]
except:
icovfname = None
tmp = dsetN.xiell + dsetS.xiell
x.xiell = xiell.xiell(icovfname=icovfname, sxilist = [tmp.svec, tmp.xi])
if 1 in tasklist:
try:
indx = np.where(np.array(tasklist) == 1)[0]
icovfname = icovlist[indx]
except:
icovfname = None
tmp = dsetN.wp + dsetS.wp
x.wp = wp.wp(icovfname=icovfname,rpwplist=[tmp.rsig, tmp.wp])
if 3 in tasklist:
try:
indx = np.where(np.array(tasklist) == 3)[0]
icovfname = icovlist[indx]
except:
icovfname = None
x.nbar2d = dsetN.nbar2d
x.nbar3d = dsetN.nbar3d
tmp = dsetN.wpcross + dsetS.wpcross
x.wpcross = wp.wp(icovfname=icovfname,rpwplist=[tmp.rsig, tmp.wp])
return x
def dsetNS(dsetN, dsetS, tasklist=[0, 1, 3], icovlist=[None, None, None]):
x = dset(fbase=None, wdir=None, tasklist=[]) ## do nothing.
x.fbase = dsetN.fbase
x.wdir = dsetN.wdir
if 0 in tasklist:
cut = 0.533655
## remove the hard-coding here soon!
try:
indx = np.where(np.array(tasklist) == 0)[0]
icovfname = icovlist[indx]
except:
icovfname = None
tmp = dsetN.xiell + dsetS.xiell
x.xiell = xiell.xiell(icovfname=icovfname, sxilist=[tmp.svec, tmp.xi])
if 1 in tasklist:
try:
indx = np.where(np.array(tasklist) == 1)[0]
icovfname = icovlist[indx]
except:
icovfname = None
tmp = dsetN.wp + dsetS.wp
x.wp = wp.wp(icovfname=icovfname, rpwplist=[tmp.rsig, tmp.wp])
if 3 in tasklist:
try:
indx = np.where(np.array(tasklist) == 3)[0]
icovfname = icovlist[indx]
except:
icovfname = None
x.nbar2d = dsetN.nbar2d
x.nbar3d = dsetN.nbar3d
tmp = dsetN.wpcross + dsetS.wpcross
x.wpcross = wp.wp(icovfname=icovfname, rpwplist=[tmp.rsig, tmp.wp])
return x
def __init__(self,
xi2dfname=None,
wpfname=None,
wpicovfname=None,
xiellfname=None,
xiellicovfname=None,
xidatfname=None):
"""
Uses file inputs to read in various correlation function statistics (xi2d, xiell, wp) into a xi object.
The main purpose of this class is to package a single model/measurement together and make a fancy plot.
"""
if xidatfname is not None:
ifp = open(xidatfname, 'r')
print 'reading info from', xidatfname
for line in ifp:
if xi2dfname is None:
if (re.match('xi2d:', line)):
xi2dfname = line.split(':')[1].strip(' \n')
if wpfname is None:
if (re.match('wp:', line)):
x = line.split(':')[1].split(',')
assert len(x) == 1 or len(x) == 2
wpfname = x[0].strip(' \n')
if (len(x) == 2 and wpicovfname is None):
wpicovfname = x[1].strip(' \n')
if xiellfname is None:
if (re.match('xiell:', line)):
x = line.split(':')[1].split(',')
assert len(x) == 1 or len(x) == 2
xiellfname = x[0].strip(' \n')
if (len(x) == 2 and xiellicovfname is None):
xiellicovfname = x[1].strip(' \n')
self.xi2dfname = xi2dfname
self.wpfname = wpfname
self.xiellfname = xiellfname
if (xi2dfname is not None):
self.xi2d = xi2d.xi2d(xi2dfname)
else:
self.xi2d = None
if (wpfname is not None):
self.wp = wp.wp(wpfname, icovfname=wpicovfname)
else:
self.wp = None
if (xiellfname is not None):
self.xiell = xiell.xiell(xiellfname, icovfname=xiellicovfname)
else:
self.xiell = None
def __init__(self,xi2dfname=None,wpfname=None,wpicovfname=None,xiellfname=None,xiellicovfname=None,xidatfname=None):
"""
Uses file inputs to read in various correlation function statistics (xi2d, xiell, wp) into a xi object.
The main purpose of this class is to package a single model/measurement together and make a fancy plot.
"""
if xidatfname is not None:
ifp = open(xidatfname,'r')
print 'reading info from',xidatfname
for line in ifp:
if xi2dfname is None:
if(re.match('xi2d:',line)):
xi2dfname = line.split(':')[1].strip(' \n')
if wpfname is None:
if(re.match('wp:',line)):
x = line.split(':')[1].split(',')
assert len(x) == 1 or len(x) == 2
wpfname = x[0].strip(' \n')
if(len(x) == 2 and wpicovfname is None):
wpicovfname = x[1].strip(' \n')
if xiellfname is None:
if(re.match('xiell:',line)):
x = line.split(':')[1].split(',')
assert len(x) == 1 or len(x) == 2
xiellfname = x[0].strip(' \n')
if(len(x) == 2 and xiellicovfname is None):
xiellicovfname = x[1].strip(' \n')
self.xi2dfname = xi2dfname
self.wpfname = wpfname
self.xiellfname = xiellfname
if(xi2dfname is not None):
self.xi2d = xi2d.xi2d(xi2dfname)
else: self.xi2d = None
if(wpfname is not None):
self.wp = wp.wp(wpfname,icovfname=wpicovfname)
else: self.wp = None
if(xiellfname is not None):
self.xiell = xiell.xiell(xiellfname,icovfname=xiellicovfname)
else: self.xiell = None
def debiasdataandcov(
xiNNd, xiangd, xiangdhigh, xiangdlow, xiNNm, xiangm, xi012m, splitxi0, splitxi2, covstatfname, nell=2, fname=None
):
"""
subtract the bias measured from the tiled mocks from the data, return a debiased combination.
print it to a file (fname) to be fed to bethalexie code in long format.
Also take in statistical covariance matrix and add two sources of systematics.
"""
xicorrdtmp = xicorrect(xiNNd, xiangd, splitxi0, splitxi2)
xicorrm = xicorrect(xiNNm, xiangm, splitxi0, splitxi2)
xidebiased = copy.deepcopy(xicorrdtmp.xi)
mydelta = xi012m.xi - xicorrm.xi
xidebiased = xidebiased + mydelta
## now the cov.
## make sure this is the cov for the corrected statistic with same splits.
if 0 == 0:
# try:
cov = np.loadtxt(covstatfname)
assert len(cov[:, 0]) == xiNNd.ndata
splitz = covstatfname.split("splits")[1].split("_")
assert len(splitz) == nell
ilist = []
for ss in splitz:
ilist.append(int(ss))
assert ilist[0] == splitxi0
assert ilist[1] == splitxi2
## new jan 2 2014!!! forgot to take into account the unbiasicov fac. derive if from
## product of cov and icov.
## guess icovfname
tmp = covstatfname.split("/")
tmp[-1] = "i" + tmp[-1]
icovstatfname = "/".join(tmp)
icov = np.loadtxt(icovstatfname)
unbiasicovfac = (ximisc.getmatrixdiag(np.matrix(cov) * np.matrix(icov))).mean()
print "using this unbiasicovfac correction, dividing cov by this", unbiasicovfac
cov = cov / unbiasicovfac
diagstat = np.zeros(xiNNd.ndata)
diagtot = np.zeros(xiNNd.ndata)
for i in range(len(diagstat)):
diagstat[i] = cov[i, i]
diagvar = np.zeros(xiNNd.ndata)
xiangdiffvar = (0.5 * (xiangdhigh.xi.flatten() - xiangdlow.xi.flatten())) ** 2
diagvar[0:splitxi0] = xiangdiffvar[0:splitxi0]
nxi0 = len(xiNNd.xi0)
diagvar[nxi0 : nxi0 + splitxi2] = xiangdiffvar[nxi0 : nxi0 + splitxi2]
print "ang high/low variance: ", diagvar / diagstat
diagvar = diagvar + (mydelta.flatten()) ** 2
print "bias variance: ", (mydelta.flatten()) ** 2 / diagstat
## add it into the covarianace matrix.
for i in range(xiNNd.ndata):
cov[i, i] += diagvar[i]
diagtot[i] = cov[i, i]
print "total sys variance fraction", diagtot / diagstat
## make it a matrix.
cov = np.matrix(cov)
icov = cov.I
fcovout = covstatfname + ".sys"
## print the covariance and icov to new file.
printcov(cov, fcovout)
tmp = fcovout.split("/")
tmp[-1] = "i" + tmp[-1]
ifcovout = "/".join(tmp)
printcov(icov, ifcovout)
xifinal = xiell.xiell(sxilist=[xiNNd.svec, xidebiased], icovfname=ifcovout)
if fname is not None:
ofp = open(fname, "w")
ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
for i in range(len(xifinal.svec.flatten())):
ofp.write("%e %e\n" % (xifinal.svec.flatten()[i], xifinal.xi.flatten()[i]))
ofp.close()
return xifinal, cov
else:
# except:
print "cov file name does not match input splits, returning None!"
xifinal = xiell.xiell(sxilist=[xiNNd.svec, xidebiased])
if fname is not None:
ofp = open(fname, "w")
ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
for i in range(len(xifinal.svec.flatten())):
ofp.write("%e %e\n" % (xifinal.svec.flatten()[i], xifinal.xi.flatten()[i]))
ofp.close()
return xifinal, None
def debiasdataandcov(xiNNd,
xiangd,
xiangdhigh,
xiangdlow,
xiNNm,
xiangm,
xi012m,
splitxi0,
splitxi2,
covstatfname,
nell=2,
fname=None):
"""
subtract the bias measured from the tiled mocks from the data, return a debiased combination.
print it to a file (fname) to be fed to bethalexie code in long format.
Also take in statistical covariance matrix and add two sources of systematics.
"""
xicorrdtmp = xicorrect(xiNNd, xiangd, splitxi0, splitxi2)
xicorrm = xicorrect(xiNNm, xiangm, splitxi0, splitxi2)
xidebiased = copy.deepcopy(xicorrdtmp.xi)
mydelta = xi012m.xi - xicorrm.xi
xidebiased = xidebiased + mydelta
## now the cov.
## make sure this is the cov for the corrected statistic with same splits.
if (0 == 0):
# try:
cov = np.loadtxt(covstatfname)
assert len(cov[:, 0]) == xiNNd.ndata
splitz = covstatfname.split('splits')[1].split('_')
assert len(splitz) == nell
ilist = []
for ss in splitz:
ilist.append(int(ss))
assert ilist[0] == splitxi0
assert ilist[1] == splitxi2
## new jan 2 2014!!! forgot to take into account the unbiasicov fac. derive if from
## product of cov and icov.
## guess icovfname
tmp = covstatfname.split('/')
tmp[-1] = 'i' + tmp[-1]
icovstatfname = '/'.join(tmp)
icov = np.loadtxt(icovstatfname)
unbiasicovfac = (ximisc.getmatrixdiag(
np.matrix(cov) * np.matrix(icov))).mean()
print 'using this unbiasicovfac correction, dividing cov by this', unbiasicovfac
cov = cov / unbiasicovfac
diagstat = np.zeros(xiNNd.ndata)
diagtot = np.zeros(xiNNd.ndata)
for i in range(len(diagstat)):
diagstat[i] = cov[i, i]
diagvar = np.zeros(xiNNd.ndata)
xiangdiffvar = (0.5 *
(xiangdhigh.xi.flatten() - xiangdlow.xi.flatten()))**2
diagvar[0:splitxi0] = xiangdiffvar[0:splitxi0]
nxi0 = len(xiNNd.xi0)
diagvar[nxi0:nxi0 + splitxi2] = xiangdiffvar[nxi0:nxi0 + splitxi2]
print 'ang high/low variance: ', diagvar / diagstat
diagvar = diagvar + (mydelta.flatten())**2
print 'bias variance: ', (mydelta.flatten())**2 / diagstat
## add it into the covarianace matrix.
for i in range(xiNNd.ndata):
cov[i, i] += diagvar[i]
diagtot[i] = cov[i, i]
print 'total sys variance fraction', diagtot / diagstat
## make it a matrix.
cov = np.matrix(cov)
icov = cov.I
fcovout = covstatfname + '.sys'
## print the covariance and icov to new file.
printcov(cov, fcovout)
tmp = fcovout.split('/')
tmp[-1] = 'i' + tmp[-1]
ifcovout = '/'.join(tmp)
printcov(icov, ifcovout)
xifinal = xiell.xiell(sxilist=[xiNNd.svec, xidebiased],
icovfname=ifcovout)
if fname is not None:
ofp = open(fname, 'w')
ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
for i in range(len(xifinal.svec.flatten())):
ofp.write('%e %e\n' %
(xifinal.svec.flatten()[i], xifinal.xi.flatten()[i]))
ofp.close()
return xifinal, cov
else:
# except:
print 'cov file name does not match input splits, returning None!'
xifinal = xiell.xiell(sxilist=[xiNNd.svec, xidebiased])
if fname is not None:
ofp = open(fname, 'w')
ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
for i in range(len(xifinal.svec.flatten())):
ofp.write('%e %e\n' %
(xifinal.svec.flatten()[i], xifinal.xi.flatten()[i]))
ofp.close()
return xifinal, None
def __init__(self,
xiellin=None,
wpin=None,
xiellwpfname=None,
icovfname=None):
if xiellin is not None:
self.xiell = xiellin
if wpin is not None:
self.wp = wpin
if xiellwpfname is not None:
self.xiellwpfname = xiellwpfname
if xiellin is not None and wpin is not None:
self.ntot = self.wp.nrsig + self.xiell.ndata
self.xiwp = np.concatenate((self.xiell.xilong, self.wp.wp))
if len(self.xiwp) != self.ntot:
print 'vector length mismatch!'
self = None
else:
if xiellwpfname is None:
self = None
return
try:
rx, self.xiwp = np.loadtxt(xiellwpfname,
usecols=[0, 1],
unpack=True)
self.ntot = len(self.xiwp)
except:
self = None
if self is not None:
try:
## find xiell/wp split, create
xx = np.where(rx[:-1] > rx[1:])[0]
## i *think* this should work, double check if you choose some wacko binning!
xwp = xx[-1] + 1
nell = len(xx)
svec = rx[:xwp].reshape(nell, len(rx[:xwp]) / nell)
xivec = self.xiwp[:xwp].reshape(nell, len(rx[:xwp]) / nell)
self.xiell = xiell.xiell(sxilist=[svec, xivec])
self.wp = wp.wp(rpwplist=[rx[xwp:], self.xiwp[xwp:]])
except: # oh well.
pass
## tmp.
# self.xiell = xiell.xiell(sxilist=[rx[:xwp],self.xiwp[:xwp]])
# self.wp = wp.wp(rpwplist=[rx[xwp:], self.xiwp[xwp:]])
if self is not None:
if icovfname is None:
self.DorT = 1
else:
self.DorT = 0
icov = np.matrix(np.loadtxt(icovfname))
self.icov = icov
try:
cov = icov.I
diagerr = np.array(
[np.sqrt(cov[i, i]) for i in range(self.ntot)])
self.diagerr = diagerr
except:
diagerr = np.zeros(self.ntot) + 1.
self.diagerr = diagerr
if not len(icov[:, 0]) == self.ntot:
self = None
def __init__(self,xiellin=None,wpin=None,xiellwpfname=None,icovfname=None):
if xiellin is not None:
self.xiell = xiellin
if wpin is not None:
self.wp = wpin
if xiellwpfname is not None:
self.xiellwpfname = xiellwpfname
if xiellin is not None and wpin is not None:
self.ntot = self.wp.nrsig + self.xiell.ndata
self.xiwp = np.concatenate((self.xiell.xilong, self.wp.wp))
if len(self.xiwp) != self.ntot:
print 'vector length mismatch!'
self = None
else:
if xiellwpfname is None:
self = None
return
try:
rx, self.xiwp = np.loadtxt(xiellwpfname,usecols=[0,1],unpack=True)
self.ntot = len(self.xiwp)
except:
self = None
if self is not None:
try:
## find xiell/wp split, create
xx = np.where(rx[:-1] > rx[1:])[0]
## i *think* this should work, double check if you choose some wacko binning!
xwp = xx[-1]+1
nell = len(xx)
svec = rx[:xwp].reshape(nell,len(rx[:xwp])/nell)
xivec = self.xiwp[:xwp].reshape(nell,len(rx[:xwp])/nell)
self.xiell = xiell.xiell(sxilist=[svec,xivec])
self.wp = wp.wp(rpwplist=[rx[xwp:], self.xiwp[xwp:]])
except: # oh well.
pass
## tmp.
# self.xiell = xiell.xiell(sxilist=[rx[:xwp],self.xiwp[:xwp]])
# self.wp = wp.wp(rpwplist=[rx[xwp:], self.xiwp[xwp:]])
if self is not None:
if icovfname is None:
self.DorT = 1
else:
self.DorT = 0
icov = np.matrix(np.loadtxt(icovfname))
self.icov = icov
try:
cov = icov.I
diagerr = np.array([np.sqrt(cov[i,i]) for i in range(self.ntot)])
self.diagerr = diagerr
except:
diagerr = np.zeros(self.ntot) + 1.
self.diagerr = diagerr
if not len(icov[:,0]) == self.ntot:
self = None