def debiasdataandcovxiMwp(
xiNNd,
xiangd,
xiangdhigh,
xiangdlow,
xiNNm,
xiangm,
xi012m,
splitxi0,
splitxi2,
wpNNd,
wpangd,
wpangdhigh,
wpangdlow,
wpNNm,
wpangm,
wp012m,
splitwp,
wpstart,
wpend,
covstatfname,
nell=2,
fname=None,
):
# def debiasdataandcovwp(wpNNd, wpangd, wpangdhigh, wpangdlow, wpNNm, wpangm, wp012m, splitwp, wpstart,wpend,covstatfname,fname=None):
"""
subtract the bias measured from the tiled mocks from the data, return a debiased combination.
print it to a file (fname) to be fed to bethalexie code in long format.
Also take in statistical covariance matrix and add two sources of systematics.
"""
# def xiwpcorrect(xiNNin, xiangin,splitxi0,splitxi2,wpNNin, wpangin, splitwp, wpstart, wpend):
xiwpcorrdtmp = xiwpcorrect(xiNNd, xiangd, splitxi0, splitxi2, wpNNd, wpangd, splitwp, wpstart, wpend)
xiwpcorrm = xiwpcorrect(xiNNm, xiangm, splitxi0, splitxi2, wpNNm, wpangm, splitwp, wpstart, wpend)
xiwpdebiased = copy.deepcopy(xiwpcorrdtmp)
# tmp!
# print xiwpdebiased.xiell
# print xiwpdebiased.wp
mydeltaxi = xi012m.xi - xiwpcorrm.xiell.xi ## subtract xi objects.
mydeltawp = wp012m.wp[wpstart : wpend + 1] - xiwpcorrm.wp.wp
xiwpdebiased.xiell.xi = xiwpdebiased.xiell.xi + mydeltaxi
xiwpdebiased.wp.wp = xiwpdebiased.wp.wp + mydeltawp
xiwpdebiased.xiwp = np.concatenate((xiwpdebiased.xiell.xilong, xiwpdebiased.wp.wp))
xiwpanghigh = xiwp.xiwp(xiangdhigh, wpangdhigh)
xiwpanglow = xiwp.xiwp(xiangdlow, wpangdlow)
## now the cov.
## make sure this is the cov for the corrected statistic with same splits.
if 0 == 0:
# try:
cov = np.loadtxt(covstatfname)
assert len(cov[:, 0]) == xiwpdebiased.ntot
splitz = covstatfname.split("splitswp")[0].split("splits")[1].split("_")
assert len(splitz) >= nell
ilist = []
tmp = 0
for ss in splitz:
tmp += 1
ilist.append(int(ss))
if tmp >= nell:
break
assert ilist[0] == splitxi0
assert ilist[1] == splitxi2
splitzwp = covstatfname.split("splitswp")[1].split("_")
print splitzwp
assert len(splitzwp) >= 3
ilist = []
tmp = 0
for ss in splitzwp:
tmp += 1
ilist.append(int(ss))
if tmp >= 3:
break
assert ilist[0] == splitwp
assert ilist[1] == wpstart
assert ilist[2] == wpend
## new jan 2 2014!!! forgot to take into account the unbiasicov fac. derive if from
## product of cov and icov.
## guess icovfname
tmp = covstatfname.split("/")
tmp[-1] = "i" + tmp[-1]
icovstatfname = "/".join(tmp)
icov = np.loadtxt(icovstatfname)
unbiasicovfac = (ximisc.getmatrixdiag(np.matrix(cov) * np.matrix(icov))).mean()
print "using this unbiasicovfac correction, dividing cov by this", unbiasicovfac
cov = cov / unbiasicovfac
diagstat = np.zeros(xiwpdebiased.ntot)
diagtot = np.zeros(xiwpdebiased.ntot)
diagvar = np.zeros(xiwpdebiased.ntot)
for i in range(len(diagstat)):
diagstat[i] = cov[i, i]
xiangdiffvar = (0.5 * (xiangdhigh.xi.flatten() - xiangdlow.xi.flatten())) ** 2
diagvar[0:splitxi0] = xiangdiffvar[0:splitxi0]
nxi0 = len(xiNNd.xi0)
nxi2 = len(xiNNd.xi2)
diagvar[nxi0 : nxi0 + splitxi2] = xiangdiffvar[nxi0 : nxi0 + splitxi2]
wpangdiffvar = (0.5 * (wpangdhigh.wp - wpangdlow.wp)) ** 2
diagvar[nxi0 + nxi2 : nxi0 + nxi2 + splitwp - wpstart] = wpangdiffvar[wpstart:splitwp]
print "ang high/low variance: ", diagvar / diagstat
diagvar[0 : nxi0 + nxi2] = diagvar[0 : nxi0 + nxi2] + (mydeltaxi.flatten()) ** 2
diagvar[nxi0 + nxi2 :] = diagvar[nxi0 + nxi2 :] + (mydeltawp) ** 2
print "bias variance xi: ", (mydeltaxi.flatten()) ** 2 / diagstat[0 : nxi0 + nxi2]
print "bias variance wp: ", (mydeltawp) ** 2 / diagstat[nxi0 + nxi2 :]
## add it into the covarianace matrix.
for i in range(xiwpdebiased.ntot):
cov[i, i] += diagvar[i]
diagtot[i] = cov[i, i]
print "total sys variance fraction", diagtot / diagstat
## make it a matrix.
cov = np.matrix(cov)
icov = cov.I
fcovout = covstatfname + ".sys"
## print the covariance and icov to new file.
printcov(cov, fcovout)
tmp = fcovout.split("/")
tmp[-1] = "i" + tmp[-1]
ifcovout = "/".join(tmp)
printcov(icov, ifcovout)
xiwpfinal = xiwp.xiwp(xiwpdebiased.xiell, xiwpdebiased.wp, icovfname=ifcovout)
if fname is not None:
ofp = open(fname, "w")
ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
for i in range(len(xiwpfinal.xiell.svec.flatten())):
ofp.write("%e %e\n" % (xiwpfinal.xiell.svec.flatten()[i], xiwpfinal.xiell.xi.flatten()[i]))
for i in range(len(xiwpfinal.wp.wp)):
ofp.write("%e %e\n" % (xiwpfinal.wp.rsig[i], xiwpfinal.wp.wp[i]))
ofp.close()
return xiwpfinal, cov
else:
print "cov file name does not match input splits, returning None!"
xiwpfinal = xiwp.xiwp(xiwpdebiased.xiell, xiwpdebiased.wp, icovfname=ifcovout)
if fname is not None:
ofp = open(fname, "w")
ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
for i in range(len(xiwpfinal.xi.svec.flatten())):
ofp.write("%e %e\n" % (xiwpfinal.xiell.svec.flatten()[i], xiwpfinal.xiell.xi.flatten()[i]))
for i in range(len(xiwpfinal.wp.wp)):
ofp.write("%e %e\n" % (xiwpfinal.wp.rsig[i], xiwpfinal.wp.wp[i]))
ofp.close()
return xiwpfinal, None
def debiasdataandcov(
xiNNd, xiangd, xiangdhigh, xiangdlow, xiNNm, xiangm, xi012m, splitxi0, splitxi2, covstatfname, nell=2, fname=None
):
"""
subtract the bias measured from the tiled mocks from the data, return a debiased combination.
print it to a file (fname) to be fed to bethalexie code in long format.
Also take in statistical covariance matrix and add two sources of systematics.
"""
xicorrdtmp = xicorrect(xiNNd, xiangd, splitxi0, splitxi2)
xicorrm = xicorrect(xiNNm, xiangm, splitxi0, splitxi2)
xidebiased = copy.deepcopy(xicorrdtmp.xi)
mydelta = xi012m.xi - xicorrm.xi
xidebiased = xidebiased + mydelta
## now the cov.
## make sure this is the cov for the corrected statistic with same splits.
if 0 == 0:
# try:
cov = np.loadtxt(covstatfname)
assert len(cov[:, 0]) == xiNNd.ndata
splitz = covstatfname.split("splits")[1].split("_")
assert len(splitz) == nell
ilist = []
for ss in splitz:
ilist.append(int(ss))
assert ilist[0] == splitxi0
assert ilist[1] == splitxi2
## new jan 2 2014!!! forgot to take into account the unbiasicov fac. derive if from
## product of cov and icov.
## guess icovfname
tmp = covstatfname.split("/")
tmp[-1] = "i" + tmp[-1]
icovstatfname = "/".join(tmp)
icov = np.loadtxt(icovstatfname)
unbiasicovfac = (ximisc.getmatrixdiag(np.matrix(cov) * np.matrix(icov))).mean()
print "using this unbiasicovfac correction, dividing cov by this", unbiasicovfac
cov = cov / unbiasicovfac
diagstat = np.zeros(xiNNd.ndata)
diagtot = np.zeros(xiNNd.ndata)
for i in range(len(diagstat)):
diagstat[i] = cov[i, i]
diagvar = np.zeros(xiNNd.ndata)
xiangdiffvar = (0.5 * (xiangdhigh.xi.flatten() - xiangdlow.xi.flatten())) ** 2
diagvar[0:splitxi0] = xiangdiffvar[0:splitxi0]
nxi0 = len(xiNNd.xi0)
diagvar[nxi0 : nxi0 + splitxi2] = xiangdiffvar[nxi0 : nxi0 + splitxi2]
print "ang high/low variance: ", diagvar / diagstat
diagvar = diagvar + (mydelta.flatten()) ** 2
print "bias variance: ", (mydelta.flatten()) ** 2 / diagstat
## add it into the covarianace matrix.
for i in range(xiNNd.ndata):
cov[i, i] += diagvar[i]
diagtot[i] = cov[i, i]
print "total sys variance fraction", diagtot / diagstat
## make it a matrix.
cov = np.matrix(cov)
icov = cov.I
fcovout = covstatfname + ".sys"
## print the covariance and icov to new file.
printcov(cov, fcovout)
tmp = fcovout.split("/")
tmp[-1] = "i" + tmp[-1]
ifcovout = "/".join(tmp)
printcov(icov, ifcovout)
xifinal = xiell.xiell(sxilist=[xiNNd.svec, xidebiased], icovfname=ifcovout)
if fname is not None:
ofp = open(fname, "w")
ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
for i in range(len(xifinal.svec.flatten())):
ofp.write("%e %e\n" % (xifinal.svec.flatten()[i], xifinal.xi.flatten()[i]))
ofp.close()
return xifinal, cov
else:
# except:
print "cov file name does not match input splits, returning None!"
xifinal = xiell.xiell(sxilist=[xiNNd.svec, xidebiased])
if fname is not None:
ofp = open(fname, "w")
ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
for i in range(len(xifinal.svec.flatten())):
ofp.write("%e %e\n" % (xifinal.svec.flatten()[i], xifinal.xi.flatten()[i]))
ofp.close()
return xifinal, None
def debiasdataandcovwp(
wpNNd, wpangd, wpangdhigh, wpangdlow, wpNNm, wpangm, wp012m, splitwp, wpstart, wpend, covstatfname, fname=None
):
"""
subtract the bias measured from the tiled mocks from the data, return a debiased combination.
print it to a file (fname) to be fed to bethalexie code in long format.
Also take in statistical covariance matrix and add two sources of systematics.
"""
wpcorrdtmp = wpcorrect(wpNNd, wpangd, splitwp, wpstart, wpend)
wpcorrm = wpcorrect(wpNNm, wpangm, splitwp, wpstart, wpend)
wpdebiased = copy.deepcopy(wpcorrdtmp.wp)
mydelta = wp012m.wp[wpstart:] - wpcorrm.wp
print "fractional wp correction:"
print mydelta / wpcorrdtmp.wp
wpdebiased = wpdebiased + mydelta
## now the cov.
## make sure this is the cov for the corrected statistic with same splits.
if 0 == 0:
# try:
cov = np.loadtxt(covstatfname)
assert len(cov[:, 0]) == len(wpdebiased)
splitz = covstatfname.split("splitswp")[1].split("_")
assert len(splitz) >= 2
ilist = []
for ss in splitz[:2]:
ilist.append(int(ss))
assert ilist[0] == splitwp
assert ilist[1] == wpstart
## new jan 2 2014!!! forgot to take into account the unbiasicov fac. derive if from
## product of cov and icov.
## guess icovfname
tmp = covstatfname.split("/")
tmp[-1] = "i" + tmp[-1]
icovstatfname = "/".join(tmp)
icov = np.loadtxt(icovstatfname)
unbiasicovfac = (ximisc.getmatrixdiag(np.matrix(cov) * np.matrix(icov))).mean()
print "using htis unbiasicovfac correction, dividing cov by this", unbiasicovfac
cov = cov / unbiasicovfac
ndatacorr = len(wpcorrdtmp.wp)
diagstat = np.zeros(ndatacorr)
diagtot = np.zeros(ndatacorr)
diagvar = np.zeros(ndatacorr)
for i in range(len(diagstat)):
diagstat[i] = cov[i, i]
## this must agree with wpcorrect assignmeents!
wpangdiffvar = (0.5 * (wpangdhigh.wp - wpangdlow.wp)) ** 2
diagvar[0 : splitwp - wpstart] = wpangdiffvar[wpstart:splitwp]
print "wp ang high/low variance: ", diagvar / diagstat
print "bias correction: ", mydelta
diagvar = diagvar + (mydelta.flatten()) ** 2
print "bias variance contribution: ", (mydelta.flatten()) ** 2 / diagstat
## add it into the covarianace matrix.
for i in range(ndatacorr):
cov[i, i] += diagvar[i]
diagtot[i] = cov[i, i]
print "total sys variance fraction", diagtot / diagstat
## make it a matrix.
cov = np.matrix(cov)
icov = cov.I
fcovout = covstatfname + ".sys"
## print the covariance and icov to new file.
printcov(cov, fcovout)
tmp = fcovout.split("/")
tmp[-1] = "i" + tmp[-1]
ifcovout = "/".join(tmp)
printcov(icov, ifcovout)
wpfinal = wp.wp(rpwplist=[wpcorrdtmp.rsig, wpdebiased], icovfname=ifcovout)
if fname is not None:
wpfinal.printwp(fname)
return wpfinal, cov
else:
# except:
print "cov file name does not match input splits, returning None!"
wpfinal = wp.wp(rpwplist=[wpcorrdtmp.rsig, wpdebiased])
if fname is not None:
wpfinal.printwp(fname)
return wpfinal, None
def debiasdataandcovwp(wpNNd,
wpangd,
wpangdhigh,
wpangdlow,
wpNNm,
wpangm,
wp012m,
splitwp,
wpstart,
wpend,
covstatfname,
fname=None):
"""
subtract the bias measured from the tiled mocks from the data, return a debiased combination.
print it to a file (fname) to be fed to bethalexie code in long format.
Also take in statistical covariance matrix and add two sources of systematics.
"""
wpcorrdtmp = wpcorrect(wpNNd, wpangd, splitwp, wpstart, wpend)
wpcorrm = wpcorrect(wpNNm, wpangm, splitwp, wpstart, wpend)
wpdebiased = copy.deepcopy(wpcorrdtmp.wp)
mydelta = wp012m.wp[wpstart:] - wpcorrm.wp
print 'fractional wp correction:'
print mydelta / wpcorrdtmp.wp
wpdebiased = wpdebiased + mydelta
## now the cov.
## make sure this is the cov for the corrected statistic with same splits.
if (0 == 0):
# try:
cov = np.loadtxt(covstatfname)
assert len(cov[:, 0]) == len(wpdebiased)
splitz = covstatfname.split('splitswp')[1].split('_')
assert len(splitz) >= 2
ilist = []
for ss in splitz[:2]:
ilist.append(int(ss))
assert ilist[0] == splitwp
assert ilist[1] == wpstart
## new jan 2 2014!!! forgot to take into account the unbiasicov fac. derive if from
## product of cov and icov.
## guess icovfname
tmp = covstatfname.split('/')
tmp[-1] = 'i' + tmp[-1]
icovstatfname = '/'.join(tmp)
icov = np.loadtxt(icovstatfname)
unbiasicovfac = (ximisc.getmatrixdiag(
np.matrix(cov) * np.matrix(icov))).mean()
print 'using htis unbiasicovfac correction, dividing cov by this', unbiasicovfac
cov = cov / unbiasicovfac
ndatacorr = len(wpcorrdtmp.wp)
diagstat = np.zeros(ndatacorr)
diagtot = np.zeros(ndatacorr)
diagvar = np.zeros(ndatacorr)
for i in range(len(diagstat)):
diagstat[i] = cov[i, i]
## this must agree with wpcorrect assignmeents!
wpangdiffvar = (0.5 * (wpangdhigh.wp - wpangdlow.wp))**2
diagvar[0:splitwp - wpstart] = wpangdiffvar[wpstart:splitwp]
print 'wp ang high/low variance: ', diagvar / diagstat
print 'bias correction: ', mydelta
diagvar = diagvar + (mydelta.flatten())**2
print 'bias variance contribution: ', (mydelta.flatten())**2 / diagstat
## add it into the covarianace matrix.
for i in range(ndatacorr):
cov[i, i] += diagvar[i]
diagtot[i] = cov[i, i]
print 'total sys variance fraction', diagtot / diagstat
## make it a matrix.
cov = np.matrix(cov)
icov = cov.I
fcovout = covstatfname + '.sys'
## print the covariance and icov to new file.
printcov(cov, fcovout)
tmp = fcovout.split('/')
tmp[-1] = 'i' + tmp[-1]
ifcovout = '/'.join(tmp)
printcov(icov, ifcovout)
wpfinal = wp.wp(rpwplist=[wpcorrdtmp.rsig, wpdebiased],
icovfname=ifcovout)
if fname is not None:
wpfinal.printwp(fname)
return wpfinal, cov
else:
# except:
print 'cov file name does not match input splits, returning None!'
wpfinal = wp.wp(rpwplist=[wpcorrdtmp.rsig, wpdebiased])
if fname is not None:
wpfinal.printwp(fname)
return wpfinal, None
def debiasdataandcovxiMwp(xiNNd,
xiangd,
xiangdhigh,
xiangdlow,
xiNNm,
xiangm,
xi012m,
splitxi0,
splitxi2,
wpNNd,
wpangd,
wpangdhigh,
wpangdlow,
wpNNm,
wpangm,
wp012m,
splitwp,
wpstart,
wpend,
covstatfname,
nell=2,
fname=None):
#def debiasdataandcovwp(wpNNd, wpangd, wpangdhigh, wpangdlow, wpNNm, wpangm, wp012m, splitwp, wpstart,wpend,covstatfname,fname=None):
"""
subtract the bias measured from the tiled mocks from the data, return a debiased combination.
print it to a file (fname) to be fed to bethalexie code in long format.
Also take in statistical covariance matrix and add two sources of systematics.
"""
#def xiwpcorrect(xiNNin, xiangin,splitxi0,splitxi2,wpNNin, wpangin, splitwp, wpstart, wpend):
xiwpcorrdtmp = xiwpcorrect(xiNNd, xiangd, splitxi0,splitxi2,\
wpNNd, wpangd, splitwp, wpstart, wpend)
xiwpcorrm = xiwpcorrect(xiNNm, xiangm, splitxi0, splitxi2,\
wpNNm,wpangm,splitwp,wpstart,wpend)
xiwpdebiased = copy.deepcopy(xiwpcorrdtmp)
#tmp!
# print xiwpdebiased.xiell
# print xiwpdebiased.wp
mydeltaxi = xi012m.xi - xiwpcorrm.xiell.xi ## subtract xi objects.
mydeltawp = wp012m.wp[wpstart:wpend + 1] - xiwpcorrm.wp.wp
xiwpdebiased.xiell.xi = xiwpdebiased.xiell.xi + mydeltaxi
xiwpdebiased.wp.wp = xiwpdebiased.wp.wp + mydeltawp
xiwpdebiased.xiwp = np.concatenate(
(xiwpdebiased.xiell.xilong, xiwpdebiased.wp.wp))
xiwpanghigh = xiwp.xiwp(xiangdhigh, wpangdhigh)
xiwpanglow = xiwp.xiwp(xiangdlow, wpangdlow)
## now the cov.
## make sure this is the cov for the corrected statistic with same splits.
if (0 == 0):
# try:
cov = np.loadtxt(covstatfname)
assert len(cov[:, 0]) == xiwpdebiased.ntot
splitz = covstatfname.split('splitswp')[0].split('splits')[1].split(
'_')
assert len(splitz) >= nell
ilist = []
tmp = 0
for ss in splitz:
tmp += 1
ilist.append(int(ss))
if tmp >= nell:
break
assert ilist[0] == splitxi0
assert ilist[1] == splitxi2
splitzwp = covstatfname.split('splitswp')[1].split('_')
print splitzwp
assert len(splitzwp) >= 3
ilist = []
tmp = 0
for ss in splitzwp:
tmp += 1
ilist.append(int(ss))
if tmp >= 3:
break
assert ilist[0] == splitwp
assert ilist[1] == wpstart
assert ilist[2] == wpend
## new jan 2 2014!!! forgot to take into account the unbiasicov fac. derive if from
## product of cov and icov.
## guess icovfname
tmp = covstatfname.split('/')
tmp[-1] = 'i' + tmp[-1]
icovstatfname = '/'.join(tmp)
icov = np.loadtxt(icovstatfname)
unbiasicovfac = (ximisc.getmatrixdiag(
np.matrix(cov) * np.matrix(icov))).mean()
print 'using this unbiasicovfac correction, dividing cov by this', unbiasicovfac
cov = cov / unbiasicovfac
diagstat = np.zeros(xiwpdebiased.ntot)
diagtot = np.zeros(xiwpdebiased.ntot)
diagvar = np.zeros(xiwpdebiased.ntot)
for i in range(len(diagstat)):
diagstat[i] = cov[i, i]
xiangdiffvar = (0.5 *
(xiangdhigh.xi.flatten() - xiangdlow.xi.flatten()))**2
diagvar[0:splitxi0] = xiangdiffvar[0:splitxi0]
nxi0 = len(xiNNd.xi0)
nxi2 = len(xiNNd.xi2)
diagvar[nxi0:nxi0 + splitxi2] = xiangdiffvar[nxi0:nxi0 + splitxi2]
wpangdiffvar = (0.5 * (wpangdhigh.wp - wpangdlow.wp))**2
diagvar[nxi0 + nxi2:nxi0 + nxi2 + splitwp -
wpstart] = wpangdiffvar[wpstart:splitwp]
print 'ang high/low variance: ', diagvar / diagstat
diagvar[0:nxi0 +
nxi2] = diagvar[0:nxi0 + nxi2] + (mydeltaxi.flatten())**2
diagvar[nxi0 + nxi2:] = diagvar[nxi0 + nxi2:] + (mydeltawp)**2
print 'bias variance xi: ', (
mydeltaxi.flatten())**2 / diagstat[0:nxi0 + nxi2]
print 'bias variance wp: ', (mydeltawp)**2 / diagstat[nxi0 + nxi2:]
## add it into the covarianace matrix.
for i in range(xiwpdebiased.ntot):
cov[i, i] += diagvar[i]
diagtot[i] = cov[i, i]
print 'total sys variance fraction', diagtot / diagstat
## make it a matrix.
cov = np.matrix(cov)
icov = cov.I
fcovout = covstatfname + '.sys'
## print the covariance and icov to new file.
printcov(cov, fcovout)
tmp = fcovout.split('/')
tmp[-1] = 'i' + tmp[-1]
ifcovout = '/'.join(tmp)
printcov(icov, ifcovout)
xiwpfinal = xiwp.xiwp(xiwpdebiased.xiell,
xiwpdebiased.wp,
icovfname=ifcovout)
if fname is not None:
ofp = open(fname, 'w')
ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
for i in range(len(xiwpfinal.xiell.svec.flatten())):
ofp.write('%e %e\n' % (xiwpfinal.xiell.svec.flatten()[i],
xiwpfinal.xiell.xi.flatten()[i]))
for i in range(len(xiwpfinal.wp.wp)):
ofp.write('%e %e\n' %
(xiwpfinal.wp.rsig[i], xiwpfinal.wp.wp[i]))
ofp.close()
return xiwpfinal, cov
else:
print 'cov file name does not match input splits, returning None!'
xiwpfinal = xiwp.xiwp(xiwpdebiased.xiell,
xiwpdebiased.wp,
icovfname=ifcovout)
if fname is not None:
ofp = open(fname, 'w')
ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
for i in range(len(xiwpfinal.xi.svec.flatten())):
ofp.write('%e %e\n' % (xiwpfinal.xiell.svec.flatten()[i],
xiwpfinal.xiell.xi.flatten()[i]))
for i in range(len(xiwpfinal.wp.wp)):
ofp.write('%e %e\n' %
(xiwpfinal.wp.rsig[i], xiwpfinal.wp.wp[i]))
ofp.close()
return xiwpfinal, None
def debiasdataandcov(xiNNd,
xiangd,
xiangdhigh,
xiangdlow,
xiNNm,
xiangm,
xi012m,
splitxi0,
splitxi2,
covstatfname,
nell=2,
fname=None):
"""
subtract the bias measured from the tiled mocks from the data, return a debiased combination.
print it to a file (fname) to be fed to bethalexie code in long format.
Also take in statistical covariance matrix and add two sources of systematics.
"""
xicorrdtmp = xicorrect(xiNNd, xiangd, splitxi0, splitxi2)
xicorrm = xicorrect(xiNNm, xiangm, splitxi0, splitxi2)
xidebiased = copy.deepcopy(xicorrdtmp.xi)
mydelta = xi012m.xi - xicorrm.xi
xidebiased = xidebiased + mydelta
## now the cov.
## make sure this is the cov for the corrected statistic with same splits.
if (0 == 0):
# try:
cov = np.loadtxt(covstatfname)
assert len(cov[:, 0]) == xiNNd.ndata
splitz = covstatfname.split('splits')[1].split('_')
assert len(splitz) == nell
ilist = []
for ss in splitz:
ilist.append(int(ss))
assert ilist[0] == splitxi0
assert ilist[1] == splitxi2
## new jan 2 2014!!! forgot to take into account the unbiasicov fac. derive if from
## product of cov and icov.
## guess icovfname
tmp = covstatfname.split('/')
tmp[-1] = 'i' + tmp[-1]
icovstatfname = '/'.join(tmp)
icov = np.loadtxt(icovstatfname)
unbiasicovfac = (ximisc.getmatrixdiag(
np.matrix(cov) * np.matrix(icov))).mean()
print 'using this unbiasicovfac correction, dividing cov by this', unbiasicovfac
cov = cov / unbiasicovfac
diagstat = np.zeros(xiNNd.ndata)
diagtot = np.zeros(xiNNd.ndata)
for i in range(len(diagstat)):
diagstat[i] = cov[i, i]
diagvar = np.zeros(xiNNd.ndata)
xiangdiffvar = (0.5 *
(xiangdhigh.xi.flatten() - xiangdlow.xi.flatten()))**2
diagvar[0:splitxi0] = xiangdiffvar[0:splitxi0]
nxi0 = len(xiNNd.xi0)
diagvar[nxi0:nxi0 + splitxi2] = xiangdiffvar[nxi0:nxi0 + splitxi2]
print 'ang high/low variance: ', diagvar / diagstat
diagvar = diagvar + (mydelta.flatten())**2
print 'bias variance: ', (mydelta.flatten())**2 / diagstat
## add it into the covarianace matrix.
for i in range(xiNNd.ndata):
cov[i, i] += diagvar[i]
diagtot[i] = cov[i, i]
print 'total sys variance fraction', diagtot / diagstat
## make it a matrix.
cov = np.matrix(cov)
icov = cov.I
fcovout = covstatfname + '.sys'
## print the covariance and icov to new file.
printcov(cov, fcovout)
tmp = fcovout.split('/')
tmp[-1] = 'i' + tmp[-1]
ifcovout = '/'.join(tmp)
printcov(icov, ifcovout)
xifinal = xiell.xiell(sxilist=[xiNNd.svec, xidebiased],
icovfname=ifcovout)
if fname is not None:
ofp = open(fname, 'w')
ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
for i in range(len(xifinal.svec.flatten())):
ofp.write('%e %e\n' %
(xifinal.svec.flatten()[i], xifinal.xi.flatten()[i]))
ofp.close()
return xifinal, cov
else:
# except:
print 'cov file name does not match input splits, returning None!'
xifinal = xiell.xiell(sxilist=[xiNNd.svec, xidebiased])
if fname is not None:
ofp = open(fname, 'w')
ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
for i in range(len(xifinal.svec.flatten())):
ofp.write('%e %e\n' %
(xifinal.svec.flatten()[i], xifinal.xi.flatten()[i]))
ofp.close()
return xifinal, None