Example #1
0
def debiasdataandcovxiMwp(
    xiNNd,
    xiangd,
    xiangdhigh,
    xiangdlow,
    xiNNm,
    xiangm,
    xi012m,
    splitxi0,
    splitxi2,
    wpNNd,
    wpangd,
    wpangdhigh,
    wpangdlow,
    wpNNm,
    wpangm,
    wp012m,
    splitwp,
    wpstart,
    wpend,
    covstatfname,
    nell=2,
    fname=None,
):
    # def debiasdataandcovwp(wpNNd, wpangd, wpangdhigh, wpangdlow, wpNNm, wpangm, wp012m, splitwp, wpstart,wpend,covstatfname,fname=None):
    """
  subtract the bias measured from the tiled mocks from the data, return a debiased combination.
  print it to a file (fname) to be fed to bethalexie code in long format.
  Also take in statistical covariance matrix and add two sources of systematics.
  """
    # def xiwpcorrect(xiNNin, xiangin,splitxi0,splitxi2,wpNNin, wpangin, splitwp, wpstart, wpend):
    xiwpcorrdtmp = xiwpcorrect(xiNNd, xiangd, splitxi0, splitxi2, wpNNd, wpangd, splitwp, wpstart, wpend)

    xiwpcorrm = xiwpcorrect(xiNNm, xiangm, splitxi0, splitxi2, wpNNm, wpangm, splitwp, wpstart, wpend)

    xiwpdebiased = copy.deepcopy(xiwpcorrdtmp)
    # tmp!
    #  print xiwpdebiased.xiell
    #  print xiwpdebiased.wp

    mydeltaxi = xi012m.xi - xiwpcorrm.xiell.xi  ## subtract xi objects.
    mydeltawp = wp012m.wp[wpstart : wpend + 1] - xiwpcorrm.wp.wp
    xiwpdebiased.xiell.xi = xiwpdebiased.xiell.xi + mydeltaxi
    xiwpdebiased.wp.wp = xiwpdebiased.wp.wp + mydeltawp
    xiwpdebiased.xiwp = np.concatenate((xiwpdebiased.xiell.xilong, xiwpdebiased.wp.wp))
    xiwpanghigh = xiwp.xiwp(xiangdhigh, wpangdhigh)
    xiwpanglow = xiwp.xiwp(xiangdlow, wpangdlow)

    ## now the cov.
    ## make sure this is the cov for the corrected statistic with same splits.
    if 0 == 0:
        #  try:
        cov = np.loadtxt(covstatfname)
        assert len(cov[:, 0]) == xiwpdebiased.ntot
        splitz = covstatfname.split("splitswp")[0].split("splits")[1].split("_")
        assert len(splitz) >= nell
        ilist = []
        tmp = 0
        for ss in splitz:
            tmp += 1
            ilist.append(int(ss))
            if tmp >= nell:
                break
        assert ilist[0] == splitxi0
        assert ilist[1] == splitxi2
        splitzwp = covstatfname.split("splitswp")[1].split("_")
        print splitzwp
        assert len(splitzwp) >= 3
        ilist = []
        tmp = 0
        for ss in splitzwp:
            tmp += 1
            ilist.append(int(ss))
            if tmp >= 3:
                break
        assert ilist[0] == splitwp
        assert ilist[1] == wpstart
        assert ilist[2] == wpend

        ## new jan 2 2014!!!  forgot to take into account the unbiasicov fac.  derive if from
        ## product of cov and icov.
        ## guess icovfname
        tmp = covstatfname.split("/")
        tmp[-1] = "i" + tmp[-1]
        icovstatfname = "/".join(tmp)
        icov = np.loadtxt(icovstatfname)

        unbiasicovfac = (ximisc.getmatrixdiag(np.matrix(cov) * np.matrix(icov))).mean()
        print "using this unbiasicovfac correction, dividing cov by this", unbiasicovfac
        cov = cov / unbiasicovfac

        diagstat = np.zeros(xiwpdebiased.ntot)
        diagtot = np.zeros(xiwpdebiased.ntot)
        diagvar = np.zeros(xiwpdebiased.ntot)
        for i in range(len(diagstat)):
            diagstat[i] = cov[i, i]

        xiangdiffvar = (0.5 * (xiangdhigh.xi.flatten() - xiangdlow.xi.flatten())) ** 2
        diagvar[0:splitxi0] = xiangdiffvar[0:splitxi0]
        nxi0 = len(xiNNd.xi0)
        nxi2 = len(xiNNd.xi2)
        diagvar[nxi0 : nxi0 + splitxi2] = xiangdiffvar[nxi0 : nxi0 + splitxi2]
        wpangdiffvar = (0.5 * (wpangdhigh.wp - wpangdlow.wp)) ** 2
        diagvar[nxi0 + nxi2 : nxi0 + nxi2 + splitwp - wpstart] = wpangdiffvar[wpstart:splitwp]
        print "ang high/low variance: ", diagvar / diagstat
        diagvar[0 : nxi0 + nxi2] = diagvar[0 : nxi0 + nxi2] + (mydeltaxi.flatten()) ** 2
        diagvar[nxi0 + nxi2 :] = diagvar[nxi0 + nxi2 :] + (mydeltawp) ** 2
        print "bias variance xi: ", (mydeltaxi.flatten()) ** 2 / diagstat[0 : nxi0 + nxi2]
        print "bias variance wp: ", (mydeltawp) ** 2 / diagstat[nxi0 + nxi2 :]
        ## add it into the covarianace matrix.
        for i in range(xiwpdebiased.ntot):
            cov[i, i] += diagvar[i]
            diagtot[i] = cov[i, i]
        print "total sys variance fraction", diagtot / diagstat

        ## make it a matrix.
        cov = np.matrix(cov)
        icov = cov.I

        fcovout = covstatfname + ".sys"
        ## print the covariance and icov to new file.
        printcov(cov, fcovout)
        tmp = fcovout.split("/")
        tmp[-1] = "i" + tmp[-1]
        ifcovout = "/".join(tmp)
        printcov(icov, ifcovout)

        xiwpfinal = xiwp.xiwp(xiwpdebiased.xiell, xiwpdebiased.wp, icovfname=ifcovout)
        if fname is not None:
            ofp = open(fname, "w")
            ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
            for i in range(len(xiwpfinal.xiell.svec.flatten())):
                ofp.write("%e %e\n" % (xiwpfinal.xiell.svec.flatten()[i], xiwpfinal.xiell.xi.flatten()[i]))
            for i in range(len(xiwpfinal.wp.wp)):
                ofp.write("%e %e\n" % (xiwpfinal.wp.rsig[i], xiwpfinal.wp.wp[i]))
            ofp.close()

        return xiwpfinal, cov

    else:
        print "cov file name does not match input splits, returning None!"
        xiwpfinal = xiwp.xiwp(xiwpdebiased.xiell, xiwpdebiased.wp, icovfname=ifcovout)
        if fname is not None:
            ofp = open(fname, "w")
            ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
            for i in range(len(xiwpfinal.xi.svec.flatten())):
                ofp.write("%e %e\n" % (xiwpfinal.xiell.svec.flatten()[i], xiwpfinal.xiell.xi.flatten()[i]))
            for i in range(len(xiwpfinal.wp.wp)):
                ofp.write("%e %e\n" % (xiwpfinal.wp.rsig[i], xiwpfinal.wp.wp[i]))
            ofp.close()

        return xiwpfinal, None
Example #2
0
def debiasdataandcov(
    xiNNd, xiangd, xiangdhigh, xiangdlow, xiNNm, xiangm, xi012m, splitxi0, splitxi2, covstatfname, nell=2, fname=None
):
    """
  subtract the bias measured from the tiled mocks from the data, return a debiased combination.
  print it to a file (fname) to be fed to bethalexie code in long format.
  Also take in statistical covariance matrix and add two sources of systematics.
  """
    xicorrdtmp = xicorrect(xiNNd, xiangd, splitxi0, splitxi2)
    xicorrm = xicorrect(xiNNm, xiangm, splitxi0, splitxi2)
    xidebiased = copy.deepcopy(xicorrdtmp.xi)
    mydelta = xi012m.xi - xicorrm.xi
    xidebiased = xidebiased + mydelta

    ## now the cov.
    ## make sure this is the cov for the corrected statistic with same splits.
    if 0 == 0:
        #  try:
        cov = np.loadtxt(covstatfname)
        assert len(cov[:, 0]) == xiNNd.ndata
        splitz = covstatfname.split("splits")[1].split("_")
        assert len(splitz) == nell
        ilist = []
        for ss in splitz:
            ilist.append(int(ss))
        assert ilist[0] == splitxi0
        assert ilist[1] == splitxi2

        ## new jan 2 2014!!!  forgot to take into account the unbiasicov fac.  derive if from
        ## product of cov and icov.
        ## guess icovfname
        tmp = covstatfname.split("/")
        tmp[-1] = "i" + tmp[-1]
        icovstatfname = "/".join(tmp)
        icov = np.loadtxt(icovstatfname)

        unbiasicovfac = (ximisc.getmatrixdiag(np.matrix(cov) * np.matrix(icov))).mean()
        print "using this unbiasicovfac correction, dividing cov by this", unbiasicovfac
        cov = cov / unbiasicovfac

        diagstat = np.zeros(xiNNd.ndata)
        diagtot = np.zeros(xiNNd.ndata)
        for i in range(len(diagstat)):
            diagstat[i] = cov[i, i]
        diagvar = np.zeros(xiNNd.ndata)
        xiangdiffvar = (0.5 * (xiangdhigh.xi.flatten() - xiangdlow.xi.flatten())) ** 2
        diagvar[0:splitxi0] = xiangdiffvar[0:splitxi0]
        nxi0 = len(xiNNd.xi0)
        diagvar[nxi0 : nxi0 + splitxi2] = xiangdiffvar[nxi0 : nxi0 + splitxi2]
        print "ang high/low variance: ", diagvar / diagstat
        diagvar = diagvar + (mydelta.flatten()) ** 2
        print "bias variance: ", (mydelta.flatten()) ** 2 / diagstat
        ## add it into the covarianace matrix.
        for i in range(xiNNd.ndata):
            cov[i, i] += diagvar[i]
            diagtot[i] = cov[i, i]
        print "total sys variance fraction", diagtot / diagstat

        ## make it a matrix.
        cov = np.matrix(cov)
        icov = cov.I

        fcovout = covstatfname + ".sys"
        ## print the covariance and icov to new file.
        printcov(cov, fcovout)
        tmp = fcovout.split("/")
        tmp[-1] = "i" + tmp[-1]
        ifcovout = "/".join(tmp)
        printcov(icov, ifcovout)

        xifinal = xiell.xiell(sxilist=[xiNNd.svec, xidebiased], icovfname=ifcovout)
        if fname is not None:
            ofp = open(fname, "w")
            ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
            for i in range(len(xifinal.svec.flatten())):
                ofp.write("%e %e\n" % (xifinal.svec.flatten()[i], xifinal.xi.flatten()[i]))
            ofp.close()

        return xifinal, cov

    else:
        #  except:
        print "cov file name does not match input splits, returning None!"
        xifinal = xiell.xiell(sxilist=[xiNNd.svec, xidebiased])
        if fname is not None:
            ofp = open(fname, "w")
            ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
            for i in range(len(xifinal.svec.flatten())):
                ofp.write("%e %e\n" % (xifinal.svec.flatten()[i], xifinal.xi.flatten()[i]))
            ofp.close()
            return xifinal, None
Example #3
0
def debiasdataandcovwp(
    wpNNd, wpangd, wpangdhigh, wpangdlow, wpNNm, wpangm, wp012m, splitwp, wpstart, wpend, covstatfname, fname=None
):
    """
  subtract the bias measured from the tiled mocks from the data, return a debiased combination.
  print it to a file (fname) to be fed to bethalexie code in long format.
  Also take in statistical covariance matrix and add two sources of systematics.
  """
    wpcorrdtmp = wpcorrect(wpNNd, wpangd, splitwp, wpstart, wpend)
    wpcorrm = wpcorrect(wpNNm, wpangm, splitwp, wpstart, wpend)
    wpdebiased = copy.deepcopy(wpcorrdtmp.wp)
    mydelta = wp012m.wp[wpstart:] - wpcorrm.wp
    print "fractional wp correction:"
    print mydelta / wpcorrdtmp.wp
    wpdebiased = wpdebiased + mydelta

    ## now the cov.
    ## make sure this is the cov for the corrected statistic with same splits.
    if 0 == 0:
        #  try:
        cov = np.loadtxt(covstatfname)
        assert len(cov[:, 0]) == len(wpdebiased)
        splitz = covstatfname.split("splitswp")[1].split("_")
        assert len(splitz) >= 2
        ilist = []
        for ss in splitz[:2]:
            ilist.append(int(ss))
        assert ilist[0] == splitwp
        assert ilist[1] == wpstart

        ## new jan 2 2014!!!  forgot to take into account the unbiasicov fac.  derive if from
        ## product of cov and icov.
        ## guess icovfname
        tmp = covstatfname.split("/")
        tmp[-1] = "i" + tmp[-1]
        icovstatfname = "/".join(tmp)
        icov = np.loadtxt(icovstatfname)

        unbiasicovfac = (ximisc.getmatrixdiag(np.matrix(cov) * np.matrix(icov))).mean()
        print "using htis unbiasicovfac correction, dividing cov by this", unbiasicovfac
        cov = cov / unbiasicovfac

        ndatacorr = len(wpcorrdtmp.wp)

        diagstat = np.zeros(ndatacorr)
        diagtot = np.zeros(ndatacorr)
        diagvar = np.zeros(ndatacorr)
        for i in range(len(diagstat)):
            diagstat[i] = cov[i, i]
        ## this must agree with wpcorrect assignmeents!
        wpangdiffvar = (0.5 * (wpangdhigh.wp - wpangdlow.wp)) ** 2
        diagvar[0 : splitwp - wpstart] = wpangdiffvar[wpstart:splitwp]
        print "wp ang high/low variance: ", diagvar / diagstat
        print "bias correction: ", mydelta
        diagvar = diagvar + (mydelta.flatten()) ** 2
        print "bias variance contribution: ", (mydelta.flatten()) ** 2 / diagstat
        ## add it into the covarianace matrix.
        for i in range(ndatacorr):
            cov[i, i] += diagvar[i]
            diagtot[i] = cov[i, i]
        print "total sys variance fraction", diagtot / diagstat

        ## make it a matrix.
        cov = np.matrix(cov)
        icov = cov.I

        fcovout = covstatfname + ".sys"
        ## print the covariance and icov to new file.
        printcov(cov, fcovout)
        tmp = fcovout.split("/")
        tmp[-1] = "i" + tmp[-1]
        ifcovout = "/".join(tmp)
        printcov(icov, ifcovout)

        wpfinal = wp.wp(rpwplist=[wpcorrdtmp.rsig, wpdebiased], icovfname=ifcovout)
        if fname is not None:
            wpfinal.printwp(fname)
        return wpfinal, cov

    else:
        #  except:
        print "cov file name does not match input splits, returning None!"
        wpfinal = wp.wp(rpwplist=[wpcorrdtmp.rsig, wpdebiased])
        if fname is not None:
            wpfinal.printwp(fname)
        return wpfinal, None
Example #4
0
def debiasdataandcovwp(wpNNd,
                       wpangd,
                       wpangdhigh,
                       wpangdlow,
                       wpNNm,
                       wpangm,
                       wp012m,
                       splitwp,
                       wpstart,
                       wpend,
                       covstatfname,
                       fname=None):
    """
  subtract the bias measured from the tiled mocks from the data, return a debiased combination.
  print it to a file (fname) to be fed to bethalexie code in long format.
  Also take in statistical covariance matrix and add two sources of systematics.
  """
    wpcorrdtmp = wpcorrect(wpNNd, wpangd, splitwp, wpstart, wpend)
    wpcorrm = wpcorrect(wpNNm, wpangm, splitwp, wpstart, wpend)
    wpdebiased = copy.deepcopy(wpcorrdtmp.wp)
    mydelta = wp012m.wp[wpstart:] - wpcorrm.wp
    print 'fractional wp correction:'
    print mydelta / wpcorrdtmp.wp
    wpdebiased = wpdebiased + mydelta

    ## now the cov.
    ## make sure this is the cov for the corrected statistic with same splits.
    if (0 == 0):
        #  try:
        cov = np.loadtxt(covstatfname)
        assert len(cov[:, 0]) == len(wpdebiased)
        splitz = covstatfname.split('splitswp')[1].split('_')
        assert len(splitz) >= 2
        ilist = []
        for ss in splitz[:2]:
            ilist.append(int(ss))
        assert ilist[0] == splitwp
        assert ilist[1] == wpstart

        ## new jan 2 2014!!!  forgot to take into account the unbiasicov fac.  derive if from
        ## product of cov and icov.
        ## guess icovfname
        tmp = covstatfname.split('/')
        tmp[-1] = 'i' + tmp[-1]
        icovstatfname = '/'.join(tmp)
        icov = np.loadtxt(icovstatfname)

        unbiasicovfac = (ximisc.getmatrixdiag(
            np.matrix(cov) * np.matrix(icov))).mean()
        print 'using htis unbiasicovfac correction, dividing cov by this', unbiasicovfac
        cov = cov / unbiasicovfac

        ndatacorr = len(wpcorrdtmp.wp)

        diagstat = np.zeros(ndatacorr)
        diagtot = np.zeros(ndatacorr)
        diagvar = np.zeros(ndatacorr)
        for i in range(len(diagstat)):
            diagstat[i] = cov[i, i]
        ## this must agree with wpcorrect assignmeents!
        wpangdiffvar = (0.5 * (wpangdhigh.wp - wpangdlow.wp))**2
        diagvar[0:splitwp - wpstart] = wpangdiffvar[wpstart:splitwp]
        print 'wp ang high/low variance: ', diagvar / diagstat
        print 'bias correction: ', mydelta
        diagvar = diagvar + (mydelta.flatten())**2
        print 'bias variance contribution: ', (mydelta.flatten())**2 / diagstat
        ## add it into the covarianace matrix.
        for i in range(ndatacorr):
            cov[i, i] += diagvar[i]
            diagtot[i] = cov[i, i]
        print 'total sys variance fraction', diagtot / diagstat

        ## make it a matrix.
        cov = np.matrix(cov)
        icov = cov.I

        fcovout = covstatfname + '.sys'
        ## print the covariance and icov to new file.
        printcov(cov, fcovout)
        tmp = fcovout.split('/')
        tmp[-1] = 'i' + tmp[-1]
        ifcovout = '/'.join(tmp)
        printcov(icov, ifcovout)

        wpfinal = wp.wp(rpwplist=[wpcorrdtmp.rsig, wpdebiased],
                        icovfname=ifcovout)
        if fname is not None:
            wpfinal.printwp(fname)
        return wpfinal, cov

    else:
        #  except:
        print 'cov file name does not match input splits, returning None!'
        wpfinal = wp.wp(rpwplist=[wpcorrdtmp.rsig, wpdebiased])
        if fname is not None:
            wpfinal.printwp(fname)
        return wpfinal, None
Example #5
0
def debiasdataandcovxiMwp(xiNNd,
                          xiangd,
                          xiangdhigh,
                          xiangdlow,
                          xiNNm,
                          xiangm,
                          xi012m,
                          splitxi0,
                          splitxi2,
                          wpNNd,
                          wpangd,
                          wpangdhigh,
                          wpangdlow,
                          wpNNm,
                          wpangm,
                          wp012m,
                          splitwp,
                          wpstart,
                          wpend,
                          covstatfname,
                          nell=2,
                          fname=None):
    #def debiasdataandcovwp(wpNNd, wpangd, wpangdhigh, wpangdlow, wpNNm, wpangm, wp012m, splitwp, wpstart,wpend,covstatfname,fname=None):
    """
  subtract the bias measured from the tiled mocks from the data, return a debiased combination.
  print it to a file (fname) to be fed to bethalexie code in long format.
  Also take in statistical covariance matrix and add two sources of systematics.
  """
    #def xiwpcorrect(xiNNin, xiangin,splitxi0,splitxi2,wpNNin, wpangin, splitwp, wpstart, wpend):
    xiwpcorrdtmp = xiwpcorrect(xiNNd, xiangd, splitxi0,splitxi2,\
                           wpNNd, wpangd, splitwp, wpstart, wpend)

    xiwpcorrm = xiwpcorrect(xiNNm, xiangm, splitxi0, splitxi2,\
                          wpNNm,wpangm,splitwp,wpstart,wpend)

    xiwpdebiased = copy.deepcopy(xiwpcorrdtmp)
    #tmp!
    #  print xiwpdebiased.xiell
    #  print xiwpdebiased.wp

    mydeltaxi = xi012m.xi - xiwpcorrm.xiell.xi  ## subtract xi objects.
    mydeltawp = wp012m.wp[wpstart:wpend + 1] - xiwpcorrm.wp.wp
    xiwpdebiased.xiell.xi = xiwpdebiased.xiell.xi + mydeltaxi
    xiwpdebiased.wp.wp = xiwpdebiased.wp.wp + mydeltawp
    xiwpdebiased.xiwp = np.concatenate(
        (xiwpdebiased.xiell.xilong, xiwpdebiased.wp.wp))
    xiwpanghigh = xiwp.xiwp(xiangdhigh, wpangdhigh)
    xiwpanglow = xiwp.xiwp(xiangdlow, wpangdlow)

    ## now the cov.
    ## make sure this is the cov for the corrected statistic with same splits.
    if (0 == 0):
        #  try:
        cov = np.loadtxt(covstatfname)
        assert len(cov[:, 0]) == xiwpdebiased.ntot
        splitz = covstatfname.split('splitswp')[0].split('splits')[1].split(
            '_')
        assert len(splitz) >= nell
        ilist = []
        tmp = 0
        for ss in splitz:
            tmp += 1
            ilist.append(int(ss))
            if tmp >= nell:
                break
        assert ilist[0] == splitxi0
        assert ilist[1] == splitxi2
        splitzwp = covstatfname.split('splitswp')[1].split('_')
        print splitzwp
        assert len(splitzwp) >= 3
        ilist = []
        tmp = 0
        for ss in splitzwp:
            tmp += 1
            ilist.append(int(ss))
            if tmp >= 3:
                break
        assert ilist[0] == splitwp
        assert ilist[1] == wpstart
        assert ilist[2] == wpend

        ## new jan 2 2014!!!  forgot to take into account the unbiasicov fac.  derive if from
        ## product of cov and icov.
        ## guess icovfname
        tmp = covstatfname.split('/')
        tmp[-1] = 'i' + tmp[-1]
        icovstatfname = '/'.join(tmp)
        icov = np.loadtxt(icovstatfname)

        unbiasicovfac = (ximisc.getmatrixdiag(
            np.matrix(cov) * np.matrix(icov))).mean()
        print 'using this unbiasicovfac correction, dividing cov by this', unbiasicovfac
        cov = cov / unbiasicovfac

        diagstat = np.zeros(xiwpdebiased.ntot)
        diagtot = np.zeros(xiwpdebiased.ntot)
        diagvar = np.zeros(xiwpdebiased.ntot)
        for i in range(len(diagstat)):
            diagstat[i] = cov[i, i]

        xiangdiffvar = (0.5 *
                        (xiangdhigh.xi.flatten() - xiangdlow.xi.flatten()))**2
        diagvar[0:splitxi0] = xiangdiffvar[0:splitxi0]
        nxi0 = len(xiNNd.xi0)
        nxi2 = len(xiNNd.xi2)
        diagvar[nxi0:nxi0 + splitxi2] = xiangdiffvar[nxi0:nxi0 + splitxi2]
        wpangdiffvar = (0.5 * (wpangdhigh.wp - wpangdlow.wp))**2
        diagvar[nxi0 + nxi2:nxi0 + nxi2 + splitwp -
                wpstart] = wpangdiffvar[wpstart:splitwp]
        print 'ang high/low variance: ', diagvar / diagstat
        diagvar[0:nxi0 +
                nxi2] = diagvar[0:nxi0 + nxi2] + (mydeltaxi.flatten())**2
        diagvar[nxi0 + nxi2:] = diagvar[nxi0 + nxi2:] + (mydeltawp)**2
        print 'bias variance xi: ', (
            mydeltaxi.flatten())**2 / diagstat[0:nxi0 + nxi2]
        print 'bias variance wp: ', (mydeltawp)**2 / diagstat[nxi0 + nxi2:]
        ## add it into the covarianace matrix.
        for i in range(xiwpdebiased.ntot):
            cov[i, i] += diagvar[i]
            diagtot[i] = cov[i, i]
        print 'total sys variance fraction', diagtot / diagstat

        ## make it a matrix.
        cov = np.matrix(cov)
        icov = cov.I

        fcovout = covstatfname + '.sys'
        ## print the covariance and icov to new file.
        printcov(cov, fcovout)
        tmp = fcovout.split('/')
        tmp[-1] = 'i' + tmp[-1]
        ifcovout = '/'.join(tmp)
        printcov(icov, ifcovout)

        xiwpfinal = xiwp.xiwp(xiwpdebiased.xiell,
                              xiwpdebiased.wp,
                              icovfname=ifcovout)
        if fname is not None:
            ofp = open(fname, 'w')
            ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
            for i in range(len(xiwpfinal.xiell.svec.flatten())):
                ofp.write('%e %e\n' % (xiwpfinal.xiell.svec.flatten()[i],
                                       xiwpfinal.xiell.xi.flatten()[i]))
            for i in range(len(xiwpfinal.wp.wp)):
                ofp.write('%e %e\n' %
                          (xiwpfinal.wp.rsig[i], xiwpfinal.wp.wp[i]))
            ofp.close()

        return xiwpfinal, cov

    else:
        print 'cov file name does not match input splits, returning None!'
        xiwpfinal = xiwp.xiwp(xiwpdebiased.xiell,
                              xiwpdebiased.wp,
                              icovfname=ifcovout)
        if fname is not None:
            ofp = open(fname, 'w')
            ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
            for i in range(len(xiwpfinal.xi.svec.flatten())):
                ofp.write('%e %e\n' % (xiwpfinal.xiell.svec.flatten()[i],
                                       xiwpfinal.xiell.xi.flatten()[i]))
            for i in range(len(xiwpfinal.wp.wp)):
                ofp.write('%e %e\n' %
                          (xiwpfinal.wp.rsig[i], xiwpfinal.wp.wp[i]))
            ofp.close()

        return xiwpfinal, None
Example #6
0
def debiasdataandcov(xiNNd,
                     xiangd,
                     xiangdhigh,
                     xiangdlow,
                     xiNNm,
                     xiangm,
                     xi012m,
                     splitxi0,
                     splitxi2,
                     covstatfname,
                     nell=2,
                     fname=None):
    """
  subtract the bias measured from the tiled mocks from the data, return a debiased combination.
  print it to a file (fname) to be fed to bethalexie code in long format.
  Also take in statistical covariance matrix and add two sources of systematics.
  """
    xicorrdtmp = xicorrect(xiNNd, xiangd, splitxi0, splitxi2)
    xicorrm = xicorrect(xiNNm, xiangm, splitxi0, splitxi2)
    xidebiased = copy.deepcopy(xicorrdtmp.xi)
    mydelta = xi012m.xi - xicorrm.xi
    xidebiased = xidebiased + mydelta

    ## now the cov.
    ## make sure this is the cov for the corrected statistic with same splits.
    if (0 == 0):
        #  try:
        cov = np.loadtxt(covstatfname)
        assert len(cov[:, 0]) == xiNNd.ndata
        splitz = covstatfname.split('splits')[1].split('_')
        assert len(splitz) == nell
        ilist = []
        for ss in splitz:
            ilist.append(int(ss))
        assert ilist[0] == splitxi0
        assert ilist[1] == splitxi2

        ## new jan 2 2014!!!  forgot to take into account the unbiasicov fac.  derive if from
        ## product of cov and icov.
        ## guess icovfname
        tmp = covstatfname.split('/')
        tmp[-1] = 'i' + tmp[-1]
        icovstatfname = '/'.join(tmp)
        icov = np.loadtxt(icovstatfname)

        unbiasicovfac = (ximisc.getmatrixdiag(
            np.matrix(cov) * np.matrix(icov))).mean()
        print 'using this unbiasicovfac correction, dividing cov by this', unbiasicovfac
        cov = cov / unbiasicovfac

        diagstat = np.zeros(xiNNd.ndata)
        diagtot = np.zeros(xiNNd.ndata)
        for i in range(len(diagstat)):
            diagstat[i] = cov[i, i]
        diagvar = np.zeros(xiNNd.ndata)
        xiangdiffvar = (0.5 *
                        (xiangdhigh.xi.flatten() - xiangdlow.xi.flatten()))**2
        diagvar[0:splitxi0] = xiangdiffvar[0:splitxi0]
        nxi0 = len(xiNNd.xi0)
        diagvar[nxi0:nxi0 + splitxi2] = xiangdiffvar[nxi0:nxi0 + splitxi2]
        print 'ang high/low variance: ', diagvar / diagstat
        diagvar = diagvar + (mydelta.flatten())**2
        print 'bias variance: ', (mydelta.flatten())**2 / diagstat
        ## add it into the covarianace matrix.
        for i in range(xiNNd.ndata):
            cov[i, i] += diagvar[i]
            diagtot[i] = cov[i, i]
        print 'total sys variance fraction', diagtot / diagstat

        ## make it a matrix.
        cov = np.matrix(cov)
        icov = cov.I

        fcovout = covstatfname + '.sys'
        ## print the covariance and icov to new file.
        printcov(cov, fcovout)
        tmp = fcovout.split('/')
        tmp[-1] = 'i' + tmp[-1]
        ifcovout = '/'.join(tmp)
        printcov(icov, ifcovout)

        xifinal = xiell.xiell(sxilist=[xiNNd.svec, xidebiased],
                              icovfname=ifcovout)
        if fname is not None:
            ofp = open(fname, 'w')
            ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
            for i in range(len(xifinal.svec.flatten())):
                ofp.write('%e %e\n' %
                          (xifinal.svec.flatten()[i], xifinal.xi.flatten()[i]))
            ofp.close()

        return xifinal, cov

    else:
        #  except:
        print 'cov file name does not match input splits, returning None!'
        xifinal = xiell.xiell(sxilist=[xiNNd.svec, xidebiased])
        if fname is not None:
            ofp = open(fname, 'w')
            ofp.write("# ellmax = %d\n" % ((nell - 1) * 2))
            for i in range(len(xifinal.svec.flatten())):
                ofp.write('%e %e\n' %
                          (xifinal.svec.flatten()[i], xifinal.xi.flatten()[i]))
            ofp.close()
            return xifinal, None