Beispiel #1
0
def jackknife_lensums(data, jackreg_col=None, weights=None):
    """
    jackknife the data. If regions are sent, use them for jackknifing,
    otherwise jackknife one object at a time

    parameters
    ----------
    data: array
        An array with fields 'dsum' and 'wsum'. If shear style
        is lensfit, dsensum is needed rather than wsum.
    jackreg_col: string, optional
        column name holding the jackknife region ids
    weights: array, optional
        Additional weights per lens

    returns
    -------
    dsig, dsig_cov

    dsig: array
        The delta sigma in radial bins [nrad]
    dsig_cov: array
        The covariance matrix of delta sigma [nrad,nrad]
    """
    import jackknife

    jdsum, jwsum = get_jackknife_sums(data, weights=weights, jackreg_col=jackreg_col)
    dsig, dsig_cov = jackknife.wjackknife(vsum=jdsum, wsum=jwsum)

    return dsig, dsig_cov
Beispiel #2
0
def calc_gmean(data):
    """
    get gtrue, gmeas, gcov
    """
    import jackknife
    gtrue = data['shear_true'].mean(axis=0)
    gmeas = numpy.zeros(2)

    wts=get_weights(data)

    '''
    g1sum = (data['g'][:,0]*wts).sum()
    g2sum = (data['g'][:,1]*wts).sum()

    g1sensum = (data['g_sens'][:,0]*wts).sum()
    g2sensum = (data['g_sens'][:,1]*wts).sum()

    gmeas[0]=g1sum/g1sensum
    gmeas[1]=g2sum/g2sensum
    print("gmeas: ",gmeas)
    '''

    wa=wts[:,newaxis]
    jdsum=data['g']*wa
    if 'g_sens' in data.dtype.names:
        jwsum=data['g_sens']*wa
    else:
        jwsum=numpy.ones( data['g'].shape )*wa
    #print(jdsum.shape)

    gmeas,gcov=jackknife.wjackknife(vsum=jdsum, wsum=jwsum)

    return gtrue, gmeas, gcov
Beispiel #3
0
def average_lensums_weighted(lout, weights_in, jackreg_col=None):
    """
    average over all the individual lensums with additional weights

    The covariance matrix is estimated from jackknifing. If regions are sent,
    use them for jackknifing, otherwise jackknife one object at a time

    parameters
    ----------
    data: array
        Array containing the outputs from xshear
    weights: array
        Additional weights for each lens
    jackreg_col: string, optional
        column name holding the jackknife region ids
    """

    import jackknife

    nlens = lout.size
    nrad = lout["rsum"][0].size

    weights = weights_in.copy()
    weights *= 1.0 / weights.max()

    if weights.size != nlens:
        raise ValueError("weights not same size as lensout, " "%d instead of %d" % (weights.size, nlens))

    totweights = weights.sum()

    shear_style = get_shear_style(lout)

    # broadcast it
    wa = weights[:, newaxis]
    comb = averaged_struct(nrad, shear_style=shear_style)

    # weight is the weight for the lens.  Call this weightsum to
    # indicate a sum over multiple lenses
    comb["weightsum"] = (lout["weight"] * weights).sum()

    comb["totpairs"] = lout["totpairs"].sum()

    comb["wsum"] = (lout["wsum"] * wa).sum(axis=0)
    comb["npair"] = lout["npair"].sum(axis=0)
    comb["rsum"] = (lout["rsum"] * wa).sum(axis=0)
    comb["dsum"] = (lout["dsum"] * wa).sum(axis=0)
    comb["osum"] = (lout["osum"] * wa).sum(axis=0)

    # averages
    comb["r"] = comb["rsum"] / comb["wsum"]

    if shear_style == "lensfit":
        comb["dsensum"] = (lout["dsensum"] * wa).sum(axis=0)
        comb["osensum"] = (lout["osensum"] * wa).sum(axis=0)
        comb["dsig"] = comb["dsum"] / comb["dsensum"]
        comb["osig"] = comb["osum"] / comb["osensum"]
    else:
        comb["dsig"] = comb["dsum"] / comb["wsum"]
        comb["osig"] = comb["osum"] / comb["wsum"]

    # we calculate boost factors from this, wsum_mean/wsum_mean_random
    comb["wsum_mean"] = comb["wsum"] / totweights

    # jackknife to get the covariance matrix
    m, cov = jackknife_lensums(lout, weights=weights, jackreg_col=jackreg_col)

    comb["dsigcov"][0] = cov
    comb["dsigcor"][0] = jackknife.covar2corr(cov)
    comb["dsigerr"][0] = sqrt(diag(cov))

    #
    # also jackknife the wsums, used for errors on boost factors
    #

    # this will broadcase
    w_wsum_all = lout["wsum"] * wa

    # but here we need fully expanded version
    weights_big = ones((lout.size, nrad)) * wa

    m, cov = jackknife.wjackknife(vsum=w_wsum_all, wsum=weights_big)

    comb["wsum_mean_err"][0] = sqrt(diag(cov))
    return comb
Beispiel #4
0
def average_lensums_weighted_slow(lout, weights):
    """

    Reduce the lens-by-lens lensums by summing over
    all the individual sums and producing averages

    """
    import jackknife

    nlens = lout.size
    nrad = lout["rsum"][0].size

    if weights.size != nlens:
        raise ValueError("weights not same size as lensout, " "%d instead of %d" % (weights.size, nlens))

    totweights = weights.sum()

    if "dsensum" in lout.dtype.names:
        shear_style = "lensfit"
    else:
        shear_style = "reduced"

    comb = averaged_struct(nrad, shear_style=shear_style)

    # weight is the weight for the lens.  Call this weightsum to
    # indicate a sum over multiple lenses

    comb["weightsum"][0] = (lout["weight"] * weights).sum()

    # should not use this for anything since weights
    # make it non-integer
    comb["totpairs"][0] = lout["totpairs"].sum()

    # these will get the extra weight
    # WE MUST MAKE A COPY!! ARRGGG THIS BIT ME!!!
    jwsum = lout["wsum"].copy()
    jdsum = lout["dsum"].copy()
    for i in xrange(nrad):

        npair = lout["npair"][:, i].sum()

        w_rsum = (lout["rsum"][:, i] * weights).sum()

        w_wsum = (lout["wsum"][:, i] * weights).sum()
        w_dsum = (lout["dsum"][:, i] * weights).sum()
        w_osum = (lout["osum"][:, i] * weights).sum()

        comb["npair"][0, i] = npair
        comb["rsum"][0, i] = w_rsum
        comb["wsum"][0, i] = w_wsum
        comb["dsum"][0, i] = w_dsum
        comb["osum"][0, i] = w_osum

        # averages
        comb["r"][0, i] = w_rsum / w_wsum

        jdsum[:, i] *= weights
        if shear_style == "lensfit":
            comb["dsensum"][0, i] = lout["dsensum"][:, i].sum()
            comb["osensum"][0, i] = lout["osensum"][:, i].sum()
            comb["dsig"][0, i] = w_dsum / comb["dsensum"][0, i]
            comb["osig"][0, i] = w_osum / comb["osensum"][0, i]
            jwsum[:, i] = comb["dsensum"] * weights
        else:

            comb["dsig"][0, i] = w_dsum / w_wsum
            comb["osig"][0, i] = w_osum / w_wsum
            jwsum[:, i] = comb["wsum"] * weights

        # this is average wsum over lenses
        # we calculate clustering correction from this, wsum_mean/wsum_mean_random
        comb["wsum_mean"][0, i] = w_wsum / totweights

    # jwsum will be wsum*weights or dsensum*weights
    m, cov = jackknife.wjackknife(vsum=jdsum, wsum=jwsum)

    comb["dsigcov"][0, :, :] = cov
    comb["dsigcor"][0, :, :] = jackknife.covar2corr(cov)
    comb["dsigerr"][0, :] = sqrt(diag(cov))

    # make weights in shape of wsum
    w = zeros(lout["wsum"].shape)
    w_wsum = zeros(lout["wsum"].shape)
    for i in xrange(lout.size):
        w_wsum[i, :] = lout["wsum"][i, :] * weights[i]
        w[i, :] = weights[i]
    m, cov = jackknife.wjackknife(vsum=w_wsum, wsum=w)
    comb["wsum_mean_err"][0, :] = sqrt(diag(cov))

    return comb
Beispiel #5
0
def average_lensums_slow(lout, weights=None):
    """

    combine the lens-by-lens lensums by summing over
    all the individual sums and producing averages

    This uses the averaged_dtype

    This is used by the binner routines

    """
    import jackknife

    if weights is not None:
        return average_lensums_weighted(lout, weights)

    nlens = lout.size
    nrad = lout["rsum"][0].size

    if "dsensum" in lout.dtype.names:
        shear_style = "lensfit"
    else:
        shear_style = "reduced"

    comb = averaged_struct(nrad, shear_style=shear_style)

    # weight is the weight for the lens.  Call this weightsum to
    # indicate a sum over multiple lenses
    comb["weightsum"] = lout["weight"].sum(axis=0)

    comb["totpairs"] = lout["totpairs"].sum(axis=0)

    for i in xrange(nrad):
        npair = lout["npair"][:, i].sum()
        rsum = lout["rsum"][:, i].sum()

        wsum = lout["wsum"][:, i].sum()
        wsum2 = (lout["wsum"][:, i] ** 2).sum()
        dsum = lout["dsum"][:, i].sum()
        osum = lout["osum"][:, i].sum()

        comb["npair"][0, i] = npair
        comb["rsum"][0, i] = rsum
        comb["wsum"][0, i] = wsum
        comb["dsum"][0, i] = dsum
        comb["osum"][0, i] = osum

        # averages
        comb["r"][0, i] = rsum / wsum

        if shear_style == "lensfit":
            comb["dsensum"][0, i] = lout["dsensum"][:, i].sum()
            comb["osensum"][0, i] = lout["osensum"][:, i].sum()
            comb["dsig"][0, i] = dsum / comb["dsensum"][0, i]
            comb["osig"][0, i] = osum / comb["osensum"][0, i]

        else:

            comb["dsig"][0, i] = dsum / wsum
            comb["osig"][0, i] = osum / wsum

        comb["wsum_mean"][0, i] = wsum / nlens

    if shear_style == "lensfit":
        m, cov = jackknife.wjackknife(vsum=lout["dsum"], wsum=lout["dsensum"])
    else:
        m, cov = jackknife.wjackknife(vsum=lout["dsum"], wsum=lout["wsum"])

    comb["dsigcov"][0, :, :] = cov
    comb["dsigcor"][0, :, :] = jackknife.covar2corr(cov)
    comb["dsigerr"][0, :] = sqrt(diag(cov))

    w = numpy.ones(lout["wsum"].shape)
    m, cov = jackknife.wjackknife(vsum=lout["wsum"], wsum=w)
    comb["wsum_mean_err"][0, :] = sqrt(diag(cov))

    return comb
Beispiel #6
0
def average_lensums(lout, weights=None, jackreg_col=None):
    """
    average over all the individual lensums

    The covariance matrix is estimated from jackknifing. If regions are sent,
    use them for jackknifing, otherwise jackknife one object at a time

    parameters
    ----------
    data: array
        Array containing the outputs from xshear
    weights: array, optional
        Optional weights
    jackreg_col: string, optional
        column name holding the jackknife region ids
    """
    import jackknife

    if weights is not None:
        print("        using extra weights in averages")
        return average_lensums_weighted(lout, weights, jackreg_col=jackreg_col)

    # print("yes using no weights")

    nlens = lout.size
    nrad = lout["rsum"][0].size

    shear_style = get_shear_style(lout)

    comb = averaged_struct(nrad, shear_style=shear_style)

    comb["nlenses"][0] = nlens

    # weight is the weight for the lens.  Call this weightsum to
    # indicate a sum over multiple lenses
    comb["weightsum"][0] = lout["weight"].sum()

    comb["totpairs"][0] = lout["totpairs"].sum()

    comb["npair"][0] = lout["npair"].sum(axis=0)
    comb["rsum"][0] = lout["rsum"].sum(axis=0)
    comb["wsum"][0] = lout["wsum"].sum(axis=0)
    comb["dsum"][0] = lout["dsum"].sum(axis=0)
    comb["osum"][0] = lout["osum"].sum(axis=0)

    # averages
    comb["r"][0] = comb["rsum"][0] / comb["wsum"][0]

    if shear_style == "lensfit":
        comb["dsensum"][0] = lout["dsensum"].sum(axis=0)
        comb["osensum"][0] = lout["osensum"].sum(axis=0)
        comb["dsig"][0] = comb["dsum"][0] / comb["dsensum"][0]
        comb["osig"][0] = comb["osum"][0] / comb["osensum"][0]
    else:
        comb["dsig"][0] = comb["dsum"][0] / comb["wsum"][0]
        comb["osig"][0] = comb["osum"][0] / comb["wsum"][0]

    # this is average wsum over lenses
    # we calculate boost factors from this, wsum_mean/wsum_mean_random
    comb["wsum_mean"][0] = comb["wsum"][0] / nlens

    # jackknife to get the covariance matrix
    m, cov = jackknife_lensums(lout, jackreg_col=jackreg_col)

    comb["dsigcov"][0] = cov
    comb["dsigcor"][0] = jackknife.covar2corr(cov)
    comb["dsigerr"][0] = sqrt(diag(cov))

    # also jackknife the wsums, used for errors on boost factors
    w = ones(lout["wsum"].shape)
    m, cov = jackknife.wjackknife(vsum=lout["wsum"], wsum=w)

    comb["wsum_mean_err"][0] = sqrt(diag(cov))
    return comb
Beispiel #7
0
def average_lensums_weighted(lout, weights):
    """

    Reduce the lens-by-lens lensums by summing over
    all the individual sums and producing averages

    """

    nlens = lout.size
    nbin = lout['rsum'][0].size

    if weights.size != nlens:
        raise ValueError("weights not same size as lensout, "
                         "%d instead of %d" % (weights.size,nlens))

    totweights = weights.sum()

    comb = averaged_struct(nbin)

    # weight is the weight for the lens.  Call this weightsum to
    # indicate a sum over multiple lenses

    comb['weightsum'][0] = (lout['weight']*weights).sum()

    if 'sshsum' in lout.dtype.names:
        comb['sshsum'][0] = (lout['sshsum']*weights).sum()
        comb['ssh'] = comb['sshsum']/comb['weightsum']
    else:
        # this makes ssh unity
        comb['sshsum'][0] = comb['weightsum'].sum()
        comb['ssh'] = comb['sshsum']/comb['weightsum']

    # should not use this for anything since weights
    # make it non-integer
    comb['totpairs'][0] = lout['totpairs'].sum()


    # these will get the extra weight
    # WE MUST MAKE A COPY!! ARRGGG THIS BIT ME!!!
    jwsum = lout['wsum'].copy()
    jdsum = lout['dsum'].copy()
    for i in xrange(nbin):

        npair = lout['npair'][:,i].sum()

        rsum  = lout['rsum'][:,i].sum()

        w_wsum = (lout['wsum'][:,i]*weights).sum()
        w_dsum = (lout['dsum'][:,i]*weights).sum()
        w_osum = (lout['osum'][:,i]*weights).sum()


        comb['npair'][0,i] = npair
        comb['rsum'][0,i]  = rsum
        comb['wsum'][0,i]  = w_wsum
        comb['dsum'][0,i]  = w_dsum
        comb['osum'][0,i]  = w_osum

        # averages
        tr = rsum/w_wsum
        if tr > 100:
            # probably old way
            comb['r'][0,i] = rsum/weights.sum()
        else:
            comb['r'][0,i] = tr

        comb['dsig'][0,i] = w_dsum/w_wsum
        comb['osig'][0,i] = w_osum/w_wsum


        # this is average wsum over lenses
        # we calculate clustering correction from this, wsum_mean/wsum_mean_random
        comb['wsum_mean'][0,i] = w_wsum/totweights

        jwsum[:,i] *= weights
        jdsum[:,i] *= weights

    m,cov=jackknife.wjackknife(vsum=jdsum, wsum=jwsum)
    comb['dsigcov'][0,:,:] = cov
    comb['dsigcor'][0,:,:] = jackknife.covar2corr(cov)
    comb['dsigerr'][0,:] = sqrt(diag(cov))

 
    # make weights in shape of wsum
    w      = zeros(lout['wsum'].shape)
    w_wsum = zeros(lout['wsum'].shape)
    for i in xrange(lout.size):
        w_wsum[i,:] = lout['wsum'][i,:]*weights[i]
        w[i,:] = weights[i]
    m,cov = jackknife.wjackknife(vsum=w_wsum, wsum=w)
    comb['wsum_mean_err'][0,:] = sqrt(diag(cov))


    return comb
Beispiel #8
0
def average_lensums(lout, weights=None):
    """

    combine the lens-by-lens lensums by summing over
    all the individual sums and producing averages

    This uses the averaged_dtype

    This is used by the binner routines

    """

    if weights is not None:
        return average_lensums_weighted(lout,weights)

    nlens = lout.size
    nbin = lout['rsum'][0].size

    comb = averaged_struct(nbin)

    # weight is the weight for the lens.  Call this weightsum to
    # indicate a sum over multiple lenses
    comb['weightsum'][0] = lout['weight'].sum()
    if 'sshsum' in lout.dtype.names:
        comb['sshsum'][0] = lout['sshsum'].sum()
        comb['ssh'] = comb['sshsum']/comb['weightsum']
    else:
        # this makes ssh unity
        comb['sshsum'][0] = comb['weightsum'][0]
        comb['ssh'] = comb['sshsum']/comb['weightsum']

    comb['totpairs'][0] = lout['totpairs'].sum()


    for i in xrange(nbin):
        npair = lout['npair'][:,i].sum()
        rsum  = lout['rsum'][:,i].sum()

        wsum = lout['wsum'][:,i].sum()
        wsum2 = (lout['wsum'][:,i]**2).sum()
        dsum = lout['dsum'][:,i].sum()
        osum = lout['osum'][:,i].sum()

        comb['npair'][0,i] = npair
        comb['rsum'][0,i] = rsum
        comb['wsum'][0,i] = wsum
        comb['dsum'][0,i] = dsum
        comb['osum'][0,i] = osum

        # averages
        tr = rsum/wsum
        if tr > 100:
            # old style probably
            comb['r'][0,i] = rsum/npair
        else:
            comb['r'][0,i] = tr

        comb['dsig'][0,i] = dsum/wsum
        comb['osig'][0,i] = osum/wsum

        comb['dsigerr_simple'][0,i] = numpy.sqrt(1.0/wsum)

        # this is average wsum over lenses
        # we calculate clustering correction from this, wsum_mean/wsum_mean_random
        wsum_mean = wsum/nlens
        comb['wsum_mean'][0,i] = wsum_mean
        comb['wsum_mean_err_simple'][0,i] = sqrt(wsum2/nlens - wsum_mean**2)/sqrt(nlens)

    m,cov=jackknife.wjackknife(vsum=lout['dsum'], wsum=lout['wsum'])
    comb['dsigcov'][0,:,:] = cov
    comb['dsigcor'][0,:,:] = jackknife.covar2corr(cov)
    comb['dsigerr'][0,:] = sqrt(diag(cov))

    # turns out this agrees with the above one
    w=ones(lout['wsum'].shape)
    m,cov = jackknife.wjackknife(vsum=lout['wsum'], wsum=w)
    comb['wsum_mean_err'][0,:] = sqrt(diag(cov))

    return comb