Esempio n. 1
0
    def test_md_pattern_random(self):
        pattern_ids, PD = md_pattern(self.rdata)
        self.assertTrue(pattern_ids.size == self.rdata.shape[0])

        for i, d in PD.items():
            rows = pattern_ids == i
            where_present = np.arange(self.rdata.shape[1])[~d["pattern"]]
            where_not_present = np.arange(self.rdata.shape[1])[d["pattern"]]
            self.assertTrue([i not in where_not_present for i in where_present])
            self.assertTrue(np.isfinite(self.rdata[np.ix_(rows, where_present)]).all())
            self.assertTrue(
                (~np.isfinite(self.rdata[np.ix_(rows, where_not_present)])).all()
            )
Esempio n. 2
0
 def test_default(self):
     pattern_ids, PD = md_pattern(self.static)
     self.assertTrue(pattern_ids.size == self.static.shape[0])
     self.assertTrue(len(PD.keys()) == 4)
     self.assertTrue(np.allclose([0, 1, 2, 3, 1], pattern_ids))
     self.assertTrue([i in PD for i in range(4)])
Esempio n. 3
0
def EMCOMP(
    X,
    threshold=None,
    tol=0.0001,
    convergence_metric=lambda A, B, t: np.linalg.norm(np.abs(A - B)) < t,
    max_iter=30,
):
    r"""
    EMCOMP replaces rounded zeros in a compositional data set based on a set of
    thresholds. After Palarea-Albaladejo and Martín-Fernández (2008) [#ref_1]_.


    Parameters
    ----------
    X  : :class:`numpy.ndarray`
        Dataset with rounded zeros
    threshold : :class:`numpy.ndarray`
        Array of threshold values for each component as a proprotion.
    tol : :class:`float`
        Tolerance to check for convergence.
    convergence_metric : :class:`callable`
        Callable function to check for convergence. Here we use a compositional distance
        rather than a maximum absolute difference, with very similar performance.
        Function needs to accept two :class:`numpy.ndarray` arguments and third
        tolerance argument.
    max_iter : :class:`int`
        Maximum number of iterations before an error is thrown.

    Returns
    --------
    X_est : :class:`numpy.ndarray`
        Dataset with rounded zeros replaced.
    prop_zeros : :class:`float`
       Proportion of zeros in the original data set.
    n_iters : :class:`int`
        Number of iterations needed for convergence.

    Notes
    -----

        * At least one component without missing values is needed for the divisor.
          Rounded zeros/missing values are replaced by values below their respective
          detection limits.

        * This routine is not completely numerically stable as written.

    Todo
    -------
        * Implement methods to deal with variable decection limits (i.e thresholds are array shape :code:`(N, D)`)
        * Conisder non-normal models for data distributions.
        * Improve numerical stability to reduce the chance of :code:`np.inf` appearing.

    References
    ----------
    .. [#ref_1] Palarea-Albaladejo J. and Martín-Fernández J. A. (2008)
            A modified EM ALR-algorithm for replacing rounded zeros in compositional data sets.
            Computers & Geosciences 34, 902–917.
            doi: `10.1016/j.cageo.2007.09.015 <https://dx.doi.org/10.1016/j.cageo.2007.09.015>`__

    """
    X = X.copy()
    n_obs, D = X.shape
    X = close(X, sumf=np.nansum)
    # ---------------------------------
    # Convert zeros into missing values
    # ---------------------------------
    X = np.where(np.isclose(X, 0.0), np.nan, X)
    # Use a divisor free of missing values
    assert np.isfinite(X).all(axis=0).any()
    pos = np.argmax(np.isfinite(X).all(axis=0))
    Yden = X[:, pos]
    # --------------------------------------
    # Compute the matrix of censure points Ψ
    # --------------------------------------
    # need an equivalent concept for ilr
    cpoints = (
        np.ones((n_obs, 1)) @ np.log(threshold[np.newaxis, :])
        - np.log(Yden[:, np.newaxis]) @ np.ones((1, D))
        - np.spacing(1.0)  # Machine epsilon
    )
    assert np.isfinite(cpoints).all()
    cpoints = cpoints[:, [i for i in range(D) if not i == pos]]  # censure points
    prop_zeroes = np.count_nonzero(~np.isfinite(X)) / (n_obs * D)
    Y = ALR(X, pos)
    # ---------------Log Space--------------------------------
    LD = Y.shape[1]
    M = np.nanmean(Y, axis=0)  # μ0
    C = nancov(Y)  # Σ0
    assert np.isfinite(M).all() and np.isfinite(C).all()

    # --------------------------------------------------
    # Stage 2: Find and enumerate missing data patterns
    # --------------------------------------------------
    pID, pD = md_pattern(Y)
    # -------------------------------------------
    # Stage 3: Regression against other variables
    # -------------------------------------------
    logger.debug(
        "Starting Iterative Regression for Matrix : ({}, {})".format(n_obs, LD)
    )
    another_iter = True
    niters = 0
    while another_iter:
        niters += 1
        Mnew, Cnew = M.copy(), C.copy()
        Ystar = Y.copy()
        V = np.zeros((LD, LD))

        for p_no in np.unique(pID):
            logger.debug("Pattern ID: {}, {}".format(p_no, pD[p_no]["pattern"]))
            rows = np.arange(pID.size)[pID == p_no]  # rows with this pattern
            varobs, varmiss = (
                np.arange(D - 1)[~pD[p_no]["pattern"]],
                np.arange(D - 1)[pD[p_no]["pattern"]],
            )
            sigmas = np.zeros((LD))
            assert np.isfinite(Y[np.ix_(rows, varobs)]).all()
            assert (~np.isfinite(Y[np.ix_(rows, varmiss)])).all()
            if varobs.size and varmiss.size:  # Non-completely missing, but missing some
                logger.debug(
                    "Regressing {} rows for pattern {} | {}.".format(
                        rows.size,
                        "".join(varobs.astype(str)),
                        "".join(varmiss.astype(str)),
                    )
                )
                B, σ2_res = _reg_sweep(Mnew, Cnew, varobs)
                assert B.shape == (varobs.size + 1, varmiss.size)
                assert σ2_res.shape == (varmiss.size, varmiss.size)
                assert np.isfinite(B).all()
                logger.debug(
                    "Current Estimator (1, {})".format(
                        ", ".join(["β{}".format(i) for i in range(B.shape[0] - 1)])
                    )
                )

                Ystar[np.ix_(rows, varmiss)] = np.ones((rows.size, 1)) * B[0, :] + (
                    (Y[np.ix_(rows, varobs)] @ B[1 : (varobs.size + 1), :])
                )
                sigmas[varmiss] = np.sqrt(np.diag(σ2_res))
                assert np.isfinite(sigmas[varmiss]).all()

                x = (  # position of threshold values relative to estimated means
                    cpoints[np.ix_(rows, varmiss)] - Ystar[np.ix_(rows, varmiss)]
                )
                x /= sigmas[varmiss][np.newaxis, :]  # as standard deviations
                assert np.isfinite(x).all()
                # ----------------------------------------------------
                # Calculate inverse Mills Ratio for Heckman correction
                # ----------------------------------------------------
                ϕ = stats.norm.pdf(x, loc=0, scale=1)  # pdf
                Φ = stats.norm.cdf(x, loc=0, scale=1)  # cdf
                Φ[np.isclose(Φ, 0)] = np.finfo(np.float64).eps * 2
                assert (Φ > 0).all()  # if its not, infinity will be introduced
                inversemills = ϕ / Φ
                Ystar[np.ix_(rows, varmiss)] = (
                    Ystar[np.ix_(rows, varmiss)] - sigmas[varmiss] * inversemills
                )
                V[np.ix_(varmiss, varmiss)] += σ2_res * rows.size
        assert np.isfinite(V).all()
        # -----------------------------------------------
        # Update and store parameter vector (μ(t), Σ(t)).
        # -----------------------------------------------
        logger.debug("Regression finished.")
        M = np.nanmean(Ystar, axis=0)
        Ydevs = Ystar - np.ones((n_obs, 1)) * M
        Ydevs[~np.isfinite(Ydevs)] = 0.0  # remove nonfinite components
        PC = np.dot(Ydevs.T, Ydevs)
        logger.debug("Correlation:\n{}".format(PC / (n_obs - 1)))
        C = (PC + V) / (n_obs - 1)

        logger.debug("Average diff: {}".format(np.mean(Ydevs, axis=0)))
        assert np.isfinite(C).all()
        # --------------------
        # Convergence checking
        # --------------------
        if convergence_metric(M, Mnew, tol) & convergence_metric(C, Cnew, tol):
            another_iter = False
            logger.debug("Convergence achieved.")

        another_iter = another_iter & (niters < max_iter)
        logger.debug("Iterations Continuing: {}".format(another_iter))
    # ----------------------------
    # Back to compositional space
    # ---------------------------
    logger.debug("Finished. Inverting to compositional space.")
    Xstar = inverse_ALR(Ystar, pos)
    return Xstar, prop_zeroes, niters