Esempio n. 1
0
    def score_samples(self, X, y=None):
        """Compute log-likelihood (or log(det(Jacobian))) for each sample.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples is the number of samples and n_features
            is the number of features.

        y : None, default=None
            Not used but kept for compatibility.

        Returns
        -------
        log_likelihood : array, shape (n_samples,)
            Log likelihood of each data point in X.

        """
        self._check_is_fitted()
        components = self._get_component_densities()
        component_log_likelihood = np.array([
            comp.score_samples(X) for comp in components
        ])  # (n_components, n_samples)
        weighted_log_likelihood = np.log(self.weights_).reshape(
            -1, 1) + component_log_likelihood
        return scipy_logsumexp(weighted_log_likelihood, axis=0)
Esempio n. 2
0
def calc_mean_binary_cross_entropy_from_scores(ytrue_N, scores_N):
    ''' Compute average cross entropy for given binary classifier's predictions

    Consumes "scores", real values between (-np.inf, np.inf),
    and then conceptually produces probabilities by feeding into sigmoid,
    and then into the BCE function to get the result.

    In practice, we compute BCE directly from scores using an implementation
    that avoids the numerical issues of the sigmoid (saturation, if underflows
    to zero would become negative infinity). This clever implementation uses
    the "logsumexp" trick.

    Computing BCE uses *base-2* logarithms, so the resulting number is a valid
    upper bound of the zero-one loss (aka error rate) when we threshold at 0.5.

    Args
    ----
    ytrue_N : 1D array, shape (n_examples,) = (N,)
        All values must be either 0 or 1. Will be cast to int dtype.
        Each entry represents the binary 'true' label of one example
        One entry per example in current dataset
    scores_N : 1D array, shape (n_examples,) = (N,)
        One entry per example in current dataset.
        Each entry is a real value, could be between -infinity and +infinity.
        Large negative values indicate strong probability of label y=0.
        Zero values indicate probability of 0.5.
        Large positive values indicate strong probability of label y=1.
        Needs to be same size as ytrue_N.

    Returns
    -------
    bce : float
        Binary cross entropy, averaged over all N provided examples

    Examples
    --------
    >>> N = 8
    >>> ytrue_N = np.asarray([0., 0., 0., 0., 1., 1., 1., 1.])

    # Try some good scores for these 8 points.
    >>> good_scores_N = np.asarray([-4., -3., -2., -1., 1, 2., 3., 4.])
    >>> good_bce = calc_mean_binary_cross_entropy_from_scores(
    ...     ytrue_N, good_scores_N)
    >>> print("%.6f" % (good_bce))
    0.182835
    
    # Try some near-perfect scores for these 8 points.
    >>> perfect_scores_N = np.asarray([-9., -9., -9,  -9., 9., 9., 9., 9.])
    >>> perfect_bce = calc_mean_binary_cross_entropy_from_scores(
    ...     ytrue_N, perfect_scores_N)
    >>> print("%.6f" % (perfect_bce))
    0.000178

    # Check same computation with "probas". Should match exactly.
    >>> perfect_bce2 = calc_mean_binary_cross_entropy_from_probas(
    ...     ytrue_N, sigmoid(perfect_scores_N))
    >>> print("%.6f" % (perfect_bce2))
    0.000178

    # Try some extreme scores (should get worse BCE with more extreme scores)
    >>> ans = calc_mean_binary_cross_entropy_from_scores([0], [+99.])
    >>> print("%.6f" % (ans))
    142.826809
    >>> ans = calc_mean_binary_cross_entropy_from_scores([0], [+999.])
    >>> print("%.6f" % (ans))
    1441.252346
    >>> ans = calc_mean_binary_cross_entropy_from_scores([0], [+9999.])
    >>> print("%.6f" % (ans))
    14425.507714

    # Try with "probas": sigmoid saturates! using scores is *better*
    >>> a = calc_mean_binary_cross_entropy_from_probas([0], sigmoid([+999.]))
    >>> print("%.6f" % (a))
    46.508147
    >>> a = calc_mean_binary_cross_entropy_from_probas([0], sigmoid([+9999.]))
    >>> print("%.6f" % (a))
    46.508147

    # Try empty
    >>> empty_bce = calc_mean_binary_cross_entropy_from_scores([], [])
    >>> np.allclose(0.0, empty_bce)
    True
    '''
    # Cast labels to integer just to be sure we're getting what's expected
    ytrue_N = np.asarray(ytrue_N, dtype=np.int32)
    N = int(ytrue_N.size)
    if N == 0:
        return 0.0

    # Convert binary y values so 0 becomes +1 and 1 becomes -1
    # See HW2 instructions on website for the math
    yflippedsign_N = -1 * np.sign(ytrue_N - 0.001)

    # Cast logit scores to float
    scores_N = np.asarray(scores_N, dtype=np.float64)

    # Use provided math to obtain numerically stable expression
    flipped_scores_N = yflippedsign_N * scores_N
    scores_and_zeros_N2 = np.hstack(
        [flipped_scores_N.reshape((N, 1)),
         np.zeros((N, 1))])
    bce_N = scipy_logsumexp(scores_and_zeros_N2, axis=1)
    return np.mean(bce_N) / np.log(2.0)
Esempio n. 3
0
def test_logsumexp():
    x = np.arange(5)
    assert np.equal(logsumexp(x), scipy_logsumexp(x))
Esempio n. 4
0
def test_logsumexp(values, axis, keepdims):
    npt.assert_almost_equal(
        logsumexp(values, axis=axis, keepdims=keepdims).eval(),
        scipy_logsumexp(values, axis=axis, keepdims=keepdims),
    )
Esempio n. 5
0
def logsumexp(arr, axis=None):
    return scipy_logsumexp(arr, axis)
Esempio n. 6
0
    def conditional_densities(self, X, cond_idx, not_cond_idx):
        """Compute conditional densities.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Data to condition on based on `cond_idx`.

        cond_idx : array-like of int
            Indices to condition on.

        not_cond_idx :
            Indices not to condition on.

        Returns
        -------
        conditional_densities : array-like of estimators
            Either a single density if all the same or a list of Gaussian
            densities with conditional variances and means.

        """
        if len(cond_idx) + len(not_cond_idx) != X.shape[1]:
            raise ValueError(
                '`cond_idx_arr` and `not_cond_idx_arr` should be complements that '
                'have the a total of X.shape[1] number of values.')
        # Handle trivial case
        if len(cond_idx) == 0:
            return self
        self._check_is_fitted()

        # Retrieve components
        components = self._get_component_densities()  # (n_components,)

        n_samples, n_features = X.shape

        # Get new weights
        X_cond = X[:, cond_idx]
        marginal_log_likelihood = np.array([
            component.marginal_density(cond_idx).score_samples(X_cond)
            for component in components
        ]).transpose()  # (n_samples, n_components)

        # Numerically safe way to get new weights if marginal_log_likelihood is very low e.g.
        # -1000, which would mean np.exp(-1000) = 0
        log_cond_weights = np.log(self.weights_) + marginal_log_likelihood
        log_cond_weights -= scipy_logsumexp(log_cond_weights,
                                            axis=1).reshape(-1, 1)
        cond_weights = np.exp(log_cond_weights)
        cond_weights = np.maximum(1e-100,
                                  cond_weights)  # Just to avoid complete zeros

        # Condition each Gaussian
        cond_components = np.array([
            self._process_conditionals(
                component.conditional_densities(X, cond_idx, not_cond_idx),
                n_samples) for component in components
        ]).transpose()  # (n_samples, n_components)

        # Create final conditional densities
        conditional_densities = np.array([
            self._set_fit_params(clone(self), cond_w, cond_c_arr)
            for cond_w, cond_c_arr in zip(cond_weights, cond_components)
        ])
        return conditional_densities
Esempio n. 7
0
import time
import numpy as np
import matplotlib.pyplot as plt

from fitr.utils import logsumexp as fitr_logsumexp
from scipy.special import logsumexp as scipy_logsumexp

n = 10000
X = np.empty((n, 2))
for i in range(n):
    x = np.ones(i + 2)
    ft0 = time.time()
    yf = fitr_logsumexp(x)
    ft = time.time() - ft0

    fs0 = time.time()
    ys = scipy_logsumexp(x)
    fs = time.time() - fs0

    X[i] = np.array([fs, ft])

fig, ax = plt.subplots(figsize=(7, 3))
ax.set_title('Log-sum-exp Speed')
ax.set_xlabel(r'Dimensionality of $\mathbf{x}$')
ax.set_ylabel(r'Time (s)')
ax.semilogy(np.arange(n) + 2, X[:, 1], c='k', ls='-', label='fitr')
ax.semilogy(np.arange(n) + 2, X[:, 0], c='k', ls='--', label='SciPy')
plt.show()