def test_chance_agreement_same_frequency(self):
        distr = np.array([0.1, 0.5, 0.4])
        anno1 = pyanno.util.random_categorical(distr, nsamples=10000)
        anno2 = pyanno.util.random_categorical(distr, nsamples=10000)

        expected = distr ** 2.
        freqs = pmh.chance_agreement_same_frequency(anno1, anno2, len(distr))

        np.testing.assert_allclose(freqs, expected, atol=1e-2, rtol=0.)
Exemple #2
0
def scotts_pi(annotations1, annotations2, nclasses=None):
    """Return Scott's pi statistic for two annotators.

    Assumes that the annotators draw random annotations with the same
    frequency as the combined observed annotations.

    See also :func:`~pyanno.measures.helpers.pairwise_matrix`.

    **References:**

    * Scott, W. (1955). "Reliability of content analysis: The case of nominal
      scale coding." Public Opinion Quarterly, 19(3), 321-325.

    * `Wikipedia entry <http://en.wikipedia.org/wiki/Scott%27s_Pi>`_

    Arguments
    ---------
    annotations1 : ndarray, shape = (n_items, )
        Array of annotations for a single annotator. Missing values should be
        indicated by :attr:`pyanno.util.MISSING_VALUE`

    annotations2 : ndarray, shape = (n_items, )
        Array of annotations for a single annotator. Missing values should be
        indicated by :attr:`pyanno.util.MISSING_VALUE`

    nclasses : int
        Number of annotation classes. If None, `nclasses` is inferred from the
        values in the annotations

    Returns
    -------
    stat : float
        The value of the statistics
    """

    if all_invalid(annotations1, annotations2):
        logger.debug('No valid annotations')
        return np.nan

    if nclasses is None:
        nclasses = compute_nclasses(annotations1, annotations2)

    chance_agreement = chance_agreement_same_frequency(annotations1,
                                                       annotations2,
                                                       nclasses)

    observed_agreement = observed_agreement_frequency(annotations1,
                                                      annotations2,
                                                      nclasses)

    return chance_adjusted_agreement(observed_agreement.sum(),
                                      chance_agreement.sum())
Exemple #3
0
def scotts_pi(annotations1, annotations2, nclasses=None):
    """Return Scott's pi statistic for two annotators.

    Assumes that the annotators draw random annotations with the same
    frequency as the combined observed annotations.

    See also :func:`~pyanno.measures.helpers.pairwise_matrix`.

    **References:**

    * Scott, W. (1955). "Reliability of content analysis: The case of nominal
      scale coding." Public Opinion Quarterly, 19(3), 321-325.

    * `Wikipedia entry <http://en.wikipedia.org/wiki/Scott%27s_Pi>`_

    Arguments
    ---------
    annotations1 : ndarray, shape = (n_items, )
        Array of annotations for a single annotator. Missing values should be
        indicated by :attr:`pyanno.util.MISSING_VALUE`

    annotations2 : ndarray, shape = (n_items, )
        Array of annotations for a single annotator. Missing values should be
        indicated by :attr:`pyanno.util.MISSING_VALUE`

    nclasses : int
        Number of annotation classes. If None, `nclasses` is inferred from the
        values in the annotations

    Returns
    -------
    stat : float
        The value of the statistics
    """

    if all_invalid(annotations1, annotations2):
        logger.debug("No valid annotations")
        return np.nan

    if nclasses is None:
        nclasses = compute_nclasses(annotations1, annotations2)

    chance_agreement = chance_agreement_same_frequency(annotations1, annotations2, nclasses)

    observed_agreement = observed_agreement_frequency(annotations1, annotations2, nclasses)

    return chance_adjusted_agreement(observed_agreement.sum(), chance_agreement.sum())