def test_chance_agreement_different_frequency(self):
distr1 = np.array([0.1, 0.5, 0.4])
distr2 = np.array([0.6, 0.2, 0.2])
anno1 = pyanno.util.random_categorical(distr1, nsamples=10000)
anno2 = pyanno.util.random_categorical(distr2, nsamples=10000)
expected = distr1 * distr2
freqs = pmh.chance_agreement_different_frequency(anno1, anno2,
len(distr1))
np.testing.assert_allclose(freqs, expected, atol=1e-2, rtol=0.)
def cohens_kappa(annotations1, annotations2, nclasses=None):
"""Compute Cohen's kappa for two annotators.
Assumes that the annotators draw annotations at random with different but
constant frequencies.
See also :func:`~pyanno.measures.helpers.pairwise_matrix`.
**References:**
* Cohen, Jacob (1960). A coefficient of agreement for nominal scales.
Educational and Psychological Measurement, 20, 37--46.
* `Wikipedia entry <http://en.wikipedia.org/wiki/Cohen%27s_kappa>`_
Arguments
---------
annotations1 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
annotations2 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
nclasses : int
Number of annotation classes. If None, `nclasses` is inferred from the
values in the annotations
Returns
-------
stat : float
The value of the statistics
"""
if all_invalid(annotations1, annotations2):
logger.debug('No valid annotations')
return np.nan
if nclasses is None:
nclasses = compute_nclasses(annotations1, annotations2)
chance_agreement = chance_agreement_different_frequency(annotations1,
annotations2,
nclasses)
observed_agreement = observed_agreement_frequency(annotations1,
annotations2,
nclasses)
return chance_adjusted_agreement(observed_agreement.sum(),
chance_agreement.sum())
def cohens_kappa(annotations1, annotations2, nclasses=None):
"""Compute Cohen's kappa for two annotators.
Assumes that the annotators draw annotations at random with different but
constant frequencies.
See also :func:`~pyanno.measures.helpers.pairwise_matrix`.
**References:**
* Cohen, Jacob (1960). A coefficient of agreement for nominal scales.
Educational and Psychological Measurement, 20, 37--46.
* `Wikipedia entry <http://en.wikipedia.org/wiki/Cohen%27s_kappa>`_
Arguments
---------
annotations1 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
annotations2 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
nclasses : int
Number of annotation classes. If None, `nclasses` is inferred from the
values in the annotations
Returns
-------
stat : float
The value of the statistics
"""
if all_invalid(annotations1, annotations2):
logger.debug("No valid annotations")
return np.nan
if nclasses is None:
nclasses = compute_nclasses(annotations1, annotations2)
chance_agreement = chance_agreement_different_frequency(annotations1, annotations2, nclasses)
observed_agreement = observed_agreement_frequency(annotations1, annotations2, nclasses)
return chance_adjusted_agreement(observed_agreement.sum(), chance_agreement.sum())
