def test_observed_agreement(self):
anno1 = np.array([0, 0, 1, 1, MV, 3])
anno2 = np.array([0, MV, 1, 1, MV, 2])
nvalid = np.sum(is_valid(anno1) & is_valid(anno2))
expected = np.array([1., 2., 0., 0.]) / nvalid
freqs = pmh.observed_agreement_frequency(anno1, anno2, 4)
np.testing.assert_array_equal(freqs, expected)
def cohens_kappa(annotations1, annotations2, nclasses=None):
"""Compute Cohen's kappa for two annotators.
Assumes that the annotators draw annotations at random with different but
constant frequencies.
See also :func:`~pyanno.measures.helpers.pairwise_matrix`.
**References:**
* Cohen, Jacob (1960). A coefficient of agreement for nominal scales.
Educational and Psychological Measurement, 20, 37--46.
* `Wikipedia entry <http://en.wikipedia.org/wiki/Cohen%27s_kappa>`_
Arguments
---------
annotations1 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
annotations2 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
nclasses : int
Number of annotation classes. If None, `nclasses` is inferred from the
values in the annotations
Returns
-------
stat : float
The value of the statistics
"""
if all_invalid(annotations1, annotations2):
logger.debug('No valid annotations')
return np.nan
if nclasses is None:
nclasses = compute_nclasses(annotations1, annotations2)
chance_agreement = chance_agreement_different_frequency(annotations1,
annotations2,
nclasses)
observed_agreement = observed_agreement_frequency(annotations1,
annotations2,
nclasses)
return chance_adjusted_agreement(observed_agreement.sum(),
chance_agreement.sum())
def scotts_pi(annotations1, annotations2, nclasses=None):
"""Return Scott's pi statistic for two annotators.
Assumes that the annotators draw random annotations with the same
frequency as the combined observed annotations.
See also :func:`~pyanno.measures.helpers.pairwise_matrix`.
**References:**
* Scott, W. (1955). "Reliability of content analysis: The case of nominal
scale coding." Public Opinion Quarterly, 19(3), 321-325.
* `Wikipedia entry <http://en.wikipedia.org/wiki/Scott%27s_Pi>`_
Arguments
---------
annotations1 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
annotations2 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
nclasses : int
Number of annotation classes. If None, `nclasses` is inferred from the
values in the annotations
Returns
-------
stat : float
The value of the statistics
"""
if all_invalid(annotations1, annotations2):
logger.debug('No valid annotations')
return np.nan
if nclasses is None:
nclasses = compute_nclasses(annotations1, annotations2)
chance_agreement = chance_agreement_same_frequency(annotations1,
annotations2,
nclasses)
observed_agreement = observed_agreement_frequency(annotations1,
annotations2,
nclasses)
return chance_adjusted_agreement(observed_agreement.sum(),
chance_agreement.sum())
def cohens_kappa(annotations1, annotations2, nclasses=None):
"""Compute Cohen's kappa for two annotators.
Assumes that the annotators draw annotations at random with different but
constant frequencies.
See also :func:`~pyanno.measures.helpers.pairwise_matrix`.
**References:**
* Cohen, Jacob (1960). A coefficient of agreement for nominal scales.
Educational and Psychological Measurement, 20, 37--46.
* `Wikipedia entry <http://en.wikipedia.org/wiki/Cohen%27s_kappa>`_
Arguments
---------
annotations1 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
annotations2 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
nclasses : int
Number of annotation classes. If None, `nclasses` is inferred from the
values in the annotations
Returns
-------
stat : float
The value of the statistics
"""
if all_invalid(annotations1, annotations2):
logger.debug("No valid annotations")
return np.nan
if nclasses is None:
nclasses = compute_nclasses(annotations1, annotations2)
chance_agreement = chance_agreement_different_frequency(annotations1, annotations2, nclasses)
observed_agreement = observed_agreement_frequency(annotations1, annotations2, nclasses)
return chance_adjusted_agreement(observed_agreement.sum(), chance_agreement.sum())
def scotts_pi(annotations1, annotations2, nclasses=None):
"""Return Scott's pi statistic for two annotators.
Assumes that the annotators draw random annotations with the same
frequency as the combined observed annotations.
See also :func:`~pyanno.measures.helpers.pairwise_matrix`.
**References:**
* Scott, W. (1955). "Reliability of content analysis: The case of nominal
scale coding." Public Opinion Quarterly, 19(3), 321-325.
* `Wikipedia entry <http://en.wikipedia.org/wiki/Scott%27s_Pi>`_
Arguments
---------
annotations1 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
annotations2 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
nclasses : int
Number of annotation classes. If None, `nclasses` is inferred from the
values in the annotations
Returns
-------
stat : float
The value of the statistics
"""
if all_invalid(annotations1, annotations2):
logger.debug("No valid annotations")
return np.nan
if nclasses is None:
nclasses = compute_nclasses(annotations1, annotations2)
chance_agreement = chance_agreement_same_frequency(annotations1, annotations2, nclasses)
observed_agreement = observed_agreement_frequency(annotations1, annotations2, nclasses)
return chance_adjusted_agreement(observed_agreement.sum(), chance_agreement.sum())