def test_chance_agreement_same_frequency(self): distr = np.array([0.1, 0.5, 0.4]) anno1 = pyanno.util.random_categorical(distr, nsamples=10000) anno2 = pyanno.util.random_categorical(distr, nsamples=10000) expected = distr ** 2. freqs = pmh.chance_agreement_same_frequency(anno1, anno2, len(distr)) np.testing.assert_allclose(freqs, expected, atol=1e-2, rtol=0.)
def scotts_pi(annotations1, annotations2, nclasses=None): """Return Scott's pi statistic for two annotators. Assumes that the annotators draw random annotations with the same frequency as the combined observed annotations. See also :func:`~pyanno.measures.helpers.pairwise_matrix`. **References:** * Scott, W. (1955). "Reliability of content analysis: The case of nominal scale coding." Public Opinion Quarterly, 19(3), 321-325. * `Wikipedia entry <http://en.wikipedia.org/wiki/Scott%27s_Pi>`_ Arguments --------- annotations1 : ndarray, shape = (n_items, ) Array of annotations for a single annotator. Missing values should be indicated by :attr:`pyanno.util.MISSING_VALUE` annotations2 : ndarray, shape = (n_items, ) Array of annotations for a single annotator. Missing values should be indicated by :attr:`pyanno.util.MISSING_VALUE` nclasses : int Number of annotation classes. If None, `nclasses` is inferred from the values in the annotations Returns ------- stat : float The value of the statistics """ if all_invalid(annotations1, annotations2): logger.debug('No valid annotations') return np.nan if nclasses is None: nclasses = compute_nclasses(annotations1, annotations2) chance_agreement = chance_agreement_same_frequency(annotations1, annotations2, nclasses) observed_agreement = observed_agreement_frequency(annotations1, annotations2, nclasses) return chance_adjusted_agreement(observed_agreement.sum(), chance_agreement.sum())
def scotts_pi(annotations1, annotations2, nclasses=None): """Return Scott's pi statistic for two annotators. Assumes that the annotators draw random annotations with the same frequency as the combined observed annotations. See also :func:`~pyanno.measures.helpers.pairwise_matrix`. **References:** * Scott, W. (1955). "Reliability of content analysis: The case of nominal scale coding." Public Opinion Quarterly, 19(3), 321-325. * `Wikipedia entry <http://en.wikipedia.org/wiki/Scott%27s_Pi>`_ Arguments --------- annotations1 : ndarray, shape = (n_items, ) Array of annotations for a single annotator. Missing values should be indicated by :attr:`pyanno.util.MISSING_VALUE` annotations2 : ndarray, shape = (n_items, ) Array of annotations for a single annotator. Missing values should be indicated by :attr:`pyanno.util.MISSING_VALUE` nclasses : int Number of annotation classes. If None, `nclasses` is inferred from the values in the annotations Returns ------- stat : float The value of the statistics """ if all_invalid(annotations1, annotations2): logger.debug("No valid annotations") return np.nan if nclasses is None: nclasses = compute_nclasses(annotations1, annotations2) chance_agreement = chance_agreement_same_frequency(annotations1, annotations2, nclasses) observed_agreement = observed_agreement_frequency(annotations1, annotations2, nclasses) return chance_adjusted_agreement(observed_agreement.sum(), chance_agreement.sum())