def chance_agreement_same_frequency(annotations1, annotations2, nclasses):
"""Expected frequency of agreement by random annotations.
Assumes that the annotators draw random annotations with the same
frequency as the combined observed annotations.
Arguments
---------
annotations1 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
annotations2 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
weights_func : function(m_i, m_j)
Weights function that receives two matrices of indices
i, j and returns the matrix of weights between them.
Default is :func:`~pyanno.measures.distances.diagonal_distance`
Return
------
result : float
Chance agreement value
"""
count1 = labels_count(annotations1, nclasses)
count2 = labels_count(annotations2, nclasses)
count_total = count1 + count2
total = count_total.sum()
chance_agreement = (count_total / total)**2.
return chance_agreement
def chance_agreement_same_frequency(annotations1, annotations2, nclasses):
"""Expected frequency of agreement by random annotations.
Assumes that the annotators draw random annotations with the same
frequency as the combined observed annotations.
Arguments
---------
annotations1 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
annotations2 : ndarray, shape = (n_items, )
Array of annotations for a single annotator. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
weights_func : function(m_i, m_j)
Weights function that receives two matrices of indices
i, j and returns the matrix of weights between them.
Default is :func:`~pyanno.measures.distances.diagonal_distance`
Return
------
result : float
Chance agreement value
"""
count1 = labels_count(annotations1, nclasses)
count2 = labels_count(annotations2, nclasses)
count_total = count1 + count2
total = count_total.sum()
chance_agreement = (count_total / total) ** 2.0
return chance_agreement
def test_labels_count(self):
annotations = np.array([[1, 2, -2, -2], [-2, -2, 3, 3], [-2, 1, 3, 1],
[-2, -2, -2, -2]])
nclasses = 5
expected = np.array([0., 3., 1., 3., 0.])
result = pu.labels_count(annotations, nclasses, missing_val=-2)
np.testing.assert_equal(result, expected)
def test_labels_count(self):
annotations = np.array(
[
[ 1, 2, -2, -2],
[-2, -2, 3, 3],
[-2, 1, 3, 1],
[-2, -2, -2, -2]
]
)
nclasses = 5
expected = np.array([0., 3., 1., 3., 0.])
result = pu.labels_count(annotations, nclasses, missing_val=-2)
np.testing.assert_equal(result, expected)