def krippendorffs_alpha(annotations, metric_func=diagonal_distance,
nclasses=None):
"""Compute Krippendorff's alpha for multiple annotators.
**References:**
* Klaus Krippendorff (2004). "Content Analysis, an Introduction to Its
Methodology", 2nd Edition. Thousand Oaks, CA: Sage Publications.
In particular, Chapter 11, pages 219--250.
* `Wikipedia entry <http://en.wikipedia.org/wiki/Krippendorff%27s_Alpha>`_
See also:
:func:`~pyanno.measures.distances.diagonal_distance`,
:func:`~pyanno.measures.distances.binary_distance`,
Arguments
---------
annotations : ndarray, shape = (n_items, n_annotators)
Array of annotations for multiple annotators. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
weights_func : function(m_i, m_j)
Weights function that receives two matrices of indices
i, j and returns the matrix of weights between them.
Default is :func:`~pyanno.measures.distances.diagonal_distance`
nclasses : int
Number of annotation classes. If None, `nclasses` is inferred from the
values in the annotations
Returns
-------
stat : float
The value of the statistics
"""
if all_invalid(annotations):
logger.debug('No valid annotations')
return np.nan
if nclasses is None:
nclasses = compute_nclasses(annotations)
coincidences = coincidence_matrix(annotations, nclasses)
nc = coincidences.sum(1)
n = coincidences.sum()
# ---- coincidences expected by chance
chance_coincidences = np.empty((nclasses, nclasses), dtype=float)
for c in range(nclasses):
for k in range(nclasses):
if c == k:
chance_coincidences[c,k] = nc[c]*(nc[k]-1.) / (n-1.)
else:
chance_coincidences[c,k] = nc[c]*nc[k] / (n-1.)
# build weights matrix from weights function
weights = np.fromfunction(metric_func, shape=(nclasses, nclasses),
dtype=float) ** 2.
alpha = 1. - ((weights*coincidences).sum()
/ (weights*chance_coincidences).sum())
return alpha
def krippendorffs_alpha(annotations, metric_func=diagonal_distance, nclasses=None):
"""Compute Krippendorff's alpha for multiple annotators.
**References:**
* Klaus Krippendorff (2004). "Content Analysis, an Introduction to Its
Methodology", 2nd Edition. Thousand Oaks, CA: Sage Publications.
In particular, Chapter 11, pages 219--250.
* `Wikipedia entry <http://en.wikipedia.org/wiki/Krippendorff%27s_Alpha>`_
See also:
:func:`~pyanno.measures.distances.diagonal_distance`,
:func:`~pyanno.measures.distances.binary_distance`,
Arguments
---------
annotations : ndarray, shape = (n_items, n_annotators)
Array of annotations for multiple annotators. Missing values should be
indicated by :attr:`pyanno.util.MISSING_VALUE`
weights_func : function(m_i, m_j)
Weights function that receives two matrices of indices
i, j and returns the matrix of weights between them.
Default is :func:`~pyanno.measures.distances.diagonal_distance`
nclasses : int
Number of annotation classes. If None, `nclasses` is inferred from the
values in the annotations
Returns
-------
stat : float
The value of the statistics
"""
if all_invalid(annotations):
logger.debug("No valid annotations")
return np.nan
if nclasses is None:
nclasses = compute_nclasses(annotations)
coincidences = coincidence_matrix(annotations, nclasses)
nc = coincidences.sum(1)
n = coincidences.sum()
# ---- coincidences expected by chance
chance_coincidences = np.empty((nclasses, nclasses), dtype=float)
for c in range(nclasses):
for k in range(nclasses):
if c == k:
chance_coincidences[c, k] = nc[c] * (nc[k] - 1.0) / (n - 1.0)
else:
chance_coincidences[c, k] = nc[c] * nc[k] / (n - 1.0)
# build weights matrix from weights function
weights = np.fromfunction(metric_func, shape=(nclasses, nclasses), dtype=float) ** 2.0
alpha = 1.0 - ((weights * coincidences).sum() / (weights * chance_coincidences).sum())
return alpha
def test_coincidence_matrix(self):
# test example from
# http://en.wikipedia.org/wiki/Krippendorff%27s_Alpha#A_computational_example
kr = self.krippendorff_example
coincidence = pmh.coincidence_matrix(kr.annotations, kr.nclasses)
np.testing.assert_allclose(coincidence, kr.coincidence)
