def compute_kappa(annotations, extract=get_doc, value=lambda x:int(x)): value_set = set() for annotation in annotations: for _, v in extract(annotation, value): value_set.add(v) value_map = {v: i for i, v in enumerate(sorted(value_set))} d = defaultdict(list) for annotation in annotations: for k, v in extract(annotation, value): d[k].append(v) return _compute_kappa(d, value_map)
def compute_kappa(annotations, key=get_query, value=get_score): annotations = list(annotations) value_set = set() for a in annotations: value_set.add(value(a)) value_map = {v: i for i, v in enumerate(sorted(value_set))} d = defaultdict(list) for a in annotations: query = key(a) d[query].append(value(a)) return _compute_kappa(d, value_map)