def compute_kappa(annotations, extract=get_doc, value=lambda x:int(x)):
value_set = set()
for annotation in annotations:
for _, v in extract(annotation, value):
value_set.add(v)
value_map = {v: i for i, v in enumerate(sorted(value_set))}
d = defaultdict(list)
for annotation in annotations:
for k, v in extract(annotation, value):
d[k].append(v)
return _compute_kappa(d, value_map)
def compute_kappa(annotations, key=get_query, value=get_score):
annotations = list(annotations)
value_set = set()
for a in annotations:
value_set.add(value(a))
value_map = {v: i for i, v in enumerate(sorted(value_set))}
d = defaultdict(list)
for a in annotations:
query = key(a)
d[query].append(value(a))
return _compute_kappa(d, value_map)
def compute_kappa(annotations, key=get_query, value=get_score):
annotations = list(annotations)
value_set = set()
for a in annotations:
value_set.add(value(a))
value_map = {v: i for i, v in enumerate(sorted(value_set))}
d = defaultdict(list)
for a in annotations:
query = key(a)
d[query].append(value(a))
return _compute_kappa(d, value_map)
