def compute_agreement(df): """ Compute the agreement (Fleiss' Kappa) for each answer field :param df: Data frame grouped by HITId """ update_types = ["weakener", "strengthener"] props = [ "gibberish_understandable_grammatical", "relevant", "correct", "explains" ] for prop in props: data = [] task_id = 0 value_id = defaultdict(count().__next__) for update_type in update_types: for rat_idx in range(6): curr = df[[ f"Answer.{update_type}_rationale{rat_idx+1}_{prop}" ]] # [(annotator, task_id, ans)] data += [(str(worker_id + 1), str(task_id + idx), str(value_id[ans])) for idx, row in curr.iterrows() for worker_id, ans in enumerate(row[0])] task_id = len(data) curr_agreement = AnnotationTask(data=data) fleiss = curr_agreement.multi_kappa() print( f"Property: {prop}, Fleiss' Kappa: {fleiss:.3f} ({get_kappa_interpretation(fleiss)})" )
def main(): #print datadiff.diff_dict(readjson('E:\elan projects\L2\submissions\extracted\L2_100020027.eaf.379.json'),readjson('E:\elan projects\L2\submissions\extracted\L2_100020049.eaf.379.json')) ## s1=readjson(r'E:\elan projects\L2\submissions\extracted\L2_100020027.eaf.379.json') ## s2=readjson(r'E:\elan projects\L2\submissions\extracted\L2_100020049.eaf.379.json') ## #s2=readjson('E:\elan projects\L2\L2v1f_DIP.eaf.379.json') ## #s2=readjson('E:\elan projects\L2\L2v1_PRI.eaf.379.json') L1_dip = readjson(r'E:\elan projects\L1\L1v1_DIP.eaf.319.json') ## s1_data= create_task_data(s1,task_type='grouped',allowed='define',annotator='s1') ## s2_data=create_task_data(s2,task_type='grouped',allowed='define',annotator='s2') L1_dip = create_task_data(L1_dip, task_type='grouped', allowed='define', annotator='DIP') ## task=AnnotationTask(data=s1_data+s2_data,distance=nltk.metrics.distance.masi_distance_mod) l1_task = AnnotationTask(data=L1_dip, distance=nltk.metrics.distance.masi_distance_mod) # print "Observed Agreement:{0}".format( task.Ao('s1','s2')) ## print "Kappa:{0}".format(task.kappa()) ## print "Alpha:{0}".format(task.alpha()) ## print "Observed Avg Agreement over all:{0}".format(task.avg_Ao()) ## s1_labeldata=create_labels_list(s1_data) ## s2_labeldata=create_labels_list(s2_data) l1_labels = create_labels_list(L1_dip) assert len(l1_labels) == len(L1_dip) indivcounts = count_occurrances(str(l1_labels)) counts = count_labels(L1_dip) print L1_dip print counts print indivcounts
def calc_agreements(nr_of_abstracts=150): # Loop over the abstracts and calculate the kappa and alpha per abstract aggregate = [] for i in range(0, nr_of_abstracts): # try: annotators = round_robin(i) annotations_A = flatten(get_annotations(i, annotators[0])) annotations_B = flatten(get_annotations(i, annotators[1])) annotations = __str_combine_annotations(annotations_A, annotations_B) a = AnnotationTask(annotations, agreement_fn) aggregate.append({ "kappa" : a.kappa(), "alpha" : a.alpha(), "annotator_A" : annotators[0], "annotator_B" : annotators[1] }) # except: # print("Could not calculate kappa for abstract %i" % (i + 1)) # pass # Summary statistics kappa = describe([a['kappa'] for a in aggregate]) print("number of abstracts %i" % kappa[0]) print("[kappa] mean: " + str(kappa[2])) print("[kappa] variance: " + str(kappa[3])) alpha = describe([a['alpha'] for a in aggregate]) print("[alpha] mean: " + str(alpha[2])) print("[alpha] variance: " + str(alpha[3]))
def calculate_kappa(filename): # save labels label_list = [] with open('data/' + filename + '_data_result.json') as json_file: tweets = json.load(json_file) for row in tweets: label_list.append(row['label']) # Generate two fake labels to calculate kappa man_1_label = change_some_values(label_list) man_2_label = change_some_values(label_list) # save the labels to a csv file save_to_csv('data/label_1.csv', man_1_label) save_to_csv('data/label_2.csv', man_2_label) # calculate inter annotator agreement civ_1 = ['c1'] * len(man_1_label) civ_2 = ['c2'] * len(man_2_label) item_num_list = range(0, len(man_1_label)) civ_1 = zip(civ_1, item_num_list, man_1_label) civ_2 = zip(civ_2, item_num_list, man_2_label) task_data = civ_1 + civ_2 task = AnnotationTask(data=task_data) # observed disagreement for the weighted kappa coefficient print 'kappa: ' + str(task.kappa())
def alpha(self, ids=None, staff="upper", common_id=None, lib='nltk', label='bigram', distance=None): if ids is None: ids = [] if staff not in ('upper', 'lower'): raise Exception( "Alpha measure only applicable one staff at a time.") data = self._staff_annotation_data(ids=ids, staff=staff, lib=lib, label=label, common_id=common_id) if distance is None and label == "bigram": distance = DScore.bigram_label_distance if lib == 'nltk': if distance is None: distance = binary_distance annot_task = AnnotationTask(data=data, distance=distance) krip = annot_task.alpha() else: if distance is None: distance = 'nominal' krip = alpha(reliability_data=data, level_of_measurement=distance) return krip
def get_alpha_single_col(annos, col): if col == 'nonsense': distance = binary_distance else: distance = interval_distance data = [(row.WorkerId, '{participant_id}-{block}-{sent_idx}'.format(**row), row[col]) for idx, row in annos.iterrows() if not np.isnan(row[col])] return AnnotationTask(data, distance=distance).alpha()
def get_alpha_overall(annos): data = [] for idx, row in annos.iterrows(): for col in ['pos', 'neg', 'nonsense']: data.append( (row.WorkerId, '{participant_id}-{block}-{sent_idx}-{col}'.format( **row, col=col), row[col])) return AnnotationTask(data).alpha()
def compute_krippendorff(sce_path, output_path='', wo_attention_check=False, bad_annotators_path='', dataset=''): """ Compute Krippendorff's alpha with krippendorff library (https://github.com/pln-fing-udelar/fast-krippendorff/blob/master/sample.py) :param sce_path: csv file with columns UID, ANSWER, ANNOTATOR :param output_path: path of the output file where the results will be printed (if empty string the results are printed in the standart output) :param wo_attention_check: if True remove the attention check when computing alpha :param bad_annotators_path: path of the pkl file containing for each threshold the list of 'bad' annotators. For each threshold remove the annotations of the annotators listed when computing alpha. If empty string no annotator's annotation it removed. :param dataset: alphanumeric characters identifying the corpus to compute the alpha (if empty string the alpha is computed with annotation from all corpora and from attention check) """ if output_path: sys.stdout = open(output_path, "w") rows = read_csv(sce_path, dataset=dataset) bad_annotators_per_th = get_bad_annotators(bad_annotators_path) for th, bad_annotators in bad_annotators_per_th.items(): print(f'--- Threshold {th}---') annotations = get_annotations_per_annotators( rows, wo_attention_check=wo_attention_check, wo_annotator=bad_annotators) print('- After filtering: -') print_annotation_statistics(annotations) ratings_per_annotator = get_annotator_tab(annotations) data = [[np.nan if not r else int(r) for r in ratings] for ratings in ratings_per_annotator] print( "Krippendorff's alpha for nominal metric: ", krippendorff.alpha(reliability_data=data, level_of_measurement='nominal')) print("Krippendorff's alpha for interval metric: ", krippendorff.alpha(reliability_data=data)) print( "Krippendorff's alpha for ordinal metric: ", krippendorff.alpha(reliability_data=data, level_of_measurement='ordinal')) # with nltk library task_data = annotations2task_data(annotations) rating_task = AnnotationTask(data=task_data, distance=ordinal) print("Krippendorff's alpha for ordinal metric (nltk): ", rating_task.alpha())
def run_closed_class_jaccard_and_masi(cls, df: pd.DataFrame) -> Dict: iaa_by_column = { column: { "df": extract_iaa_df_by_column_name(df, column) } for column in cls.CLOSED_CLASS_COLUMNS } for column in iaa_by_column: task = AnnotationTask(distance=jaccard_distance) task.load_array( extract_records_for_nltk(iaa_by_column[column]['df'])) iaa_by_column[column]['alpha_jaccard'] = task.alpha() task = AnnotationTask(distance=masi_distance) task.load_array( extract_records_for_nltk(iaa_by_column[column]['df'])) iaa_by_column[column]['alpha_masi'] = task.alpha() return iaa_by_column
def nltk_with_kippendorff_data(): # needs data to be shaped in triples: (coder,item,label) input_eval_dp = "../data/krippendorff-evaluation-dataset.csv" eval_df = pd.read_table(input_eval_dp, delimiter=',', index_col=0) print("\ninput data:\n", eval_df.head()) # reshape rcsi data eval_nltk_df = pd.DataFrame() for index, row in eval_df.iterrows(): eval_nltk_df = eval_nltk_df.append( { 'coder': 'obs_1', 'item': index, 'label': row['obs1'] }, ignore_index=True) eval_nltk_df = eval_nltk_df.append( { 'coder': 'obs_2', 'item': index, 'label': row['obs2'] }, ignore_index=True) eval_nltk_df = eval_nltk_df.append( { 'coder': 'obs_3', 'item': index, 'label': row['obs3'] }, ignore_index=True) eval_nltk_df = eval_nltk_df.append( { 'coder': 'obs_4', 'item': index, 'label': row['obs4'] }, ignore_index=True) eval_nltk_df = eval_nltk_df.append( { 'coder': 'obs_5', 'item': index, 'label': row['obs5'] }, ignore_index=True) print("\nreshaped data:\n\n", eval_nltk_df.head()) print(eval_nltk_df.tail()) annotation_triples = eval_nltk_df.values.tolist() # print(annotation_triples) t = AnnotationTask(annotation_triples) print("\nKrippendorff alpha as per NLTK:\t", t.alpha(), "\n===========================================\n")
def calculate_iaa(data_dict): i = 0 data = [] for key, value in data_dict.items(): i += 1 data.append( ('Annotator1', i, frozenset((value['label1'], value['label1_2'])))) data.append( ('Annotator2', i, frozenset((value['label2'], value['label2_2'])))) print(data) t = AnnotationTask(data=data, distance=masi_distance) print(t.avg_Ao())
def test_easy2(self): ''' Same simple test with 1 rating removed. Removal of that rating should not matter: K-Apha ignores items with only 1 rating. ''' data = [('coder1', 'dress1', 'YES'), ('coder2', 'dress1', 'NO'), ('coder3', 'dress1', 'NO'), ('coder1', 'dress2', 'YES'), ('coder2', 'dress2', 'NO'), ] annotation_task = AnnotationTask(data) self.assertAlmostEqual(annotation_task.alpha(), -0.3333333)
def test_advanced(self): ''' More advanced test, based on http://www.agreestat.com/research_papers/onkrippendorffalpha.pdf ''' data = [ ('A', '1', '1'), ('B', '1', '1'), ('D', '1', '1'), ('A', '2', '2'), ('B', '2', '2'), ('C', '2', '3'), ('D', '2', '2'), ('A', '3', '3'), ('B', '3', '3'), ('C', '3', '3'), ('D', '3', '3'), ('A', '4', '3'), ('B', '4', '3'), ('C', '4', '3'), ('D', '4', '3'), ('A', '5', '2'), ('B', '5', '2'), ('C', '5', '2'), ('D', '5', '2'), ('A', '6', '1'), ('B', '6', '2'), ('C', '6', '3'), ('D', '6', '4'), ('A', '7', '4'), ('B', '7', '4'), ('C', '7', '4'), ('D', '7', '4'), ('A', '8', '1'), ('B', '8', '1'), ('C', '8', '2'), ('D', '8', '1'), ('A', '9', '2'), ('B', '9', '2'), ('C', '9', '2'), ('D', '9', '2'), ('B', '10', '5'), ('C', '10', '5'), ('D', '10', '5'), ('C', '11', '1'), ('D', '11', '1'), ('C', '12', '3'), ] annotation_task = AnnotationTask(data) self.assertAlmostEqual(annotation_task.alpha(), 0.743421052632)
def test_advanced2(self): """ Same more advanced example, but with 1 rating removed. Again, removal of that 1 rating should not matter. """ data = [ ("A", "1", "1"), ("B", "1", "1"), ("D", "1", "1"), ("A", "2", "2"), ("B", "2", "2"), ("C", "2", "3"), ("D", "2", "2"), ("A", "3", "3"), ("B", "3", "3"), ("C", "3", "3"), ("D", "3", "3"), ("A", "4", "3"), ("B", "4", "3"), ("C", "4", "3"), ("D", "4", "3"), ("A", "5", "2"), ("B", "5", "2"), ("C", "5", "2"), ("D", "5", "2"), ("A", "6", "1"), ("B", "6", "2"), ("C", "6", "3"), ("D", "6", "4"), ("A", "7", "4"), ("B", "7", "4"), ("C", "7", "4"), ("D", "7", "4"), ("A", "8", "1"), ("B", "8", "1"), ("C", "8", "2"), ("D", "8", "1"), ("A", "9", "2"), ("B", "9", "2"), ("C", "9", "2"), ("D", "9", "2"), ("B", "10", "5"), ("C", "10", "5"), ("D", "10", "5"), ("C", "11", "1"), ("D", "11", "1"), ("C", "12", "3"), ] annotation_task = AnnotationTask(data) self.assertAlmostEqual(annotation_task.alpha(), 0.743421052632)
def test_advanced(self): """ More advanced test, based on http://www.agreestat.com/research_papers/onkrippendorffalpha.pdf """ data = [ ("A", "1", "1"), ("B", "1", "1"), ("D", "1", "1"), ("A", "2", "2"), ("B", "2", "2"), ("C", "2", "3"), ("D", "2", "2"), ("A", "3", "3"), ("B", "3", "3"), ("C", "3", "3"), ("D", "3", "3"), ("A", "4", "3"), ("B", "4", "3"), ("C", "4", "3"), ("D", "4", "3"), ("A", "5", "2"), ("B", "5", "2"), ("C", "5", "2"), ("D", "5", "2"), ("A", "6", "1"), ("B", "6", "2"), ("C", "6", "3"), ("D", "6", "4"), ("A", "7", "4"), ("B", "7", "4"), ("C", "7", "4"), ("D", "7", "4"), ("A", "8", "1"), ("B", "8", "1"), ("C", "8", "2"), ("D", "8", "1"), ("A", "9", "2"), ("B", "9", "2"), ("C", "9", "2"), ("D", "9", "2"), ("B", "10", "5"), ("C", "10", "5"), ("D", "10", "5"), ("C", "11", "1"), ("D", "11", "1"), ("C", "12", "3"), ] annotation_task = AnnotationTask(data) self.assertAlmostEqual(annotation_task.alpha(), 0.743421052632)
def test_advanced2(self): ''' Same more advanced example, but with 1 rating removed. Again, removal of that 1 rating shoudl not matter. ''' data = [ ('A', '1', '1'), ('B', '1', '1'), ('D', '1', '1'), ('A', '2', '2'), ('B', '2', '2'), ('C', '2', '3'), ('D', '2', '2'), ('A', '3', '3'), ('B', '3', '3'), ('C', '3', '3'), ('D', '3', '3'), ('A', '4', '3'), ('B', '4', '3'), ('C', '4', '3'), ('D', '4', '3'), ('A', '5', '2'), ('B', '5', '2'), ('C', '5', '2'), ('D', '5', '2'), ('A', '6', '1'), ('B', '6', '2'), ('C', '6', '3'), ('D', '6', '4'), ('A', '7', '4'), ('B', '7', '4'), ('C', '7', '4'), ('D', '7', '4'), ('A', '8', '1'), ('B', '8', '1'), ('C', '8', '2'), ('D', '8', '1'), ('A', '9', '2'), ('B', '9', '2'), ('C', '9', '2'), ('D', '9', '2'), ('B', '10', '5'), ('C', '10', '5'), ('D', '10', '5'), ('C', '11', '1'), ('D', '11', '1'), ('C', '12', '3'), ] annotation_task = AnnotationTask(data) self.assertAlmostEqual(annotation_task.alpha(), 0.743421052632)
def test_easy(self): ''' Simple test, based on https://github.com/foolswood/krippendorffs_alpha/raw/master/krippendorff.pdf. ''' data = [('coder1', 'dress1', 'YES'), ('coder2', 'dress1', 'NO'), ('coder3', 'dress1', 'NO'), ('coder1', 'dress2', 'YES'), ('coder2', 'dress2', 'NO'), ('coder3', 'dress3', 'NO'), ] annotation_task = AnnotationTask(data) self.assertAlmostEqual(annotation_task.alpha(), -0.3333333)
def get_alpha(annotations, annotator, item_format, result, distance_type='interval'): if distance_type == 'binary': distance = binary_distance elif distance_type == 'interval': distance = interval_distance else: raise KeyError(f"Unknown type of distance: {distance_type}") data = [(row[annotator], item_format.format(**row), row[result]) for idx, row in annotations.iterrows() if not np.isnan(row[result])] return AnnotationTask(data, distance=distance).alpha()
def test_easy(self): """ Simple test, based on https://github.com/foolswood/krippendorffs_alpha/raw/master/krippendorff.pdf. """ data = [ ("coder1", "dress1", "YES"), ("coder2", "dress1", "NO"), ("coder3", "dress1", "NO"), ("coder1", "dress2", "YES"), ("coder2", "dress2", "NO"), ("coder3", "dress3", "NO"), ] annotation_task = AnnotationTask(data) self.assertAlmostEqual(annotation_task.alpha(), -0.3333333)
def test_easy2(self): """ Same simple test with 1 rating removed. Removal of that rating should not matter: K-Apha ignores items with only 1 rating. """ data = [ ("coder1", "dress1", "YES"), ("coder2", "dress1", "NO"), ("coder3", "dress1", "NO"), ("coder1", "dress2", "YES"), ("coder2", "dress2", "NO"), ] annotation_task = AnnotationTask(data) self.assertAlmostEqual(annotation_task.alpha(), -0.3333333)
def get_iaa(project, ac1_name: str, ac2_name: str, tag_filter: list = None, filter_both_ac: bool = False, level: str = 'tag', distance: str = 'binary'): """Computes Inter Annotator Agreement for 2 Annotation Collections. Args: project (CatmaProject): CatmaProject object ac1_name (str): AnnotationCollection name to be compared ac2_name (str): AnnotationCollection name to be compared with tag_filter (list, optional): Which Tags should be included. If None all are included. Default to None. level (str, optional): Whether the Annotation Tag or a specified Property should be compared. distance (str, optional): The IAA distance function. Either 'binary' or 'interval'. See https://www.nltk.org/api/nltk.metrics.html. Default to 'binary'. """ from nltk.metrics.agreement import AnnotationTask from nltk.metrics import interval_distance, binary_distance if distance == 'inteval': distance_function = interval_distance else: distance_function = binary_distance ac1 = project.ac_dict[ac1_name] ac2 = project.ac_dict[ac2_name] annotation_pairs = get_annotation_pairs(ac1, ac2, tag_filter=tag_filter, filter_both_ac=filter_both_ac) data = list(get_iaa_data(annotation_pairs, level=level)) annotation_task = AnnotationTask(data=data, distance=distance_function) print( textwrap.dedent(f""" Scott's pi: {annotation_task.pi()} Cohen's Kappa: {annotation_task.kappa()} Krippendorf Alpha: {annotation_task.alpha()} """)) return get_confusion_matrix(pair_list=annotation_pairs)
def alpha_krippendorff( answers: pd.DataFrame, distance: Callable[[Hashable, Hashable], float] = binary_distance ) -> float: """Inter-annotator agreement coefficient (Krippendorff 1980). Amount that annotators agreed on label assignments beyond what is expected by chance. The value of alpha should be interpreted as follows. alpha >= 0.8 indicates a reliable annotation, alpha >= 0.667 allows making tentative conclusions only, while the lower values suggest the unreliable annotation. Args: answers: A data frame containing `task`, `worker` and `label` columns. distance: Distance metric, that takes two arguments, and returns a value between 0.0 and 1.0 By default: binary_distance (0.0 for equal labels 1.0 otherwise). Returns: Float value. Examples: Consistent answers. >>> alpha_krippendorff(pd.DataFrame.from_records([ >>> {'task': 'X', 'worker': 'A', 'label': 'Yes'}, >>> {'task': 'X', 'worker': 'B', 'label': 'Yes'}, >>> {'task': 'Y', 'worker': 'A', 'label': 'No'}, >>> {'task': 'Y', 'worker': 'B', 'label': 'No'}, >>> ])) 1.0 Partially inconsistent answers. >>> alpha_krippendorff(pd.DataFrame.from_records([ >>> {'task': 'X', 'worker': 'A', 'label': 'Yes'}, >>> {'task': 'X', 'worker': 'B', 'label': 'Yes'}, >>> {'task': 'Y', 'worker': 'A', 'label': 'No'}, >>> {'task': 'Y', 'worker': 'B', 'label': 'No'}, >>> {'task': 'Z', 'worker': 'A', 'label': 'Yes'}, >>> {'task': 'Z', 'worker': 'B', 'label': 'No'}, >>> ])) 0.4444444444444444 """ _check_answers(answers) data: List[Tuple[Any, Hashable, Hashable]] = answers[['worker', 'task', 'label']].values.tolist() return AnnotationTask(data, distance).alpha()
def agree_tags(delta, column): """ egytokenes címkézési feladatokra számol egyetértést :param delta: az összevetett adat :param column: az az oszlop, amelyre egyetértést akarunk számolni :return: """ by_field = reverse_tags(delta, column) task = AnnotationTask(data=by_field) oa = task.avg_Ao() # observed agreement s = task.S() # Bennett, Albert and Goldstein S (1954) all categories are equally likely pi = task.pi() # Scott pi (1955) single distribution kappa = task.kappa() # Cohen kappa (1960) individual coder distribution w_kappa = task.weighted_kappa() alpha = task.alpha() # Krippendorff alpha (1980) return oa, s, pi, kappa, w_kappa, alpha
def compute_agreement(sce_path, nb_turns_per_hit, nb_annotators=None, wo_attention_check=False): # Compute Kappa coefficient and Krippendorff's alpha with nltk library (https://www.nltk.org/api/nltk.metrics.html) rows = read_csv(sce_path) if nb_annotators: annotations = get_annotations(rows, nb_turns_per_hit, wo_attention_check) else: annotations = get_annotations_per_annotators(rows, wo_attention_check) print_annotation_statistics(annotations) task_data = annotations2task_data(annotations, nb_annotators) # print(task_data) rating_task = AnnotationTask(data=task_data, distance=ordinal) print(f"Cohen's Kappa: {rating_task.kappa()}") print(f"Fleiss' Kappa: {rating_task.multi_kappa()}") print(f"Krippendorff's alpha with ordial metric: {rating_task.alpha()}")
def kappa(self): """Data is a list of list. Each element is a list : [annotator, element, label] """ if not KAPPA: return 'Not installed' #if self.last is None: return # must be specific to a feature data = [] nb = 1 for elem in self.ano: u1 = elem[1] u2 = elem[2] if u1 is None or u2 is None: continue else: data.append([self.ano1.get_code(), nb, u1.get(self.last)]) data.append([self.ano2.get_code(), nb, u2.get(self.last)]) nb += 1 task = AnnotationTask(data) return task.kappa()
def calculate_round_kappa(round_estimates=[]): from nltk.metrics.agreement import AnnotationTask # Calculating the distance between two different estimate categories, and return the difference ratio def distance_cal(v1, v2): # all estimate categories: 1 hour, half a day, one day, half a week, one week, two weeks, # and more than two weeks (-1) labels = ['1.0', '4.0', '8.0', '20.0', '40.0', '80.0', '-1.0'] i1 = labels.index(v1) i2 = labels.index(v2) return abs(i1 - i2) / 6 # prepare estimate for the annotation task data, i = [], 1 for estimate in round_estimates: data.append(["c" + str(i), 1, str(estimate)]) i += 1 task = AnnotationTask(data=data, distance=distance_cal) agreement_level = task.multi_kappa() return agreement_level
def getagreement(tpl,datadir,task_type='all'): """Get agreement values for annotators in the :data:'tpl' list Args: tpl (list): combination group of annotators datadir (str): Cache data directory used by joblib Returns: namedtuple defined as ``Agree = collections.namedtuple('Agree', ['kappa', 'alpha','avg_ao'], verbose=True)`` """ mem = Memory(cachedir=datadir) readjson=mem.cache(json2taskdata.readjson,mmap_mode='r') create_task_data= mem.cache(json2taskdata.create_task_data) count_occurrances=mem.cache(json2taskdata.count_occurrances) count_labels=mem.cache(json2taskdata.count_labels) annotators=set() lectask=[] #------------------------------------------------------------------------------- # for each annotator in group tpl #------------------------------------------------------------------------------- for stditem in tpl: aname=stditem.split('.')[0][3:][-2:] annotators.add(aname) lecdict=readjson(stditem) newlectask= create_task_data(lecdict,task_type=task_type,annotator=aname) label_data=json2taskdata.create_labels_list(newlectask) abscount=count_occurrances(str(label_data)) yaml.dump(abscount,open(os.path.join( datadir,'abscount-'+aname+'.yaml'),'w')) setcount=count_labels(newlectask) yaml.dump(setcount,open(os.path.join( datadir,'setcount-'+aname+'.yaml'),'w')) lectask=lectask+newlectask task=AnnotationTask(data=lectask,distance=nltk.metrics.distance.masi_distance_mod) return {frozenset(annotators): Agree(task.kappa(),task.alpha(),task.avg_Ao())}
data = [] sentiment_r1_5_scale = [] sentiment_r2_5_scale = [] for r1, r2 in zip(sentences_r1, sentences_r2): sentiment_r1_5_scale.append(int(r1[5])) data.append((6, r1[0], r1[5])) sentiment_r2_5_scale.append(int(r2[5])) data.append((7, r2[0], r2[5])) if (r1[0] != r2[0]): print r1[0] except Exception, e: print e # disconnect from server db.close() print i print skll.kappa(sentiment_r1_5_scale, sentiment_r2_5_scale) annotation = AnnotationTask(data=data) print annotation.kappa() print annotation.alpha()
data = pd.read_csv('../input_data/labels-C.csv', sep=';', index_col=0) allcoders = data.columns experts = ['KEY', 'MG', 'MS', 'TM'] novices = ['KEY', 'CK', 'GK', 'RM'] cols = novices # Total values taskdata = [] for coder in cols: for i in data[coder].index: taskdata.append([coder, i, data[coder][i]]) ratingtask = AnnotationTask(data=taskdata) print("kappa " + str(ratingtask.kappa())) print("fleiss " + str(ratingtask.multi_kappa())) print("alpha " + str(ratingtask.alpha())) print("scotts " + str(ratingtask.pi())) # Pairwise values similarities = [] for coders in itertools.product(cols, repeat=2): if coders[0] == coders[1]: similarities.append(1) else: taskdata = [] for coder in coders: for i in data[coder].index: taskdata.append([coder, i, data[coder][i]])
def __init__(self, S): this_dir = os.path.dirname(os.path.realpath(__file__)) dir1 = os.path.join(this_dir, S, "G1") dir2 = os.path.join(this_dir, S, "G2") self.annotation_task = AnnotationTask(data=self.__readfile(dir1, dir2))