def print_krippendorff_per_leaning(job, raters_normal): all_agreement_scores = [ ] # contains agreement obj with job, leaning and value for leaning in [-1, 0, 1]: leaning_taskdata = [] article_id_current_leaning = [ article.article_id for article in articles if int(article.leaning) == leaning ] for i, rater in enumerate(raters_normal): name = 'rater' + str(i) my_triplets = [] for j, article_rating in enumerate(rater): item = 'item_' + str(j) + '_' if article_rating['id'] in article_id_current_leaning: my_article_triplets = [ (name, item + str(k), rating) for k, rating in enumerate(article_rating[job]) ] my_triplets += my_article_triplets leaning_taskdata += my_triplets ratingtask = agreement.AnnotationTask(data=leaning_taskdata, distance=interval_distance) ratingtask.K = categories[job] all_agreement_scores.append({leaning: ratingtask.alpha()}) print('Krippendorff ' + job + ':') print(all_agreement_scores) return all_agreement_scores
def print_krippendorff_per_job(raters, category): all_agreement_scores = [] # contains agreement obj with job and value for job in jobs: job_taskdata = [] for i, rater in enumerate(raters): name = 'rater' + str(i) my_triplets = [] for j, article_rating in enumerate(rater): item = 'item_' + str(j) + '_' my_article_triplets = [ (name, item + str(k), rating) for k, rating in enumerate(article_rating[job]) ] my_triplets += my_article_triplets job_taskdata += my_triplets ratingtask = agreement.AnnotationTask(data=job_taskdata, distance=interval_distance) ratingtask.K = category[job] all_agreement_scores.append({job: ratingtask.alpha()}) print('Krippendorff:') print(all_agreement_scores) return all_agreement_scores
def overall_agreement(combined_df,coders,frame_types,distance_metric=masi_distance): results = [] for frame_type in frame_types: all_labels = format_labels(combined_df,coders,frame_type) task = agreement.AnnotationTask(data=all_labels,distance=distance_metric) results.append((frame_type,task.alpha())) return pd.DataFrame(results,columns=['Frame Type','Alpha'])
def overall_agreement(df_list, frame_types, distance_metric=masi_distance, convert_mixed=True): results = [] for frame in frame_types: if frame == 'Narrative' and convert_mixed == True: new_df_list = convert_narrative_mixed_to_both(df_list) all_labels = format_labels(new_df_list, frame) else: all_labels = format_labels(df_list, frame) task = agreement.AnnotationTask(data=all_labels, distance=distance_metric) results.append((frame, task.alpha())) return pd.DataFrame(results, columns=['Frame Type', 'Alpha'])
def get_article_agreement_helper(my_data, my_cat, article_id): ratingtask = agreement.AnnotationTask(data=my_data, distance=interval_distance) ratingtask.K = my_cat article_agreement = {} article_agreement['id'] = article_id try: article_agreement['krippendorff'] = ratingtask.alpha() except ZeroDivisionError: article_agreement['krippendorff'] = 1.0 return article_agreement
def toy_cohens_kappa(): # rater1 = [1, 1, 1, 0] # rater2 = [1, 1, 0, 0] # rater3 = [0, 1, 1] rater1 = ['s', 's', 's', 'g', 'u'] rater2 = ['s', 's', 'g', 'g', 's'] taskdata = [[0, str(i), str(rater1[i])] for i in range(0, len(rater1)) ] + [[1, str(i), str(rater2[i])] for i in range(0, len(rater2))] # + [ # [2, str(i), str(rater3[i])] for i in range(0, len(rater3))] print(taskdata) ratingtask = agreement.AnnotationTask(data=taskdata) print("kappa " + str(ratingtask.kappa())) print("fleiss " + str(ratingtask.multi_kappa())) print("alpha " + str(ratingtask.alpha())) print("scotts " + str(ratingtask.pi())) print("sklearn kappa " + str(cohen_kappa_score(rater1, rater2)))
def vectors_to_annotation_task(*args, drop: List[str] = []): """transform vectors of labels into a nltk AnnotationTask object. :param args: vector of labels for each annotator; add one argument per annotator. :type args: 1d np.array() of labels :param drop: list of labels that should be ignored :type drop: List[str] :return: the AnnotationTask object :rtype: nltk.metrics.agreement.AnnotationTask """ from nltk.metrics import agreement v = np.vstack(args) it = np.nditer(v, flags=["multi_index"]) if len(drop): data = [(it.multi_index[0], it.multi_index[1], str(x)) for x in it if str(x) not in drop] else: data = [(it.multi_index[0], it.multi_index[1], str(x)) for x in it] return agreement.AnnotationTask(data=data)
def cohens_kappa(): data_folder = '/Users/fpena/UCC/Thesis/datasets/context/manuallyLabeledReviews/' business_type = Constants.ITEM_TYPE file_name = data_folder + '%s_%s_reviews.json' labelers = [ # 'francisco', 'diego', 'mesut', 'rohit', ] all_records = [ load_data(file_name % (labeler, business_type)) for labeler in labelers ] rater1 = [record['review_type'] for record in all_records[0]] rater2 = [record['review_type'] for record in all_records[1]] rater3 = [record['review_type'] for record in all_records[2]] taskdata = [[0, str(i), str(rater1[i])] for i in range(0, len(rater1))] + [ [1, str(i), str(rater2[i])] for i in range(0, len(rater2)) ] + [[2, str(i), str(rater3[i])] for i in range(0, len(rater3))] print(taskdata) ratingtask = agreement.AnnotationTask(data=taskdata) print("Observed agreement " + str(ratingtask.avg_Ao())) print("kappa " + str(ratingtask.kappa())) print("fleiss " + str(ratingtask.multi_kappa())) print("alpha " + str(ratingtask.alpha())) print("scotts " + str(ratingtask.pi())) print("sklearn kappa " + str(cohen_kappa_score(rater1, rater2))) print("sklearn kappa " + str(cohen_kappa_score(rater1, rater3))) print("sklearn kappa " + str(cohen_kappa_score(rater2, rater3)))
#data5.append(("a4", idx, row["frank5"])) #data5.append(("a5", idx, row["pum5"])) #label 6 data6 = [] for idx, row in df_final.iterrows(): data6.append(("a1", idx, row["palang6"])) #data6.append(("a2", idx, row["pum6"])) data6.append(("a3", idx, row["vicky6"])) #data6.append(("a4", idx, row["frank6"])) #data6.append(("a5", idx, row["gill6"])) #calculate agreement label_1 = agreement.AnnotationTask(data=data1) label_2 = agreement.AnnotationTask(data=data2) label_3 = agreement.AnnotationTask(data=data3) label_4 = agreement.AnnotationTask(data=data4) label_5 = agreement.AnnotationTask(data=data5) label_6 = agreement.AnnotationTask(data=data6) #print print("Cohen's Kappa Label 1:", label_1.kappa()) print("Cohen's Kappa Label 2:", label_2.kappa()) print("Cohen's Kappa Label 3:", label_3.kappa()) print("Cohen's Kappa Label 4:", label_4.kappa()) print("Cohen's Kappa Label 5:", label_5.kappa()) print("Cohen's Kappa Label 6:", label_6.kappa()) print("\n") print("Fleiss's Kappa Label 1:", label_1.multi_kappa())
print( "Cohens kappa for ukrainian comments between annotator labeler1 and labeler2: {:.3f}" .format(cohens_ua)) print( "Cohens kappa for russian comments between annotator labeler3 and labeler4: {:.3f}" .format(cohens_ru)) matthews_corrcoef_score_ua = matthews_corrcoef(labeler1, labeler2) matthews_corrcoef_score_ru = matthews_corrcoef(labeler3, labeler4) print( "Matthews corrcoef score for ukrainian comments between annotator labeler1 and labeler2: {:.3f}" .format(matthews_corrcoef_score_ua)) print( "Matthews corrcoef score for russian comments between annotator labeler3 and labeler4: {:.3f}" .format(matthews_corrcoef_score_ru)) from nltk.metrics import agreement # Reformat the data into the form AnnotationTask # expects. labels_ru = labels_ru.dropna(subset=['Type of HS']) data = [] for idx, row in labels_ru.iterrows(): data.append(("a1", idx, row["Type of HS"])) data.append(("a2", idx, row["Type of 1 char"])) atask = agreement.AnnotationTask(data=data) print("Cohen's Kappa:", atask.kappa()) print("Fleiss's Kappa:", atask.multi_kappa())