def run_closed_class_jaccard_and_masi(cls, df: pd.DataFrame) -> Dict: iaa_by_column = { column: { "df": extract_iaa_df_by_column_name(df, column) } for column in cls.CLOSED_CLASS_COLUMNS } for column in iaa_by_column: task = AnnotationTask(distance=jaccard_distance) task.load_array( extract_records_for_nltk(iaa_by_column[column]['df'])) iaa_by_column[column]['alpha_jaccard'] = task.alpha() task = AnnotationTask(distance=masi_distance) task.load_array( extract_records_for_nltk(iaa_by_column[column]['df'])) iaa_by_column[column]['alpha_masi'] = task.alpha() return iaa_by_column
def read_labels(annotator, lisst): annotator = "annotator" + str(annotator) for i, annot in enumerate(lisst): #,annot_2_bin): # print("A", annot) i = str(i) myset = (annotator, i, frozenset(annot)) tryme.append(myset) #print("trme", tryme) read_labels("1", annot_1) #print("2", tryme) read_labels("2", annot_2) #print("3", tryme) #task_data = [('coder1','Item0',frozenset(['l1','l2'])), #('coder2','Item0',frozenset(['l1'])), #('coder1','Item1',frozenset(['l1','l2'])), #('coder2','Item1',frozenset(['l1','l2'])), #('coder1','Item2',frozenset(['l1'])), #('coder2','Item2',frozenset(['l1']))] #print(krippendorff.alpha(reliability_data=data)) #toy_data = [('1', 5723, (1)),('2', 5723, (2))] #task = AnnotationTask(data=[x.split() for x in open(os.path.join(os.path.dirname(__file__), "artstein_poesio_example.txt"))]) task = AnnotationTask(distance=masi_distance) task.load_array(tryme) print(task.alpha())