예제 #1
0
def test_fleiss_kappa_irr():
    fleiss = Holder()
    #> r = kappam.fleiss(diagnoses)
    #> cat_items(r, pref="fleiss.")
    fleiss.method = "Fleiss' Kappa for m Raters"
    fleiss.irr_name = 'Kappa'
    fleiss.value = 0.4302445
    fleiss.stat_name = 'z'
    fleiss.statistic = 17.65183
    fleiss.p_value = 0
    data_ = aggregate_raters(diagnoses)[0]
    res1_kappa = fleiss_kappa(data_)
    assert_almost_equal(res1_kappa, fleiss.value, decimal=7)
예제 #2
0
def test_fleiss_kappa_irr():
    fleiss = Holder()
    #> r = kappam.fleiss(diagnoses)
    #> cat_items(r, pref="fleiss.")
    fleiss.method = "Fleiss' Kappa for m Raters"
    fleiss.irr_name = 'Kappa'
    fleiss.value = 0.4302445
    fleiss.stat_name = 'z'
    fleiss.statistic = 17.65183
    fleiss.p_value = 0
    data_ = aggregate_raters(diagnoses)[0]
    res1_kappa = fleiss_kappa(data_)
    assert_almost_equal(res1_kappa, fleiss.value, decimal=7)
예제 #3
0
파일: utils.py 프로젝트: ned2/tator
def get_iaa(question_num,
            queries=None,
            users=None,
            collection=None,
            level='fine'):
    data = get_annotations(question_num,
                           queries=queries,
                           users=users,
                           level=level,
                           collection=collection)
    #n_cat = Annotation.get_num_categories(question_num)
    results = aggregate_raters(data, n_cat=None)
    kappa = fleiss_kappa(results[0])
    return kappa
예제 #4
0
def test_aggregate_raters():
    data = diagnoses
    resf = aggregate_raters(data)
    colsum = np.array([26, 26, 30, 55, 43])
    assert_equal(resf[0].sum(0), colsum)
예제 #5
0
def test_aggregate_raters():
    data = diagnoses
    resf = aggregate_raters(data)
    colsum = np.array([26, 26, 30, 55, 43])
    assert_equal(resf[0].sum(0), colsum)
예제 #6
0
def get_kappa_marking(data, c, result_file, results):
    """ Fleiss' Kappa for marking task - Task 1.1 """

    data = data.loc[data['AssignmentStatus'] !=
                    'Rejected']  #Taking only the raters that were approved
    titles = data['Input.threadtitle'].unique()  #List of threads in batch
    aggregate_dataframe = pd.DataFrame()
    fks = []
    if 'Answer.noreply' not in data.columns:
        data['Answer.noreply'] = 0
    for title in titles:
        df = data.loc[data['Input.threadtitle'] == title]
        marked_posts = [
            col for col in df.columns
            if re.search(r'Answer\.\d$', col) or 'Answer.noreply' in col
        ]
        df = df.loc[:, marked_posts].fillna(0)
        df = df.replace('unclear', 99)
        df = df.replace('none', 99)
        length = 0

        ### THREAD NAMES WITH QUOTES DOESN'T WORK IN SQLITE

        # c.execute('select thread_id from post2 inner join thread on post2.thread_id= thread.id \
        #    where original=1 and post2.courseid=? and thread.title=?',('"%%'+course+'%%"','"%%'+title+'%%"'))

        ############################  Get the total number of posts+comments in that thread  ############################
        try:
            c.execute(
                'select thread_id from post2 inner join thread on post2.thread_id= thread.id \
                where original=1 and post2.courseid like ' + '"%%' + course +
                '%%"' + ' and thread.title like \
                ' + '"%%' + title + '%%"')

            thread_id = c.fetchone()

            c.execute('select count(1) from post2 where thread_id like ' +
                      '"%%' + str(thread_id[0]) + '%%"' + ' and \
                courseid like ' + '"%%' + course + '%%"')

            post2 = c.fetchone()
            c.execute('select count(1) from comment2 where thread_id like ' +
                      '"%%' + str(thread_id[0]) + '%%"' + ' \
                and courseid like ' + '"%%' + course + '%%"')
            comment2 = c.fetchone()
            length = post2[0] + comment2[0]

        except:
            continue

        #################################################################################################################

        #####################  Calculating Fleiss Kappa using the fleiss_kappa class above  #############################

        print(df)
        df1 = pd.DataFrame()
        print('length of thread: ', title, length)
        for i in range(1, length + 1):
            try:
                df1['Answer.' + str(i)] = df['Answer.' + str(i)]
            except Exception as e:
                #print('error raised', e)
                #raise
                pass
        df1['Answer.noreply'] = df['Answer.noreply']
        ## df1 is a dataframe with dimensions (raters X posts). aggregate_raters (below) converts that to
        ## (posts X categories) with input as counts

        aggregate = aggregate_raters(df1.T)
        #print(aggregate[0])

        fk = fleiss_kappa(aggregate[0])
        fks.append(fk.calc_fleiss_kappa())

        print(title + " -- " + str(fk.calc_fleiss_kappa()))
        result_file.write(str(fk.calc_fleiss_kappa()) + "\n")

    #################################################################################################################
    results = np.append(results, np.mean(fks))
    #print("\nAverage Kappa:" + str(np.mean(fks)))
    print("Std Dev:" + str(np.std(fks)))
    return results
예제 #7
0
def get_kappa_categorization(data, c, result_file, results):
    """ Fleiss Kappa for categorisation tasks - Task 2.1 and Task 2.2 """

    data = data.loc[data['AssignmentStatus'] != 'Rejected']
    titles = data['Input.threadtitle'].unique()
    aggregate_dataframe = pd.DataFrame()
    fks = []
    ##########################  Mapping the categories to numbers for input into kappa  ####################################

    cat_to_num_2 = {  # Categories for Task 2
        "summary": 1,
        "justify": 2,
        "feedback": 3,
        "clarification": 4,
        "extension": 5,
        "juxtaposition": 6,
        "refinement": 7,
        "critique": 8,
        "agreement": 9,
        "disagreement": 10,
        "answer": 11,
        "appreciation": 12,
        "completion": 13,
        "paraphrase": 14,
        "other": 15
    }

    ########################################################################################################################

    if 'Answer.noreply' not in data.columns:  #Add Answer.noreply if it does not exist in the dataframe
        data['Answer.noreply'] = ""

    marked_posts = [col for col in data.columns if 'Answer.' in col]
    #print(marked_posts)
    for post in marked_posts:
        data[post] = data[post].map(cat_to_num_2).fillna(0).astype(
            int)  # Substituting the categories to
        #numbers above for input into kappa

    # The next block of code is same as above

    for title in titles:

        df = data.loc[data['Input.threadtitle'] == title]
        marked_posts = [col for col in df.columns if 'Answer.' in col]
        df = df.loc[:, marked_posts].fillna(0)

        ############################  Get the total number of posts+comments in that thread  ############################
        try:
            c.execute(
                'select thread_id from post2 inner join thread on post2.thread_id= thread.id \
                where original=1 and post2.courseid like ' + '"%%' + course +
                '%%"' + ' and thread.title like \
                ' + '"%%' + title + '%%"')
            thread_id = c.fetchone()

            c.execute('select count(1) from post2 where thread_id like ' +
                      '"%%' + str(thread_id[0]) + '%%"' + ' \
                and courseid like ' + '"%%' + course + '%%"')

            post2 = c.fetchone()
            c.execute('select count(1) from comment2 where thread_id like ' +
                      '"%%' + str(thread_id[0]) + '%%"' + ' \
                and courseid like ' + '"%%' + course + '%%"')
            comment2 = c.fetchone()
            length = post2[0] + comment2[0]

        except:
            continue

        #################################################################################################################

        #####################  Calculating Fleiss Kappa using the fleiss_kappa class above  #############################

        df1 = pd.DataFrame()
        for i in range(length):
            try:
                df1['Answer.' + str(i + 1)] = df['Answer.' + str(i + 1) +
                                                 '_discourse_type']
            except:
                pass
        df1['Answer.noreply'] = df['Answer.noreply']

        ## df1 is a dataframe with dimensions (raters X posts). aggregate_raters (below) converts that to (posts X categories)
        ## with input as counts

        aggregate = aggregate_raters(df1.T)
        fk = fleiss_kappa(aggregate[0])
        fks.append(fk.calc_fleiss_kappa())
        result_file.write(str(fk.calc_fleiss_kappa()) + "\n")
        #print(title+" -- "+str(fk.calc_fleiss_kappa()))

        #################################################################################################################

    #print("\nAverage Kappa:"+str(np.mean(fks)))
    results = np.append(results, np.mean(fks))
    return results
예제 #8
0
def get_kappa_marking(df, c, result_file, results):

    """ Lenient Fleiss' Kappa for marking task - Task 1.1 """

    df = df.loc[df['AssignmentStatus'] != 'Rejected'] #Taking only the mturk assignments that were accepted
    threads = df['Input.threadtitle'].unique()  #Getting a list of unique threads in the list
    fks = [] #List of kappas in each thread in the given batch
    if 'Answer.noreply' not in df.columns:
        df['Answer.noreply'] = ""

    for thread in threads:
        
        ############################  Get the total number of posts+comments in that thread  ############################
        
        try:    
            c.execute('select thread_id from post2 inner join thread on post2.thread_id= thread.id where \
                original=1 and post2.courseid like '+'"%%'+course+'%%"'+' and thread.title like '+'"%%'+ \
                thread+'%%"')
            thread_id = c.fetchone()
            
            c.execute('select count(1) from post2 where thread_id like '+'"%%'+str(thread_id[0])+'%%"'+ ' and \
                courseid like '+'"%%'+course+'%%"' )

            post2 = c.fetchone()
            c.execute('select count(1) from comment2 where thread_id like '+'"%%'+str(thread_id[0])+'%%"'+ ' and \
                courseid like '+'"%%'+course+'%%"' )
            comment2 = c.fetchone()
            length = post2[0]+comment2[0]
        
        except:
            continue     

        #################################################################################################################

        ###############################  Selecting post(s) with maximum markings  ########################################

        filter_col = [col for col in df if col.startswith('Answer')]
        counts = (df.loc[df['Input.threadtitle'] == thread, \
            filter_col].count(axis = 0))

        counts_sorted = (counts.sort_values(ascending = False))

        post_max_agreement = np.argwhere(counts == np.max(counts)).flatten().tolist()

        post_max_agreement = ((counts.iloc[post_max_agreement].index.values))
        #print(post_max_agreement)
        df = df.replace('unclear',99)
        df = df.replace('none',99)

        ##################################################################################################################
        
        #####################  Calculating Fleiss Kappa using the fleiss_kappa class above  #############################
        
        #df1 = df.loc[df['Input.threadtitle'] == thread]
        df1 = pd.DataFrame()
      
        
        for i in range(length):
            if 'Answer.'+ str(i+1) in df.columns:
                    df1['Answer.'+ str(i+1)]  = 0
            else:
                pass
        df1['Answer.noreply'] = 0
        df1[post_max_agreement] = df.loc[df['Input.threadtitle'] == thread, post_max_agreement]  
        
        

        #print(df1.fillna(0).astype(int))

        aggregate = aggregate_raters(df1.fillna(0).astype(int).T)
        fk = fleiss_kappa(aggregate[0])
        #print(aggregate)
        fks.append(fk.calc_fleiss_kappa())
        #print(thread+ "("+str(length)+")"+" -- "+str(fk.calc_fleiss_kappa())+"\n")
        result_file.write(str(fk.calc_fleiss_kappa())+"\n")

        #################################################################################################################

    print("\nAverage Kappa:"+str(np.mean(fks)))
    results = np.append(results,np.mean(fks))
    return results
예제 #9
0
def get_kappa_categorization(df, c, result_file, results):

    """ Fleiss Kappa for categorisation tasks - Task 2.1 and Task 2.2 """

    df = df.loc[df['AssignmentStatus'] != 'Rejected']
    threads = df['Input.threadtitle'].unique()
    #print(len(threads))
    entries = []
    fks = []
    ##########################  Mapping the categories to numbers for input into kappa  ####################################

    cat_to_num_2 = { # Categories for Task 2.1
                      "resolves":1,
                      "elaborates":2,
                      "requests":3,
                      "social":4,
                      "none":5,
                      #Categories for Task 2.2

                      #elaborates
                      "clarifies":1,
                      "extension":2,
                      "juxtaposition":3,
                      "refinement":4,
                      "critique":5,
                      
                      #resolves
                       "agreement":1,
                      "disagreement":2,
                      "generic":3,
                      "appreciation":4,
                      "completion":5,

                      "none":6,
                      "nota":6,

                }
                
    ########################################################################################################################             

    if 'Answer.noreply' not in df.columns: # Add Answer.noreply if it does not exist in the dataframe
        df['Answer.noreply'] = ""
    marked_posts = [col for col in df.columns if 'Answer.' in col]

    for post in marked_posts:
        df[post] = df[post].map(cat_to_num_2) # Substituting the categories to numbers
    
    for thread in threads:
        
            ############################  Get the total number of posts+comments in that thread  ############################
        
        try: 
            c.execute('select thread_id from post2 inner join thread on post2.thread_id= thread.id where \
                original=1 and post2.courseid like '+'"%%'+course+'%%"'+' and thread.title like '+'"%%'+thread+'%%"')
            thread_id = c.fetchone()
            #print(thread_id)
            c.execute('select count(1) from post2 where thread_id like'+'"%%'+str(thread_id[0])+'%%"'+ ' and \
                courseid like '+'"%%'+course+'%%"' )
            post2 = c.fetchone()
            c.execute('select count(1) from comment2 where thread_id like '+'"%%'+str(thread_id[0])+'%%"'+ ' and \
                courseid like '+'"%%'+course+'%%"' )
            comment2 = c.fetchone()
            length = post2[0]+comment2[0]
        
        except:
            continue    
        
        #################################################################################################################

        ###############################  Selecting post(s) with maximum markings  #######################################

        counts = (df.loc[df['Input.threadtitle'] == thread, \
            marked_posts].count(axis = 0))

        counts_sorted = (counts.sort_values(ascending = False))

        post_max_agreement = np.argwhere(counts == np.max(counts)).flatten().tolist()
        post_max_agreement = ((counts.iloc[post_max_agreement].index.values))

        #################################################################################################################

        #####################  Calculating Fleiss Kappa using the fleiss_kappa class above  #############################
        #print(post_max_agreement)
        df1 = pd.DataFrame()
        for i in range(length) :
            if 'Answer.'+ str(i+1)+'_discourse_type' in df.columns:
                    df1['Answer.'+ str(i+1)+'_discourse_type']  = 0
            else:
                pass
        df1['Answer.noreply'] = 0
        df1[post_max_agreement] = df.loc[df['Input.threadtitle'] == thread, post_max_agreement]
        
        #print(df1)
        aggregate = aggregate_raters(df1.fillna(0).astype(int).T)
        #print(aggregate[0])
        fk = fleiss_kappa(aggregate[0])
        fks.append(fk.calc_fleiss_kappa())
        result_file.write(str(fk.calc_fleiss_kappa())+"\n")
        #print(thread+ "("+str(length)+")"+" -- "+str(fk.calc_fleiss_kappa()))
  
    #################################################################################################################
        
    print("\nAverage Kappa:"+str(np.mean(fks))+"\n")
    results = np.append(results,np.mean(fks))
    return results