def eval_ranking_methods(file_prefix, config_file, 
                         test_directory, 
                         tm_query, 
                         limit = 1000, 
                         img_extension  = '.eps'):
    
    lucene_query = 'all:(%s)' % tm_query # search in all fields 
    print 'Lucene query:', lucene_query
    print 'TM query:', tm_query
    positive_dir = os.path.join(test_directory, "1") # TRUE positive documents 
    TOP_K_TOPICS = 5 # the number topics used for Topic-LDA 
    rocs_file_name = '%s-ROCs' % file_prefix + img_extension
    rocs_img_title = '' # %s: ROC curves' % file_prefix 
    roc_labels = ['Lucene ranking', 
                  'Keyword-LDA ranking' , 
                  'Keyword-LDA * Lucene ranking', 
                  'Topic-LDA ranking' , 
                  'Topic-LDA * Lucene Ranking',
                  'Keyword-LSI ranking']
    
    line_styles = ['ro-','kx-','b+-','c^-','yv-.','gd-'] 
    
    
    #---------------------------------------------- Reads the configuration file
    
    mdl_cfg = read_config(config_file)
    
    
    #------------ Checks whether the keywords are there in the corpus dictionary
    
    dictionary = load_dictionary(mdl_cfg['CORPUS']['dict_file'])
    valid_tokens  = 0 
    for token in tm_query.split():
        if token.strip() not in dictionary.values():
            print token, "is not in the corpus vocabulary. Hence, this word will be ignored from the topic search."
        else: 
            valid_tokens  += 1
            
    if valid_tokens  == 0:
        print 'None of the tokens exist in the dictionary. Exiting topic search!'
        exit()
        
        
    #------------------------------------------------------------- Lucene search

    print 'Lucene ranking'
    lu_docs = search_li(lucene_query, limit, mdl_cfg)
    lu_docs_dict, lu_docs_list = lu_append_nonresp(lu_docs, test_directory)
    lu_res = convert_to_roc_format(lu_docs_list, positive_dir)
    print 
    
    
    #---------------------------------------------------------------- LDA search
    
    # Loads the LDA model 
    lda_dictionary, lda_mdl, lda_index, lda_file_path_index, lda_theta, lda_beta = load_lda_parameters(mdl_cfg)
    
    # To display the LDA model topics based on the 
    # increasing order of entropy   
    # print_lda_topics_on_entropy(lda_mdl, file_name='%s-topic-words.csv' % file_prefix, topn=50) 
    
    # Gets the dominant topics from the LDA model 
    dominant_topics = get_dominant_query_topics(tm_query, lda_dictionary, lda_mdl, TOP_K_TOPICS)
    dominant_topics_idx = [idx for (idx, _) in dominant_topics] # get the topic indices 
    
    
    print 'LDA (w/ keywords) ranking'
    lda_docs = search_tm(tm_query, limit, lda_dictionary, lda_mdl, lda_index, lda_file_path_index)
    lda_res = convert_to_roc_format(lda_docs, positive_dir)
    
    # plot_doc_class_predictions(lda_res, '%s-Keyword-LDA' % file_prefix, img_extension)
    
    
    print 'LDA (w/ keywords) * Lucene ranking'
    lu_tm_docs = fuse_lucene_tm_scores(lu_docs_dict, lda_docs)
    lda_lu_res = convert_to_roc_format(lu_tm_docs, positive_dir)
    
    # plot_doc_class_predictions(lda_lu_res, '%s-Keyword-LDA-Lucene' % file_prefix, img_extension)
    
    
    print 'LDA (w/ query topics) ranking'
    lda_tts_docs = search_tm_topics(dominant_topics_idx, limit, lda_file_path_index, lda_theta) 
    lda_tts_res = convert_to_roc_format(lda_tts_docs, positive_dir)
    
    # plot_doc_class_predictions(lda_tts_res, '%s-Topic-LDA' % file_prefix, img_extension)
    
    print 'LDA (w/ query topics) * Lucene Ranking'
    final_docs_tts = fuse_lucene_tm_scores(lu_docs_dict, lda_tts_docs)
    lda_tts_lu_res = convert_to_roc_format(final_docs_tts, positive_dir)
    
    # plot_doc_class_predictions(lda_tts_lu_res, '%s-Topic-LDA-Lucene' % file_prefix, img_extension)
    
    
    
    #---------------------------------------------------------------- LSI search
    
    print 'LSI (w/ keywords) ranking'
    lsi_docs = search_lsi(tm_query, limit, mdl_cfg)
    lsi_res = convert_to_roc_format(lsi_docs, positive_dir)

    
    
    ## Plot ROC curves  

    results_list = [lu_res, 
                    lda_res, lda_lu_res, 
                    lda_tts_res, lda_tts_lu_res, 
                    lsi_res]
    
    roc_data_list = [ROCData(result, linestyle=line_styles[idx]) 
                     for idx, result in enumerate(results_list)]
    plot_multiple_roc(roc_data_list, title=rocs_img_title, 
                      labels=roc_labels, include_baseline=True, 
                      file_name=rocs_file_name)
Exemplo n.º 2
0
def eval_ranking_varying_topics(query_id, dir_path, 
                                 keywords, 
                                 limit = 1000, 
                                 img_extension  = '.eps'):
    
    tm_query = ' '.join( lemmatize_tokens( regex_tokenizer(keywords)  ) ) # Lemmatization 
    lucene_query = 'all:(%s)' % tm_query # search in all fields 
    print 'Lucene query:', lucene_query
    print 'TM query:', tm_query

    test_directory = "%s%d" % (dir_path, query_id)
    positive_dir = os.path.join(test_directory, "1") # TRUE positive documents 

    TOP_K_TOPICS = 5 # the number topics used for Topic-LDA 
    topiclda_rocs_file_name = '%d-LT-Topic-LDA-VaryingTopics-ROCs' % query_id + img_extension
    topiclda_rocs_img_title = 'Q%d Topic-LDA with Varying Number of Topics' % query_id  
    keywordlda_rocs_file_name = '%d-LT-Keyword-LDA-VaryingTopics-ROCs' % query_id + img_extension
    keywordlda_rocs_img_title = 'Q%d Keyword-LDA with Varying Number of Topics' % query_id  
    topics = [5, 10, 15, 20, 30, 40, 50, 60, 70, 80]
    roc_labels = []
    roc_topiclda_list = []
    roc_keywordlda_list = []

    for idx, num_topics in enumerate(topics): 

        #---------------------------------------------- Reads the configuration file
        
        config_file = "%sQ%d-LT-%dT.cfg" % (dir_path, query_id, num_topics)  # configuration file, created using the SMARTeR GUI 
        mdl_cfg = read_config(config_file)
        
        # Loads the LDA model 
        lda_dictionary, lda_mdl, _, _ = load_tm(mdl_cfg)
        
        
        #------------ Checks whether the keywords are there in the corpus dictionary
    
        valid_tokens  = 0 
        for token in tm_query.split():
            if token.strip() not in lda_dictionary.values():
                print token, "is not in the corpus vocabulary. Hence, this word will be ignored from the topic search."
            else: 
                valid_tokens  += 1
                
        if valid_tokens  == 0:
            print 'None of the tokens exist in the dictionary. Exiting topic search!'
            exit()
            
            
        #------------------------------------------------------------- Lucene search
    
        if idx == 0: # the first Lucene ranking is added as a reference 
            print 'Lucene ranking'
            lu_docs = search_li(lucene_query, limit, mdl_cfg)
            _, lu_docs_list = lu_append_nonresp(lu_docs, test_directory)
            lu_res = convert_to_roc_format(lu_docs_list, positive_dir)
            roc_topiclda_list.append(ROCData(lu_res))
            roc_keywordlda_list.append(ROCData(lu_res))
            roc_labels.append('Lucene')
        
        #---------------------------------------------------------------- LDA search
        
        # Gets the dominant topics from the LDA model 
        dominant_topics = get_dominant_query_topics(tm_query, lda_dictionary, lda_mdl, TOP_K_TOPICS)
        dominant_topics_idx = [idx for (idx, _) in dominant_topics] # get the topic indices 
        
        
        print 'LDA (w/ keywords) ranking'
        lda_docs = search_tm(tm_query, limit, mdl_cfg)
        lda_res = convert_to_roc_format(lda_docs, positive_dir)
        
        
        print 'LDA (w/ query topics) ranking'
        lda_tts_docs = search_tm_topics(dominant_topics_idx, limit, mdl_cfg) 
        lda_tts_res = convert_to_roc_format(lda_tts_docs, positive_dir)
    
        
        roc_topiclda_list.append(ROCData(lda_tts_res))
        roc_keywordlda_list.append(ROCData(lda_res))
        roc_labels.append('%d topics' % num_topics)
        
    
    ## Plot ROC curves  
    
    plot_multiple_roc(roc_topiclda_list, title=topiclda_rocs_img_title, 
                      labels=roc_labels, include_baseline=True, 
                      file_name=topiclda_rocs_file_name)
     
    plot_multiple_roc(roc_keywordlda_list, title=keywordlda_rocs_img_title, 
                      labels=roc_labels, include_baseline=True, 
                      file_name=keywordlda_rocs_file_name)