def build_raw_stemmed_bigrams_stopwords_vocabulary(corpus,stop_words,): from ng20_globals import min_tf,min_df from commons.stemming_tokenizer import RawStemmingTokenizer tokenizer = RawStemmingTokenizer() max_ngram_size = 2 vocabulary = build_vocabulary(corpus,tokenizer,stop_words,max_ngram_size,min_df,min_tf) # save to DB tbl_name = 'ng20_raw_stems_bigrams_stopwords_df{0}_tf{1}'.format(min_df,min_tf) save_vocabulary(vocabulary,tbl_name) print 'done '+tbl_name
def build_raw_stemmed_unigrams_stopwords_vocabulary(corpus,stop_words,vocabulary_src): from clef_globals import min_df, min_tf from commons.stemming_tokenizer import RawStemmingTokenizer tokenizer = RawStemmingTokenizer() max_ngram_size = 1 vocabulary = build_vocabulary(corpus,tokenizer,stop_words,max_ngram_size,min_df,min_tf) # save to DB tbl_name = 'clef_2010_{0}_raw_stems_unigrams_stopwords_df{1}_tf{2}'.format(vocabulary_src,min_df,min_tf) save_vocabulary(vocabulary,tbl_name) print 'done '+tbl_name
def test_stemmed_bigrams_unigrams(bigrams_src, corpus_train_data, corpus_test_data, label_names, with_stopwords_removal, use_chi_features, use_raw_tokens): import numpy as np from commons.stemming_tokenizer import StemmingTokenizer from commons.stemming_tokenizer import RawStemmingTokenizer from ng20_globals import max_labels, min_df, min_tf from ng20_vocabulary_loader import load_common_vocabulary_extend_unigrams max_ngram_size = 2 if with_stopwords_removal == False: stopwords_pattern = '' else: stopwords_pattern = '_stopwords' if use_chi_features == False: chi_features_pattern = '' else: chi_features_pattern = '_chi' if use_raw_tokens == False: raw_tokens_pattern = '' tokenizer = StemmingTokenizer() else: raw_tokens_pattern = '_raw' tokenizer = RawStemmingTokenizer() # load vocabulary vocabulary_tbl_name = 'ng20{0}_stems{1}_bigrams{2}_df{3}_tf{4}'.format( raw_tokens_pattern, chi_features_pattern, stopwords_pattern, min_df, min_tf) if len(bigrams_src) == 1: vocabulary_tbl_intersect = '{0}_bigrams'.format(bigrams_src[0]) else: vocabulary_tbl_intersect = '{0}_'.format(bigrams_src[0]) for i in range(len(bigrams_src) - 1): vocabulary_tbl_intersect = '{0}{1}_'.format( vocabulary_tbl_intersect, bigrams_src[i + 1]) vocabulary_tbl_intersect = '{0}bigrams_vw'.format( vocabulary_tbl_intersect) vocabulary = load_common_vocabulary_extend_unigrams( vocabulary_tbl_name, vocabulary_tbl_intersect, 'stem') print 'done loading vocabulary' # generate tfidf vectors vectorizer, corpus_train_tfidf_vectors = vectorize_corpus( corpus_train_data['corpus'], tokenizer, vocabulary, max_ngram_size) _, corpus_test_tfidf_vectors = vectorize_corpus(corpus_test_data['corpus'], tokenizer, vocabulary, max_ngram_size) # classify & evaluate results = classify(corpus_train_tfidf_vectors, corpus_train_data['labels'], corpus_test_tfidf_vectors, corpus_test_data['labels'], max_labels) print_top_feature_names( results['features_weights'], np.asarray(vectorizer.get_feature_names()), vocabulary_tbl_name + '_' + vocabulary_tbl_intersect, label_names) print vocabulary_tbl_name, '^', vocabulary_tbl_intersect, '(extended unigrams) --> ', 'accuracy ', results[ 'accuracy'] #print vocabulary_tbl_name,'^',vocabulary_tbl_intersect,'(extended unigrams) --> ','precision ',results['precision'],'recall ',results['recall'],'f1 ',results['f1']
def run_experiment(corpus_train_data, corpus_test_data, label_names, with_stopwords_removal=False, use_chi_features=False, preprocess_tokens='raw', ngram_size=1, knowledge_set=None): import numpy as np from commons.lemmatizing_tokenizer import RawTokenizer from commons.lemmatizing_tokenizer import RawLemmaTokenizer from commons.stemming_tokenizer import RawStemmingTokenizer from ng20_globals import max_labels, min_df, min_tf from ng20_vocabulary_loader import load_vocabulary from ng20_vocabulary_loader import load_common_vocabulary from commons.globals import knowledge_dic if with_stopwords_removal == False: stopwords_pattern = '' else: stopwords_pattern = '_stopwords' if use_chi_features == False: chi_features_pattern = '' else: chi_features_pattern = '_chi' if preprocess_tokens == 'lemmatize': raw_tokens_pattern = '_raw_lemmas' tokenizer = RawLemmaTokenizer() elif preprocess_tokens == 'stem': raw_tokens_pattern = '_raw_stems' tokenizer = RawStemmingTokenizer() else: raw_tokens_pattern = '_raw' tokenizer = RawTokenizer() if ngram_size == 1: ngram_pattern = '_unigrams' elif ngram_size == 2: ngram_pattern = '_bigrams' # load vocabulary if knowledge_set != None: vocabulary_tbl_name1 = 'ng20{0}{1}_unigrams{2}_df{3}_tf{4}'.format( raw_tokens_pattern, chi_features_pattern, stopwords_pattern, min_df, min_tf) vocabulary_tbl_name2 = 'ng20{0}{1}_bigrams{2}_df{3}_tf{4}'.format( raw_tokens_pattern, chi_features_pattern, stopwords_pattern, min_df, min_tf) vocabulary_tbl_intersect = knowledge_dic[knowledge_set.lower()] vocabulary = load_common_vocabulary(vocabulary_tbl_name1, vocabulary_tbl_name2, vocabulary_tbl_intersect, 'stem') vocabulary_tbl_name = vocabulary_tbl_name1, '^', vocabulary_tbl_name2, '^', vocabulary_tbl_intersect else: vocabulary_tbl_name = 'ng20{0}{1}{2}{3}_df{4}_tf{5}'.format( raw_tokens_pattern, chi_features_pattern, ngram_pattern, stopwords_pattern, min_df, min_tf) vocabulary = load_vocabulary(vocabulary_tbl_name) # generate tfidf vectors vectorizer, corpus_train_tfidf_vectors = vectorize_corpus( corpus_train_data['corpus'], tokenizer, vocabulary, ngram_size) _, corpus_test_tfidf_vectors = vectorize_corpus(corpus_test_data['corpus'], tokenizer, vocabulary, ngram_size) # classify & evaluate results = classify_cv(corpus_train_tfidf_vectors, corpus_train_data['labels'], corpus_test_tfidf_vectors, corpus_test_data['labels'], max_labels) print_top_feature_names(results['features_weights'], np.asarray(vectorizer.get_feature_names()), vocabulary_tbl_name, label_names) print vocabulary_tbl_name, ' --> ', 'accuracy', results[ 'accuracy'] #print vocabulary_tbl_name,' --> ','precision ',results['precision'],'recall ',results['recall'],'f1 ',results['f1']