def save_train_and_test(data): with open(data, 'rb') as file: # read the data as binary data stream print("... Reading the pre-processed data from local binary file...") documents = pickle.load(file) documents = extract_important_words_tfidf( documents, 0.60) # extracting top 60% (TF-IDF) terms per document documents = remove_low_high_frequent_words(documents, 0.03, 1.0) corpus = get_tfidf(documents)["corpus_tfidf"] dictionary = get_tfidf(documents)["index2word"] corpus = list(corpus) random.shuffle(corpus) train = corpus[:18000] test = corpus[18000:] with open('data/train_corpus.data', 'wb') as file: print("...Saving training corpus into local binary file...") pickle.dump(train, file) with open('data/test_corpus.data', 'wb') as file: print("...Saving test corpus into local binary file...") pickle.dump(test, file) with open('data/common_dictionary.data', 'wb') as file: print( "...Saving common dictionary for train and test corpus into binary file..." ) pickle.dump(dictionary, file)
def get_corpus_words(data): with open(data, 'rb') as file: # read the data as binary data stream print("... Reading the pre-processed data from local binary file...") documents = pickle.load(file) documents = extract_important_words_tfidf( documents, 0.60) # extracting top 60% (TF-IDF) terms per document documents = remove_low_high_frequent_words(documents, 0.03, 1.0) dictionary = Dictionary(documents) return dictionary.values() # returning all words in the data corpus
def words_to_self_trained_word2vec(train_data, words): with open(train_data, 'rb') as file: # read the data as binary data stream print("... Reading the pre-processed data from local binary file...") documents = pickle.load(file) documents = extract_important_words_tfidf( documents, 0.60) # extracting top 60% (TF-IDF) terms per document documents = remove_low_high_frequent_words(documents, 0.03, 1.0) # train word2vec model with the documents word2vec_model = Word2Vec(documents, size=100, window=10, min_count=1, workers=10) vectors = [word2vec_model.wv[word] for word in words] return vectors
def train_hlda_model(data): with open(data, 'rb') as file: # read the data as binary data stream print("... Reading the pre-processed data from local binary file...") documents = pickle.load(file) documents = extract_important_words_tfidf( documents, 0.60) # extracting top 60% (TF-IDF) terms per document documents = remove_low_high_frequent_words(documents, 0.03, 1.0) # ++++++ preparing dictionary and corpus for h-LDA library ++++++ # dictionary_set = set() for doc in documents: for word in doc: dictionary_set.add(word) vocab = sorted(list(dictionary_set)) vocab_index = {} for i, w in enumerate(vocab): vocab_index[w] = i new_corpus = [] for doc in documents: new_doc = [] for word in doc: word_idx = vocab_index[word] new_doc.append(word_idx) new_corpus.append(new_doc) # --------------------------------------------------------------- # hlda_model = HierarchicalLDA( corpus=new_corpus, vocab=vocab, alpha=10.0, # default = 10.0 gamma=1.0, # default = 1.0 eta=1.0, # default = 0.1 seed=0, # default = 0 verbose=True, # default = True num_levels=3) # default = 3 return hlda_model, new_corpus, vocab_index
def compute_coherence_values_topic_num(data, limit, start=2, step=1): """ Compute u_mass coherence for various number of topics Returns: ------- model_list : List of LDA topic models coherence_values : Coherence values corresponding to the LDA model with respective number of topics """ with open(data, 'rb') as file: # read the data as binary data stream print("... Reading the pre-processed data from local binary file...") documents = pickle.load(file) documents = extract_important_words_tfidf( documents, 0.60) # extracting top 60% (TF-IDF) terms per document documents = remove_low_high_frequent_words(documents, 0.03, 1.0) corpus = get_tfidf(documents)["corpus_tfidf"] dictionary = get_tfidf(documents)["index2word"] coherence_values = [] model_list = [] for num_topics in range(start, limit, step): model = models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=num_topics, eta=0.3) model_list.append(model) coherencemodel = CoherenceModel(model=model, dictionary=dictionary, corpus=corpus, coherence='u_mass') coherence_values.append(coherencemodel.get_coherence()) return model_list, coherence_values
def self_trained_word2vec(training_corpus, topic_words): with open(training_corpus, 'rb') as file: # read the data as binary data stream print("... Reading the pre-processed data from local binary file...") documents = pickle.load(file) documents = extract_important_words_tfidf( documents, 0.60) # extracting top 60% (TF-IDF) terms per document documents = remove_low_high_frequent_words(documents, 0.03, 1.0) # train word2vec model with the documents word2vec_model = Word2Vec(documents, size=100, window=10, min_count=1, workers=10) topics_in_vector = [] for terms_list in topic_words: topics_in_vector.append( sent_vectorizer_word2vec(terms_list, word2vec_model)) return topics_in_vector # list of vectors (vector per topic)
def self_trained_doc2vec(training_corpus, topic_words): with open(training_corpus, 'rb') as file: # read the data as binary data stream print("... Reading the pre-processed data from local binary file...") documents = pickle.load(file) documents = extract_important_words_tfidf( documents, 0.60) # extracting top 60% (TF-IDF) terms per document documents = remove_low_high_frequent_words(documents, 0.03, 1.0) documents = [" ".join(doc) for doc in documents] documents = [TaggedDocument(doc, [i]) for i, doc in enumerate(documents)] doc2vec_model = Doc2Vec(documents, vector_size=10, window=2, min_count=1, workers=5) topics_in_vector = [] for terms_list in topic_words: topics_in_vector.append(doc2vec_model.infer_vector(terms_list)) return topics_in_vector
def train_lda_model(data, num_topics): with open(data, 'rb') as file: # read the data as binary data stream print("... Reading the pre-processed data from local binary file...") documents = pickle.load(file) documents = extract_important_words_tfidf( documents, 0.60) # extracting top 60% (TF-IDF) terms per document documents = remove_low_high_frequent_words(documents, 0.03, 1.0) corpus = get_tfidf(documents)["corpus_tfidf"] in2word = get_tfidf(documents)["index2word"] lda_model = models.ldamodel.LdaModel( corpus=corpus, num_topics=num_topics, id2word=in2word, distributed=False, # default: False chunksize=2000, # default: 2000 passes=1, # default: 1 update_every=1, # default: 1 alpha='symmetric', # default: 'symmetric' eta=0.3, # default: None ; ** taking non-default value ** decay=0.5, # default: 0.5 offset=1.0, # default: 1.0 eval_every=10, # default: 10 iterations=50, # default: 50 gamma_threshold=0.001, # default: 0.001 minimum_probability=0.01, # default: 0.01 random_state=None, # default: None ns_conf=None, # default: None minimum_phi_value=0.01, # default: 0.01 per_word_topics=False, # default: False callbacks=None # default: None ) return lda_model, corpus, in2word