def testMallet2Model(self):
if not self.mallet_path:
return
tm1 = ldamallet.LdaMallet(self.mallet_path,
corpus=corpus,
num_topics=2,
id2word=dictionary)
tm2 = ldamallet.malletmodel2ldamodel(tm1)
# set num_topics=-1 to exclude random influence
self.assertEqual(tm1.show_topics(-1, 10), tm2.show_topics(-1, 10))
for document in corpus:
element1_1, element1_2 = tm1[document][0]
element2_1, element2_2 = tm2[document][0]
self.assertAlmostEqual(element1_1, element2_1)
self.assertAlmostEqual(element1_2, element2_2, 1)
element1_1, element1_2 = tm1[document][1]
element2_1, element2_2 = tm2[document][1]
self.assertAlmostEqual(element1_1, element2_1)
self.assertAlmostEqual(element1_2, element2_2, 1)
logging.debug('%d %d', element1_1, element2_1)
logging.debug('%d %d', element1_2, element2_2)
logging.debug('%s %s', tm1[document][1], tm2[document][1])
def create_document_topic_df(self, model=None, no_topics=10):
"""
Creates a dataframe containing the the result of the LDA model for each document. Will set the topic with the
highest share within the document as the dominant topic.
:param model: LDA model to use for the calculation of the topic distribution of each document.
:param no_topics: Number of topics in case no LDA model is provided.
"""
if model is None:
model = self.lda_model
if isinstance(model, LdaMallet):
model = malletmodel2ldamodel(model)
topic_result_list = []
for doc in model.get_document_topics(bow=self.bag_of_words):
temp_dict = {}
for topic, probability in doc:
temp_dict[topic] = probability
topic_result_list.append(temp_dict)
self.result_df = pd.DataFrame(data=topic_result_list,
columns=range(model.num_topics))
self.result_df = self.result_df.fillna(0)
if self.document_ids is not None and not self.language_detection:
self.result_df.index = self.document_ids
elif self.document_ids is not None and self.language_detection:
raise Warning(
"Using document ids and language detection together is not implemented (yet)."
)
dominant_topic = np.argmax(self.result_df.values, axis=1)
self.result_df['dominant_topic'] = dominant_topic
def topic_modelling(data_object_name):
"""
perform topic modelign for a given set of posts (data object)
:param data_object_name: raw data for topic modeling
"""
data_words = Serialization.load_obj(data_object_name)
stop_words = stopwords.words('english')
print('removing stopwords and unfrequent words...')
ranks = Serialization.load_obj('dict.ranks')
data_words = Utils.remove_noncontent_words(data_words, stop_words,
ranks)
id2word = corpora.Dictionary(data_words)
corpus = [id2word.doc2bow(post) for post in data_words]
topics = CS_TOPICS
print('performing topic modeling with', topics, 'topics')
ldamodel = LdaMallet(mallet_path,
corpus=corpus,
num_topics=topics,
id2word=id2word)
pprint(
malletmodel2ldamodel(ldamodel).top_topics(corpus, data_words,
id2word))
'''
def load(self, model_file):
"""
Loads a LDA model from a given file
:param model_file: the file which contains the model, which should be loaded
"""
self.ldamodel = gensim.models.wrappers.LdaMallet.load(model_file)
from gensim.models.wrappers.ldamallet import malletmodel2ldamodel
self.ldamodel = malletmodel2ldamodel(self.ldamodel)
print(self.ldamodel.__dict__)
def testMallet2Model(self):
if not self.mallet_path:
return
passed = False
tm1 = ldamallet.LdaMallet(self.mallet_path, corpus=corpus, num_topics=2, id2word=dictionary)
tm2 = ldamallet.malletmodel2ldamodel(tm1)
for document in corpus:
self.assertEqual(tm1[document][0], tm2[document][0])
self.assertEqual(tm1[document][1], tm2[document][1])
logging.debug('%d %d', tm1[document][0], tm2[document][0])
logging.debug('%d %d', tm1[document][1], tm2[document][1])
def testMallet2Model(self):
if not self.mallet_path:
return
passed = False
tm1 = ldamallet.LdaMallet(self.mallet_path, corpus=corpus, num_topics=2, id2word=dictionary)
tm2 = ldamallet.malletmodel2ldamodel(tm1)
for document in corpus:
self.assertEqual(tm1[document][0], tm2[document][0])
self.assertEqual(tm1[document][1], tm2[document][1])
logging.debug('%d %d', tm1[document][0], tm2[document][0])
logging.debug('%d %d', tm1[document][1], tm2[document][1])
def testMallet2ModelOn20NewsGroups(self):
corpus = [simple_preprocess(doc["data"]) for doc in api.load("20-newsgroups")]
dictionary = Dictionary(corpus)
corpus = [dictionary.doc2bow(text) for text in corpus]
lda_mallet_model = ldamallet.LdaMallet(
self.mallet_path, corpus=corpus,
num_topics=20, id2word=dictionary, iterations=500)
lda_gensim_model = ldamallet.malletmodel2ldamodel(lda_mallet_model, iterations=1000)
self.assertEqual(lda_mallet_model.show_topics(20, 50), lda_gensim_model.show_topics(20, 50))
def train_model(num_topics, documents):
# documents = get_dictionary()
dictionary = corpora.Dictionary(documents)
max_tokens = len(dictionary.keys())
# print(f'Num tokens before cleanup {len(dictionary.keys())}')
dictionary.filter_extremes(no_below=10, no_above=0.7, keep_n=max_tokens)
# print(f'Num tokens after cleanup {len(dictionary.keys())}')
corpus_bow = [dictionary.doc2bow(doc) for doc in documents]
mallet_model = LdaMallet(mallet_path=MALLET_BINARY_PATH,
corpus=corpus_bow,
id2word=dictionary,
num_topics=num_topics)
lda_model = ldamallet.malletmodel2ldamodel(mallet_model)
return lda_model, corpus_bow, dictionary
def testMallet2ModelOn20NewsGroups(self):
corpus = [
simple_preprocess(doc["data"]) for doc in api.load("20-newsgroups")
]
dictionary = Dictionary(corpus)
corpus = [dictionary.doc2bow(text) for text in corpus]
lda_mallet_model = ldamallet.LdaMallet(self.mallet_path,
corpus=corpus,
num_topics=20,
id2word=dictionary,
iterations=500)
lda_gensim_model = ldamallet.malletmodel2ldamodel(lda_mallet_model,
iterations=1000)
self.assertEqual(lda_mallet_model.show_topics(20, 50),
lda_gensim_model.show_topics(20, 50))
def testMallet2Model(self):
if not self.mallet_path:
return
tm1 = ldamallet.LdaMallet(self.mallet_path, corpus=corpus, num_topics=2, id2word=dictionary)
tm2 = ldamallet.malletmodel2ldamodel(tm1)
for document in corpus:
element1_1, element1_2 = tm1[document][0]
element2_1, element2_2 = tm2[document][0]
self.assertAlmostEqual(element1_1, element2_1)
self.assertAlmostEqual(element1_2, element2_2, 1)
element1_1, element1_2 = tm1[document][1]
element2_1, element2_2 = tm2[document][1]
self.assertAlmostEqual(element1_1, element2_1)
self.assertAlmostEqual(element1_2, element2_2, 1)
logging.debug('%d %d', element1_1, element2_1)
logging.debug('%d %d', element1_2, element2_2)
logging.debug('%d %d', tm1[document][1], tm2[document][1])
def testMallet2Model(self):
if not self.mallet_path:
return
passed = False
tm1 = ldamallet.LdaMallet(self.mallet_path,
corpus=corpus,
num_topics=2,
id2word=dictionary)
tm2 = ldamallet.malletmodel2ldamodel(tm1)
for document in corpus:
element1_1, element1_2 = tm1[document][0]
element2_1, element2_2 = tm2[document][0]
self.assertAlmostEqual(element1_1, element2_1)
self.assertAlmostEqual(element1_2, element2_2, 1)
element1_1, element1_2 = tm1[document][1]
element2_1, element2_2 = tm2[document][1]
self.assertAlmostEqual(element1_1, element2_1)
self.assertAlmostEqual(element1_2, element2_2, 1)
logging.debug('%d %d', element1_1, element2_1)
logging.debug('%d %d', element1_2, element2_2)
logging.debug('%d %d', tm1[document][1], tm2[document][1])
def model_mallet(clean_doc, dictionary, doc_term_matrix):
lda_mallet = LdaMallet(mallet_path,
corpus=doc_term_matrix,
id2word=dictionary,
num_topics=25,
workers=3)
print("Topics generated with the mallet LDA model are:\n")
pprint(lda_mallet.show_topics(formatted=False))
print("----------------------------------------------------")
coherence_model_mallet = CoherenceModel(model=lda_mallet,
texts=clean_doc,
dictionary=dictionary,
coherence='c_v')
coherence_mallet = coherence_model_mallet.get_coherence()
print(f"coherence score: {coherence_mallet}")
mallet_2 = ldamallet.malletmodel2ldamodel(lda_mallet)
return mallet_2
def testMallet2Model(self):
if not self.mallet_path:
return
tm1 = ldamallet.LdaMallet(self.mallet_path, corpus=corpus, num_topics=2, id2word=dictionary)
tm2 = ldamallet.malletmodel2ldamodel(tm1)
# set num_topics=-1 to exclude random influence
self.assertEqual(tm1.show_topics(-1, 10), tm2.show_topics(-1, 10))
for document in corpus:
element1_1, element1_2 = tm1[document][0]
element2_1, element2_2 = tm2[document][0]
self.assertAlmostEqual(element1_1, element2_1)
self.assertAlmostEqual(element1_2, element2_2, 1)
element1_1, element1_2 = tm1[document][1]
element2_1, element2_2 = tm2[document][1]
self.assertAlmostEqual(element1_1, element2_1)
self.assertAlmostEqual(element1_2, element2_2, 1)
logging.debug('%d %d', element1_1, element2_1)
logging.debug('%d %d', element1_2, element2_2)
logging.debug('%s %s', tm1[document][1], tm2[document][1])
def get_optimum_topics(df, dictionary, doc_term_matrix, clean_doc, start,
limit):
list_models, list_coherence = compute_coherence_values(
dnary=dictionary,
corpus=doc_term_matrix,
texts=clean_doc,
limit=30,
start=2,
step=1)
limit = limit
start = start
x = range(start, limit)
plt.plot(x, list_coherence)
plt.xlabel("Num Topics")
plt.ylabel("Coherence Score")
plt.show()
final_model = list_models[list_coherence.index(max(list_coherence))]
final_model_use = ldamallet.malletmodel2ldamodel(final_model)
df["topic"] = df["reviewContent"].apply(get_review_topic,
args=(dictionary, final_model_use))
def evaluate_pyldavis(self, model=None, use_jupyter=None):
"""
Method for a visual evaluation of the LDA topic model using pyldavis.
:param model: LDA model that is to be evaluated. If 'None', it will use the last model that has been saved
within the class.
:param use_jupyter: set how the pyldavis panel is displayed. If default (None), it will try to find out if run
from jupyter and set the method accordingly
:return:
"""
if model is None:
if self.lda_model is None:
raise Exception(
"Please create a LDA model for evaluation before running this method."
)
model = self.lda_model
if isinstance(model, LdaMallet):
model = malletmodel2ldamodel(model)
panel = pyLDAvis.gensim.prepare(model, self.bag_of_words, self.id2word)
if use_jupyter is None:
try:
is_jupyter = os.environ['_'].split(
"/")[-1] == "jupyter-notebook"
if is_jupyter:
pyLDAvis.enable_notebook()
except KeyError:
is_jupyter = False
if is_jupyter:
pyLDAvis.display(panel)
else:
pyLDAvis.show(panel)
else:
if use_jupyter:
pyLDAvis.enable_notebook()
pyLDAvis.display(panel)
elif not use_jupyter:
pyLDAvis.show(panel)
def topical_differences_sig_analysis():
"""
testing code-switching and monolingual english posts for topical differences
(1) partition code-switched posts into two random sets
(2) perform topic modeling of each partition and compute the similarity between the two parts and
their individual similarity to topics extracted from monolingual posts
(3) test the multiple-experiment similarity scores for significance
"""
data_object_name = 'monolingual.preprocessed'
data_words = Serialization.load_obj(data_object_name)
stop_words = stopwords.words('english')
print('removing stopwords and infrequent words...')
ranks = Serialization.load_obj('dict.ranks')
data_words = Utils.remove_noncontent_words(data_words, stop_words,
ranks)
print('after pre-processing: total of', len(data_words), 'posts')
topics = MONOLINGUAL_TOPICS
for i in range(EXPERIMENTS):
shuffle(data_words)
part1 = data_words[:math.floor(len(data_words) / 2)]
part2 = data_words[math.floor(len(data_words) / 2):]
model = Utils.model_topic(part1, topics)
Serialization.save_obj(model,
'lda.mallet.monolingual.part1.' + str(i))
print('saved topic model: part1,', i)
model = Utils.model_topic(part2, topics)
Serialization.save_obj(model,
'lda.mallet.monolingual.part2.' + str(i))
print('saved topic model: part2,', i)
sys.stdout.flush()
# end for
inter = []
intra = []
ldamodel_cs = malletmodel2ldamodel(
Serialization.load_obj('lda.mallet.cs'))
for i in range(30):
print('processing', i)
ldamodel_mono1 = malletmodel2ldamodel(
Serialization.load_obj('lda.mallet.monolingual.part1.' +
str(i)))
ldamodel_mono2 = malletmodel2ldamodel(
Serialization.load_obj('lda.mallet.monolingual.part2.' +
str(i)))
diff_matrix1, _ = ldamodel_cs.diff(ldamodel_mono1,
distance='jaccard')
diff_matrix2, _ = ldamodel_cs.diff(ldamodel_mono2,
distance='jaccard')
#intra.append(np.mean([np.mean(np.matrix(diff_matrix1)), np.mean(np.matrix(diff_matrix2))]))
intra.append(
np.mean([
np.min(np.matrix(diff_matrix1)),
np.min(np.matrix(diff_matrix2))
]))
diff_matrix3, _ = ldamodel_mono1.diff(ldamodel_mono2,
distance='jaccard')
#inter.append(np.mean(np.matrix(diff_matrix3)))
inter.append(np.min(np.matrix(diff_matrix3)))
# end for
print(np.mean(intra), np.mean(inter))
_, pval = ranksums(intra, inter)
print('pval:', pval)
def build(mallet_path,
tweets_df,
output_dir,
num_topics=100,
num_iterations=1000,
verbose=False):
"""
Builds a topic model from the given tweets. Writes the following files into
the output directory:
* summary.json: A JSON summary of the model, including top word
weights
* tweet_topics.csv: A CSV file indexed by tweet ID where each column represents
the weight of a given topic in the tweet.
* model.pkl: A pickle file that contains the LDA model.
"""
if not os.path.exists(output_dir):
os.mkdir(output_dir)
print("Preprocessing tweets...")
initial_data = tweets_df.standardized_text.values.tolist()
lemmatized_data = [tweet.split() for tweet in tqdm.tqdm(initial_data)]
# Create dictionary and term-frequency mapping
print("Building corpus...")
id2word = corpora.Dictionary(lemmatized_data)
tf = [id2word.doc2bow(tweet) for tweet in lemmatized_data]
id2word.save(os.path.join(output_dir, "dictionary.pkl"))
if verbose:
print(
'The corpus (token_id, #occurances in this doc) for example tweets are:'
)
print(
'Note words are in order of token_id, not order of original tweet')
for i in range(20):
print(tf[i])
print('The first 20 words in the dictionary are:')
for i in range(20):
print(i, id2word[i])
# Build the LDA model
start_time = datetime.datetime.now()
print("Building model...")
if verbose: print('Started at ', str(start_time))
lda_model = gensim.models.wrappers.LdaMallet(mallet_path=os.path.join(
mallet_path, "bin", "mallet"),
corpus=tf,
num_topics=num_topics,
iterations=num_iterations,
id2word=id2word)
end_time = datetime.datetime.now()
print("Saving model...")
lda_model.save(os.path.join(output_dir, "model.pkl"))
if verbose:
print('Elapsed time: {}'.format(str(end_time - start_time)))
# Write outputs
gensim_lda_model = malletmodel2ldamodel(lda_model)
write_summary(lda_model,
gensim_lda_model,
lemmatized_data,
id2word,
tf,
output_dir,
num_topics=num_topics)
write_tweet_topics(tweets_df,
gensim_lda_model,
id2word,
tf,
output_dir,
num_topics=num_topics)
print("Done.")
class Topics(object):
__dict_path = os.path.join(os.path.dirname(__file__),
'models/mallet-dict.pkl')
__model_path = os.path.join(os.path.dirname(__file__),
'models/mallet-model.model')
__mallet_path = os.path.join(os.path.dirname(__file__),
'models/mallet/bin/mallet')
__topic_file_path = os.path.join(os.path.dirname(__file__),
'models/topic-files/')
dictionary = unpickle(__dict_path)
model = LdaMallet.load(__model_path)
model.mallet_path = __mallet_path
model.prefix = __topic_file_path
model_fast = malletmodel2ldamodel(model, 0.1, 1000)
topic_map = {
0: 'education',
1: 'dating',
2: 'change',
3: 'communication',
4: 'broken relationship', # relationship status
5: 'finances and accounting',
6: 'excessive thoughts',
7: 'politics',
8: 'financial investments',
9: 'physical health',
10: 'work',
11: 'sleep',
12: 'emotions',
13: 'medication regimen',
14: 'past experiences / decisions', # or decisions
15: 'general apathy',
16: 'NaN', # ignore
17: 'relocation',
18: 'social stressors',
19: 'memories',
20: 'financial decisions',
21: 'family',
22: 'nutrition and weight',
23: 'relationships',
24: 'marital issues',
25: 'religion and belief systems',
26: 'experiences',
27: 'financial pressure',
28: 'romantic relationship',
29: 'relationship issues',
30: 'routines',
31: 'taxes and claims', # income and benefits
32: 'symptoms of mental illness',
33: 'dispute and argument',
34: 'lack of motivation',
35: 'reflection and mindfulness',
36: 'event or festivity',
37: 'self-harm', # suicide
38: 'resources and information',
39: 'addiction',
40: 'addiction recovery',
41: 'leisure'
}
def get_topics(self, topics):
top_topics = topics[:, 1].argsort()[-5:][::-1]
# TODO: weight down scoring
scores = 0.
results = {}
for idx, entry in enumerate(top_topics):
topic = int(topics[entry][0])
score = topics[entry][1]
if idx == 0 and score <= .1:
return None
if scores < .55:
if self.topic_map[topic] != 'NaN':
results[self.topic_map[topic]] = score
scores += score
else:
break
return results
# def retrieve(self, doc):
# tokens = self.get_tokens(doc)
# bow = self.dictionary.doc2bow(tokens)
# topics = np.array(self.model[bow])
# return self.get_topics(topics)
def retrieve(self, doc):
tokens = self.get_tokens(doc)
bow = self.dictionary.doc2bow(tokens)
topics = np.array(self.model_fast[bow])
return self.get_topics(topics)
@staticmethod
def get_tokens(doc):
result = []
for tok in doc:
if tok.pos_ in ['IN', 'MD', 'CD']:
continue
if tok.is_digit or tok.like_num:
continue
if tok.is_punct:
continue
elif tok.is_stop:
continue
else:
result.append(tok.text.lower())
return result
doc_term_mat_train = [dictionary.doc2bow(doc) for doc in docs_train]
doc_term_mat_test = [dictionary.doc2bow(doc) for doc in docs_test]
path_to_mallet_binary = r'C:\mallet\bin\mallet'
if __name__ == "__main__":
model = LdaMallet(path_to_mallet_binary,
corpus=doc_term_mat_train,
alpha=5,
num_topics=10,
id2word=dictionary,
optimize_interval=50)
topics = model.print_topics()
for topic in topics:
print(topic)
# Compute Coherence Score for base model
coherence_model_lda = CoherenceModel(model=model,
corpus=doc_term_mat_train,
texts=docs_train,
dictionary=dictionary,
coherence='c_v')
coherence_lda = coherence_model_lda.get_coherence()
gensim_model = ldamallet.malletmodel2ldamodel(model)
# Visualize the topics
vis_prepared = pyLDAvis.gensim.prepare(gensim_model, doc_term_mat_train,
dictionary)
pyLDAvis.save_html(vis_prepared, "mallet.html")
print('\nCoherence Score: ', coherence_lda)
plot_word_cloud(gensim_model)
df['lemmatized'] = df['no_stop_words'].parallel_apply(lemmatization)
print(df['lemmatized'].head(15))
dictionary = corpora.Dictionary(df['lemmatized'])
dictionary.filter_extremes(no_below=10, no_above=0.1)
df['doc_to_bowed'] = df['lemmatized'].parallel_apply(dictionary.doc2bow)
print(df['doc_to_bowed'].head(5))
corpus = df['doc_to_bowed']
ldamallet = gensim.models.wrappers.LdaMallet(mallet_path,
corpus=corpus,
num_topics=TOPICS_NUMBER,
id2word=dictionary)
lda_native = malletmodel2ldamodel(ldamallet)
lda_native.save("model/mallet_to_native.lda")
corpora.Dictionary.save(dictionary, "model/dictionary.dict")
corpora.BleiCorpus.save_corpus(fname="model/corpus.lda-c", corpus=corpus)
pprint(lda_native.show_topics(20))
# MIGHT BE USEFUL, SO DONT REMOVE IT
# bigram = gensim.models.Phrases(
# df['bigrams'], min_count=5, threshold=100
# ) # higher threshold fewer phrases.
# trigram = gensim.models.Phrases(bigram[df['text_new']], threshold=100)
# # Faster way to get a sentence clubbed as a trigram/bigram
# bigram_mod = gensim.models.phrases.Phraser(bigram)
# Create a corpus from a list of texts
data = [a.split() for a in res]
dictionary = Dictionary(data)
corpus = [dictionary.doc2bow(t) for t in data]
# os.environ['MALLET_HOME'] = 'X:\\Programs\\mallet\\mallet-2.0.8\\'
mallet_path = 'X:\\Programs\\mallet\\mallet-2.0.8\\bin\\mallet.bat'
# Train the model on the corpus.
lda = LdaMallet(mallet_path, corpus, id2word=dictionary, num_topics=10)
# lda = LdaModel(corpus, id2word=dictionary, num_topics=10, alpha='auto', eval_every=5, chunksize=10, passes=10, decay=0.9)
elapsed_time = time.time() - start_time
print(time.strftime("Lda model criado, demorou %H:%M:%S:%m", time.gmtime(elapsed_time)))
for index, topic in lda.show_topics(formatted=False, num_words=10):
print('Topic: {} \nWords: {}'.format(
index, [pipe.predict([w[0]])[0][0] for w in topic]))
# for index, topic in lda.show_topics(formatted=False, num_words=20):
# print('Topic: {} \nWords: {}'.format(index, [w[0] for w in topic]))
model = malletmodel2ldamodel(lda)
vis = pyLDAvis.gensim.prepare(model, corpus, dictionary)
vis
pyLDAvis.save_html(vis, 'lda-pt-mallet.html')