def test_doc_topic_strengths_over_periods():
# 2 topics, 2 periods(2+3 docs)
doc_topic_matrix = np.asarray([[0.1, 0.9],
[0.2, 0.8],
[0.8, 0.2],
[0.7, 0.3],
[0.3, 0.7]])
period2docs = {'p1': [0, 1],
'p2': [2, 3, 4]}
actual = doc_topic_strengths_over_periods(doc_topic_matrix, period2docs)
expected = {'p1': np.asarray([0.15, 0.85]),
'p2': np.asarray([0.6, 0.4])}
assert_equal(len(actual), 2)
assert_array_almost_equal(actual['p1'], expected['p1'])
assert_array_almost_equal(actual['p2'], expected['p2'])
def main():
# parameters
collection_name = "nips"
years = xrange(2008, 2015) # 10 ~ 14
n_topics = 6
n_top_words = 15
# load corpus
corpus_paths = map(lambda y:
"data/{}-{}.dat".format(collection_name, y),
years)
all_corpus = []
year2corpus = {}
for year, path in zip(years, corpus_paths):
corpus = list(load_line_corpus(path))
all_corpus.append(corpus)
year2corpus[year] = corpus
all_corpus = list(itertools.chain.from_iterable(all_corpus))
preprocessor = lambda doc: ' '.join(transform(doc, ALL_PIPELINE_NAMES))
tokenizer = lambda doc: doc.split()
with codecs.open('data/lemur-stopwords.txt',
'r' 'utf8') as f:
stop_words = map(lambda s: s.strip(), f.readlines())
vectorizer = CountVectorizer(preprocessor=preprocessor,
tokenizer=tokenizer,
stop_words=stop_words,
min_df=5)
X = vectorizer.fit_transform(all_corpus)
id2word = {id_: word
for word, id_ in vectorizer.vocabulary_.items()}
# build the model
model = lda.LDA(n_topics=n_topics, n_iter=700,
# alpha=1.0, eta=1.0,
random_state=1)
model.fit(X)
# print topics
for i, topic_dist in enumerate(model.topic_word_):
top_word_ids = np.argsort(topic_dist)[:-n_top_words:-1]
topic_words = [id2word[id_] for id_ in top_word_ids]
print('Topic {}: {}'.format(i, ' '.join(topic_words)))
year2docs = {}
start_document_index = 0
for year in years:
corpus_size = len(year2corpus[year])
end_document_index = start_document_index + corpus_size
year2docs[year] = np.arange(start_document_index, end_document_index)
start_document_index = end_document_index
tbl = doc_topic_strengths_over_periods(model.doc_topic_, year2docs)
print tbl
print np.array(tbl.values())
