def main():
collection_name = "nips"
years = xrange(2010, 2015) # 10 ~ 14
n_topics = 10
corpus_paths = map(lambda y:
"data/{}-{}.dat".format(collection_name, y),
years)
all_corpus = []
year2corpus = {}
for year, path in zip(years, corpus_paths):
corpus = list(load_line_corpus(path))
all_corpus.append(proc_corpus(corpus))
year2corpus[year] = corpus
all_corpus = list(itertools.chain.from_iterable(all_corpus))
dictionary = Dictionary(all_corpus)
all_corpus = [dictionary.doc2bow(doc)
for doc in all_corpus]
import pdb
pdb.set_trace()
# print all_corpus
model = LdaModel(all_corpus, num_topics=n_topics,
id2word=dictionary,
eval_every=10, passes=100)
print model.show_topics()
def test_load_line_corpus():
corpus = load_line_corpus("test_data/line_corpus.dat")
corpus = list(corpus)
assert_equal(len(corpus), 3)
assert_equal(corpus[0], u"We introduce the Randomized Dependence Coefficient (RDC), a measure of non-linear dependence between random variables of arbitrary dimension based on the Hirschfeld-Gebelein-Rényi Maximum Correlation Coefficient. RDC is defined in terms of correlation of random non-linear copula projections; it is invariant with respect to marginal distribution transformations, has low computational cost and is easy to implement: just five lines of R code, included at the end of the paper.")
def main():
# parameters
collection_name = "nips"
years = xrange(2008, 2015) # 10 ~ 14
n_topics = 6
n_top_words = 15
# load corpus
corpus_paths = map(lambda y:
"data/{}-{}.dat".format(collection_name, y),
years)
all_corpus = []
year2corpus = {}
for year, path in zip(years, corpus_paths):
corpus = list(load_line_corpus(path))
all_corpus.append(corpus)
year2corpus[year] = corpus
all_corpus = list(itertools.chain.from_iterable(all_corpus))
preprocessor = lambda doc: ' '.join(transform(doc, ALL_PIPELINE_NAMES))
tokenizer = lambda doc: doc.split()
with codecs.open('data/lemur-stopwords.txt',
'r' 'utf8') as f:
stop_words = map(lambda s: s.strip(), f.readlines())
vectorizer = CountVectorizer(preprocessor=preprocessor,
tokenizer=tokenizer,
stop_words=stop_words,
min_df=5)
X = vectorizer.fit_transform(all_corpus)
id2word = {id_: word
for word, id_ in vectorizer.vocabulary_.items()}
# build the model
model = lda.LDA(n_topics=n_topics, n_iter=700,
# alpha=1.0, eta=1.0,
random_state=1)
model.fit(X)
# print topics
for i, topic_dist in enumerate(model.topic_word_):
top_word_ids = np.argsort(topic_dist)[:-n_top_words:-1]
topic_words = [id2word[id_] for id_ in top_word_ids]
print('Topic {}: {}'.format(i, ' '.join(topic_words)))
year2docs = {}
start_document_index = 0
for year in years:
corpus_size = len(year2corpus[year])
end_document_index = start_document_index + corpus_size
year2docs[year] = np.arange(start_document_index, end_document_index)
start_document_index = end_document_index
tbl = doc_topic_strengths_over_periods(model.doc_topic_, year2docs)
print tbl
print np.array(tbl.values())
