def getSentenceDistributions(self):
# computes topic distributions for each sentence
# output: list of lists
# each list corresponds to a document and stores a tuple per sentence
# the 1st element is the sentence number in the group
# the 2nd element is a tuple of (topic_id, weight)
distributions = list()
get_bow = self.dictionary.doc2bow
get_document_topics = self.lda.get_document_topics
for sentences in self.sentence_groups:
sentence_distributions = list()
for k, sentence in sentences:
tkns = tokenizer(sentence)
if tkns is None:
continue
bow = get_bow(tkns)
dist = get_document_topics(bow)
# this is to get list of dominant indices in decreasing order
#dist.sort(key=lambda x: x[1], reverse=True)
#dist = [d[0] for d in dist]
#
# this is to get the dominant index only (not a list)
try:
dist = max(dist, key=lambda x: x[1])
except ValueError:
continue
sentence_distributions.append((k, dist))
distributions.append(sentence_distributions)
return distributions
def getSentenceDistributions(self):
# computes topic distributions for each sentence
# output: list of lists
# each list corresponds to a document and stores a tuple per sentence
# the 1st element is the sentence number in the group
# the 2nd element is a tuple of (topic_id, weight)
distributions = list()
get_bow = self.dictionary.doc2bow
get_document_topics = self.lda.get_document_topics
for sentences in self.sentence_groups:
sentence_distributions = list()
for k, sentence in sentences:
tkns = tokenizer(sentence)
if tkns is None:
continue
bow = get_bow(tkns)
dist = get_document_topics(bow)
# this is to get list of dominant indices in decreasing order
#dist.sort(key=lambda x: x[1], reverse=True)
#dist = [d[0] for d in dist]
#
# this is to get the dominant index only (not a list)
try:
dist = max(dist, key=lambda x: x[1])
except ValueError, ve:
continue
sentence_distributions.append((k, dist))
distributions.append(sentence_distributions)
def summarize(self, documents):
tokens = [tokenizer(document) for document in documents]
tokens = [self.bigramizer[tkn] for tkn in tokens]
corpus = [self.dictionary.doc2bow(tkn) for tkn in tokens]
self.dominant_topic_ids = self.getDominantTopics(corpus)
self.sentence_groups = self.splitIntoSentences(documents)
self.distributions = self.getSentenceDistributions()
self.summary_data = self.sentenceSelection(verbose=False)
def summarize(self, documents):
tokens = [tokenizer(document) for document in documents]
tokens = [self.bigramizer[tkn] for tkn in tokens]
corpus = [self.dictionary.doc2bow(tkn) for tkn in tokens]
self.dominant_topic_ids = self.getDominantTopics(corpus)
self.sentence_groups = self.splitIntoSentences(documents)
self.distributions = self.getSentenceDistributions()
self.summary_data = self.sentenceSelection(verbose=False)