def test_order_information(self):
"""
Test ordering Sentences by MEAD score
:return:
"""
doc_id_1 = 'TST_ENG_20190101.0001'
sentence_1 = 'Puppies love playing fetch.'
sentence_2 = 'They all ran around with their tails wagging ' \
'and their tongues hanging out having loads of fun in the sun.'
sentence_3 = "He loves playing so he liked to run around with the other dogs playing fetch."
expected_info = [
Sentence(sentence_1, 1, doc_id_1),
Sentence(sentence_3, 3, doc_id_1),
Sentence(sentence_2, 2, doc_id_1)
]
WordMap.word_set = self.w_set
WordMap.create_mapping()
Vectors().create_freq_vectors(self.topics)
generator = MeadSummaryGenerator(self.doc_list, MeadContentSelector(),
self.args)
generator.select_content(self.idf)
generator.order_information()
first_sentences = generator.content_selector.selected_content[:3]
self.assertListEqual(expected_info, first_sentences)
def test_realize_content(self):
"""
Test applying redundancy penalty during realize_content
:return:
"""
expected_content = "I took my small puppy to the dog park today.\n" \
"In a park somewhere, a bunch of puppies played fetch with their owners today.\n" \
"There were many bigger puppies but he didn't get in a fight with any of them, " \
"they just played together with their toys and chased each other.\n" \
"They all ran around with their tails wagging and their tongues hanging out having " \
"loads of fun in the sun.\n" \
"He loves playing so he liked to run around with the other dogs playing fetch.\n" \
"Puppies love playing fetch."
WordMap.word_set = self.w_set
WordMap.create_mapping()
Vectors().create_freq_vectors(self.topics)
generator = MeadSummaryGenerator(self.doc_list, MeadContentSelector(),
self.args)
generator.select_content(self.idf)
generator.order_information()
generator.content_selector.selected_content = generator.content_selector.selected_content
realized_content = generator.realize_content()
self.assertEqual(expected_content, realized_content)
def test_generate_summary(self):
topics = {
'PUP1A': [
Document('TST_ENG_20190101.0001'),
Document('TST_ENG_20190101.0002'),
Document('TST20190201.0001'),
Document('TST20190201.0002')
],
'WAR2A': [
Document('TST_ENG_20190301.0001'),
Document('TST_ENG_20190301.0002'),
Document('TST20190401.0001'),
Document('TST20190401.0002')
]
}
WordMap.create_mapping()
vec = Vectors()
vec.create_freq_vectors(topics)
idf = MeadSummaryGenerator(self.doc_list, MeadContentSelector(),
self.args).get_idf_array()
for topic_id, documents in topics.items():
summarizer = MeadSummaryGenerator(documents, MeadContentSelector(),
self.args)
summary = summarizer.generate_summary(idf)
self.assertIsNot(summary, None)
def test_sentence_vector(self):
s = self.topics.get(1)[1].sens[1] # s1 is a Sentence object
# s text: 'He loves playing so he liked to run around with the other dogs playing fetch.'
id_of_playing = WordMap.id_of('playing')
self.assertEqual(s.vector.getcol(id_of_playing).sum(), 1)
for word in s.tokens:
id_of_word = WordMap.id_of(word)
self.assertGreater(s.vector.getcol(id_of_word).sum(), 0)
def test_melda_generate_summary(self):
WordMap.word_set = self.w_set
WordMap.create_mapping()
Vectors().create_freq_vectors(self.topics)
Vectors().create_term_doc_freq(self.topics)
for topic_id, documents in self.topics.items():
summarizer = MeldaSummaryGenerator(documents,
MeldaContentSelector(),
self.args)
summary = summarizer.generate_summary(self.idf)
self.assertIsNot(summary, None)
def test_document_topics(self):
WordMap.word_set = self.w_set
WordMap.create_mapping()
Vectors().create_term_doc_freq(self.topics)
selector = MeldaContentSelector()
lda_model = selector.build_lda_model(self.doc_list, self.args.lda_topics)
testtok = ['puppy', 'soldier', 'war', 'fetch']
testsen = Vectors().create_term_sen_freq(testtok)
document_topics = lda_model.get_document_topics(testsen, minimum_probability=0)
topic_dist = [prob[1] for prob in document_topics]
self.assertEqual(len(topic_dist), self.args.lda_topics)
self.assertAlmostEquals(sum(topic_dist), 1, 2)
def test_get_lda_scores(self):
WordMap.word_set = self.w_set
WordMap.create_mapping()
Vectors().create_term_doc_freq(self.topics)
selector = MeldaContentSelector()
lda_model = selector.build_lda_model(self.doc_list, self.args.lda_topics)
sentence = self.doc_list[0].sens[0]
selector.calculate_lda_scores([sentence], lda_model)
lda_scores = sentence.lda_scores
self.assertEqual(len(lda_scores), self.args.lda_topics)
self.assertAlmostEqual(sum(lda_scores), 1, 2)
def test_get_top_n(self):
WordMap.word_set = self.w_set
WordMap.create_mapping()
Vectors().create_freq_vectors(self.topics)
Vectors().create_term_doc_freq(self.topics)
selector = MeldaContentSelector()
lda_model = selector.build_lda_model(self.doc_list, self.args.lda_topics)
sentences = selector.calculate_mead_scores(self.doc_list, self.args, self.idf)
sentences = selector.calculate_lda_scores(sentences, lda_model)
sentences = selector.calculate_melda_scores(sentences)
selector.select_top_n(sentences, self.args.lda_topics, 1)
self.assertEqual(len(selector.selected_content), self.args.lda_topics)
def test_melda_info_ordering(self):
WordMap.word_set = self.w_set
WordMap.create_mapping()
Vectors().create_freq_vectors(self.topics)
Vectors().create_term_doc_freq(self.topics)
summarizer = MeldaSummaryGenerator(self.doc_list,
MeldaContentSelector(), self.args)
content_selector = summarizer.select_content(self.idf)
expected_len = len(content_selector)
summarizer.order_information()
actual_len = len(content_selector)
self.assertEqual(expected_len, actual_len)
def test_create_mapping(self):
Preprocessor.load_models()
WordMap.word_set = set()
WordMap.word_to_id = {}
Document("TST_ENG_20190101.0001")
Document("TST_ENG_20190101.0002")
WordMap.create_mapping()
mapping = WordMap.get_mapping()
self.assertCountEqual(self.word_set, mapping.keys()) # each word in word_set got added to the dictionary
self.assertEqual(len(mapping), len(set(mapping.items()))) # each id value in the dict is unique
def test_term_topics(self):
WordMap.word_set = self.w_set
WordMap.create_mapping()
Vectors().create_term_doc_freq(self.topics)
selector = MeldaContentSelector()
lda_model = selector.build_lda_model(self.doc_list, self.args.lda_topics)
puppy_topics = lda_model.get_term_topics(WordMap.id_of('puppy'), minimum_probability=0)
war_topics = lda_model.get_term_topics(WordMap.id_of('war'), minimum_probability=0)
puppy_dist = [prob[1] for prob in puppy_topics]
enemy_dist = [prob[1] for prob in war_topics]
puppy_war = puppy_dist[0] > enemy_dist[0] and puppy_dist[1] < enemy_dist[1]
war_puppy = puppy_dist[0] < enemy_dist[0] and puppy_dist[1] > enemy_dist[1]
self.assertTrue(puppy_war or war_puppy)
def get_idf_array(self):
"""
Use external corpus to get IDF scores
for cluster centroid calculations
:return: numpy array of idf values
"""
corpus = brown
if self.args.corpus == 'R':
corpus = reuters
num_words = Vectors().num_unique_words
n = len(corpus.fileids()) # number of documents in corpus
docs_word_matrix = np.zeros([n, num_words])
for doc_idx, doc_id in enumerate(corpus.fileids()):
sentences = list(corpus.sents(doc_id))
words_in_doc = set()
for s in sentences:
s = ' '.join(s)
proc_s = Preprocessor.get_processed_tokens(Preprocessor.get_processed_sentence(s))
if proc_s:
words_in_doc = words_in_doc.union(proc_s)
for word in words_in_doc:
word_idx = WordMap.id_of(word)
if word_idx:
docs_word_matrix[doc_idx, word_idx] = 1
docs_per_word = np.sum(docs_word_matrix, axis=0)
self.idf_array = np.log10(np.divide(n, docs_per_word + 1)) # add one to avoid divide by zero error
return self.idf_array
def create_term_doc_freq(self, topics):
"""
create term freq on each doc over each topic
:param topics:
:return: set tdf to topic_list of [doc_list of (wordid, freq)]
e.g.,[[(0, 1), (1, 2), (2, 1), (3, 1), (4, 1), (6, 5), (9, 2), (10, 1), (11, 1)...],[...]]
"""
for cluster in topics.values():
for document in cluster:
term_doc_freq_dict = {}
for sentence in document.sens:
for word in sentence.tokenized():
word_id = WordMap.id_of(word)
if word_id is None:
warnings.warn('Word \'' + word + '\' not in WordMap', Warning)
continue
if word_id not in term_doc_freq_dict:
term_doc_freq_dict[word_id] = 0
term_doc_freq_dict[word_id] += 1
term_doc_freq_list = []
for word_id in sorted(term_doc_freq_dict):
term_doc_freq_list.append((word_id, term_doc_freq_dict[word_id]))
document.set_tdf(term_doc_freq_list)
def create_freq_vectors(self, topics):
"""
creates a frequency vector for each sentence in each document in each topic in topics; stores single vectors in
relevant Sentence objects and per-document matrices in relevant Document objects
:param topics: Dictionary {topic -> list of Documents}
:return: None
pre: WordMap.create_mapping has been called (should happen in run_summarization document loading)
"""
for cluster in topics.values():
for document in cluster:
doc_vectors = dok_matrix((0, self.num_unique_words))
for sentence in document.sens:
sentence_vector = dok_matrix((1, self.num_unique_words))
for word in sentence.tokenized(): # maybe check that sentence.tokenized() is the right thing here
word_id = WordMap.id_of(word)
if word_id is None:
warnings.warn('Word \'' + word + '\' not in WordMap', Warning)
warnings.warn('Sentence:' + sentence.raw_sentence, Warning)
else:
sentence_vector[0, word_id] += 1
# assign vector to sentence object
sentence.set_vector(sentence_vector)
# add sentence vector to document matrix
doc_vectors = vstack([doc_vectors, sentence_vector])
# assign matrix to document
document.set_vectors(doc_vectors)
def load_documents_for_topics(topic_soup):
"""
Load documents for each topic
:param topic_soup:
:return:
"""
topics = {}
for topic in topic_soup.find_all('topic'):
documents = load_documents(topic)
topics[topic['id']] = documents
# At this point, all docs have been loaded and all unique words are stored in WordMap set
# Need to trigger creation of mapping and of vectors
WordMap.create_mapping()
vec = Vectors()
vec.create_freq_vectors(topics) # do we need to have this here if we don't run mead based content selection
vec.create_term_doc_freq(topics)
return topics
def get_centroid_score(self, sentence, centroid):
"""
Get the centroid score for this sentence
:param: sentence, centroid
:return: float
"""
centroid_score = 0
for word in sentence.tokens:
id = WordMap.id_of(word)
centroid_score += centroid[id] if id is not None else 0
# return centroid_score/(sentence.word_count() + 1)
return centroid_score
class VectorsTests(unittest.TestCase):
Preprocessor.load_models()
topics = {
1:
[Document('TST_ENG_20190101.0001'),
Document('TST_ENG_20190101.0002')]
}
WordMap.create_mapping()
mapping = WordMap.get_mapping()
topic_one = topics.get(1) # list of Documents
def test_create_freq_vectors(self):
Vectors().create_freq_vectors(self.topics)
for doc_list in self.topics.values():
for doc in doc_list:
# check that there's a vector for each sentence
doc_matrix_shape = doc.vectors.get_shape()
expected_rows = 3
self.assertEqual(doc_matrix_shape[0], expected_rows)
def test_sentence_vector(self):
s = self.topics.get(1)[1].sens[1] # s1 is a Sentence object
# s text: 'He loves playing so he liked to run around with the other dogs playing fetch.'
id_of_playing = WordMap.id_of('playing')
self.assertEqual(s.vector.getcol(id_of_playing).sum(), 1)
for word in s.tokens:
id_of_word = WordMap.id_of(word)
self.assertGreater(s.vector.getcol(id_of_word).sum(), 0)
def test_get_topic_matrix(self):
# make sure all sentences from all topic docs make it into the matrix
topic_one_matrix = Vectors().get_topic_matrix(self.topic_one)
expected_num_sentences = 6
self.assertEqual(expected_num_sentences,
topic_one_matrix.get_shape()[0])
def test_mead_summary_length(self):
"""
Test length of summary is less than 100 words
:return:
"""
topics = {
'PUP1A': [
Document('TST_ENG_20190101.0001'),
Document('TST_ENG_20190101.0002'),
Document('TST20190201.0001'),
Document('TST20190201.0002')
],
'WAR2A': [
Document('TST_ENG_20190301.0001'),
Document('TST_ENG_20190301.0002'),
Document('TST20190401.0001'),
Document('TST20190401.0002')
]
}
WordMap.create_mapping()
vec = Vectors()
vec.create_freq_vectors(topics)
idf = MeadSummaryGenerator(self.doc_list, MeadContentSelector(),
self.args).get_idf_array()
max_length = 100
for topic_id, documents in topics.items():
generator = MeadSummaryGenerator(documents, MeadContentSelector(),
self.args)
generator.select_content(idf)
generator.order_information()
realized_content = generator.realize_content()
realized_content = [
w for w in realized_content.split(" ") if not " "
]
content_length = len(realized_content)
self.assertLessEqual(content_length, max_length)
def __init__(self, raw_sentence, sent_pos, doc_id=None):
"""
initialize Sentence class with methods for plain/raw and tokenized sentence
options, word count, position of sentence in document and document id
:param raw_sentence:
:param sent_pos:
"""
self.raw_sentence = ' '.join(raw_sentence.rstrip().split())
self.raw_sentence = Preprocessor.strip_beginning(self.raw_sentence)
self.tokens = []
self.processed = Preprocessor.get_processed_sentence(self.raw_sentence)
self.__tokenize_sentence(self.processed)
self.sent_pos = int(sent_pos) # position of sentence in document
self.doc_id = doc_id
self.vector = [] # placeholder
self.order_by = self.sent_pos
self.c_score = self.p_score = self.f_score = self.mead_score = self.lda_scores = self.melda_scores = None
self.compressed = self.raw_sentence
# update global mapping of words to indices
WordMap.add_words(
self.tokens) # make sure self.tokens is the right thing here
def test_get_idf_array(self):
words = [
"i", "eat", "cake", "is", "delicious", "puppies", "are", "cute",
"cats", "furry", "bank", "company", "sugar", "dollar", "however",
"say"
]
# Must override WordMap dictionary for test
WordMap.word_to_id = {
'delicious': 0,
'eat': 1,
'furry': 2,
'puppies': 3,
'i': 4,
'cats': 5,
'are': 6,
'is': 7,
'cute': 8,
'cake': 9,
'bank': 10,
'company': 11,
'sugar': 12,
'dollar': 13,
'however': 14,
'say': 15
}
idf = MeadSummaryGenerator(self.doc_list, MeadContentSelector(),
self.args).get_idf_array()
scores = []
for word in words:
curr_score = idf[WordMap.id_of(word)]
scores.append("{:.5f}".format(curr_score))
expected_scores = [
'2.69897', '0.80688', '1.49485', '2.69897', '2.69897', '2.69897',
'2.69897', '1.92082', '2.69897', '2.69897', '1.04576', '0.65365',
'1.44370', '0.98297', '0.24718', '0.10018'
]
self.assertListEqual(scores, expected_scores, 5)
def create_term_sen_freq(self, sen):
"""
create term freq on a tokenized sentence
:param sent:
:return:
"""
term_doc_freq_dict = {}
for tok in sen:
word_id = WordMap.id_of(tok)
if word_id is None:
warnings.warn('Word \'' + tok + '\' not in WordMap', Warning)
continue
else:
if word_id not in term_doc_freq_dict:
term_doc_freq_dict[word_id] = 0
term_doc_freq_dict[word_id] += 1
term_doc_freq_list = []
for word_id in sorted(term_doc_freq_dict):
term_doc_freq_list.append((word_id, term_doc_freq_dict[word_id]))
return term_doc_freq_list
def build_lda_model(self, documents, num_topics):
"""
Build the LDA model
:param documents: the list of documents
:param num_topics: the number of topics to use
:return: the LDA model
"""
topic_tdf = []
for doc in documents:
topic_tdf.append(doc.tdf)
lda_model = gensim.models.ldamodel.LdaModel(
corpus=topic_tdf,
id2word=WordMap.get_id2word_mapping(),
num_topics=num_topics,
random_state=0,
update_every=1,
chunksize=100,
passes=10,
alpha='auto',
per_word_topics=True)
return lda_model
def __init__(self):
self.num_unique_words = len(WordMap.get_mapping())
class MeadSummaryGeneratorTests(unittest.TestCase):
"""
Tests for MeadSummaryGenerator
"""
# variables used in multiple tests
Preprocessor.load_models()
doc_1 = Document("TST_ENG_20190101.0001")
doc_2 = Document("TST_ENG_20190101.0002")
doc_list = [doc_1, doc_2]
topics = {'PUP1A': [doc_1, doc_2]}
w_set = {
'he', 'owner', 'i', 'play', 'big', 'chase', 'fetch', 'park', 'dog',
'fun', 'toy', 'tongue', 'take', 'ran', 'in', 'sun', 'love',
'somewhere', 'many', 'together', 'around', 'puppy', 'today', 'load',
'fight', 'small', "n't", '-PRON-', 'wag', 'hang', 'loads', 'bunch',
'get', 'playing', 'they', 'like', 'tail', 'run', 'there'
}
idf = [
4.032940937780854, 2.420157081061118, 1.3730247377110034,
2.8868129021026157, 2.7776684326775474, 3.7319109421168726,
3.25478968739721, 2.7107216430469343, 3.7319109421168726,
4.032940937780854, 3.3339709334448346, 4.032940937780854,
1.9257309681329853, 2.5705429398818973, 0.21458305982249878,
2.3608430798451363, 3.5558196830611912, 3.3339709334448346,
1.5660733174267443, 2.024340766018936, 1.2476111027700865,
4.032940937780854, 0.9959130580250786, 3.7319109421168726,
2.5415792439465807, 1.7107216430469343, 4.032940937780854,
3.4308809464528913, 4.032940937780854, 3.4308809464528913,
3.5558196830611912, 3.5558196830611912, 4.032940937780854,
1.734087861371147, 3.0786984283415286, 0.9055121599292547,
3.5558196830611912, 3.5558196830611912, 1.9876179589941962
]
args = parse_args(['test_data/test_topics.xml', 'test'])
WordMap.reset()
def test_order_information(self):
"""
Test ordering Sentences by MEAD score
:return:
"""
doc_id_1 = 'TST_ENG_20190101.0001'
sentence_1 = 'Puppies love playing fetch.'
sentence_2 = 'They all ran around with their tails wagging ' \
'and their tongues hanging out having loads of fun in the sun.'
sentence_3 = "He loves playing so he liked to run around with the other dogs playing fetch."
expected_info = [
Sentence(sentence_1, 1, doc_id_1),
Sentence(sentence_3, 3, doc_id_1),
Sentence(sentence_2, 2, doc_id_1)
]
WordMap.word_set = self.w_set
WordMap.create_mapping()
Vectors().create_freq_vectors(self.topics)
generator = MeadSummaryGenerator(self.doc_list, MeadContentSelector(),
self.args)
generator.select_content(self.idf)
generator.order_information()
first_sentences = generator.content_selector.selected_content[:3]
self.assertListEqual(expected_info, first_sentences)
def test_realize_content(self):
"""
Test applying redundancy penalty during realize_content
:return:
"""
expected_content = "I took my small puppy to the dog park today.\n" \
"In a park somewhere, a bunch of puppies played fetch with their owners today.\n" \
"There were many bigger puppies but he didn't get in a fight with any of them, " \
"they just played together with their toys and chased each other.\n" \
"They all ran around with their tails wagging and their tongues hanging out having " \
"loads of fun in the sun.\n" \
"He loves playing so he liked to run around with the other dogs playing fetch.\n" \
"Puppies love playing fetch."
WordMap.word_set = self.w_set
WordMap.create_mapping()
Vectors().create_freq_vectors(self.topics)
generator = MeadSummaryGenerator(self.doc_list, MeadContentSelector(),
self.args)
generator.select_content(self.idf)
generator.order_information()
generator.content_selector.selected_content = generator.content_selector.selected_content
realized_content = generator.realize_content()
self.assertEqual(expected_content, realized_content)
def test_get_idf_array(self):
words = [
"i", "eat", "cake", "is", "delicious", "puppies", "are", "cute",
"cats", "furry", "bank", "company", "sugar", "dollar", "however",
"say"
]
# Must override WordMap dictionary for test
WordMap.word_to_id = {
'delicious': 0,
'eat': 1,
'furry': 2,
'puppies': 3,
'i': 4,
'cats': 5,
'are': 6,
'is': 7,
'cute': 8,
'cake': 9,
'bank': 10,
'company': 11,
'sugar': 12,
'dollar': 13,
'however': 14,
'say': 15
}
idf = MeadSummaryGenerator(self.doc_list, MeadContentSelector(),
self.args).get_idf_array()
scores = []
for word in words:
curr_score = idf[WordMap.id_of(word)]
scores.append("{:.5f}".format(curr_score))
expected_scores = [
'2.69897', '0.80688', '1.49485', '2.69897', '2.69897', '2.69897',
'2.69897', '1.92082', '2.69897', '2.69897', '1.04576', '0.65365',
'1.44370', '0.98297', '0.24718', '0.10018'
]
self.assertListEqual(scores, expected_scores, 5)
def test_mead_summary_length(self):
"""
Test length of summary is less than 100 words
:return:
"""
topics = {
'PUP1A': [
Document('TST_ENG_20190101.0001'),
Document('TST_ENG_20190101.0002'),
Document('TST20190201.0001'),
Document('TST20190201.0002')
],
'WAR2A': [
Document('TST_ENG_20190301.0001'),
Document('TST_ENG_20190301.0002'),
Document('TST20190401.0001'),
Document('TST20190401.0002')
]
}
WordMap.create_mapping()
vec = Vectors()
vec.create_freq_vectors(topics)
idf = MeadSummaryGenerator(self.doc_list, MeadContentSelector(),
self.args).get_idf_array()
max_length = 100
for topic_id, documents in topics.items():
generator = MeadSummaryGenerator(documents, MeadContentSelector(),
self.args)
generator.select_content(idf)
generator.order_information()
realized_content = generator.realize_content()
realized_content = [
w for w in realized_content.split(" ") if not " "
]
content_length = len(realized_content)
self.assertLessEqual(content_length, max_length)
def test_generate_summary(self):
topics = {
'PUP1A': [
Document('TST_ENG_20190101.0001'),
Document('TST_ENG_20190101.0002'),
Document('TST20190201.0001'),
Document('TST20190201.0002')
],
'WAR2A': [
Document('TST_ENG_20190301.0001'),
Document('TST_ENG_20190301.0002'),
Document('TST20190401.0001'),
Document('TST20190401.0002')
]
}
WordMap.create_mapping()
vec = Vectors()
vec.create_freq_vectors(topics)
idf = MeadSummaryGenerator(self.doc_list, MeadContentSelector(),
self.args).get_idf_array()
for topic_id, documents in topics.items():
summarizer = MeadSummaryGenerator(documents, MeadContentSelector(),
self.args)
summary = summarizer.generate_summary(idf)
self.assertIsNot(summary, None)