def summarizeLSA(text):
parser_string = PlaintextParser.from_string(text, Tokenizer('english'))
from sumy.summarizers.lsa import LsaSummarizer
print('\nLSA')
summarizer = LsaSummarizer()
# Summarize the document with 2 sentences
sentences = summarizer(parser_string.document, 2)
#print('\nSummarizing text from string:')
summary = ''
for sent in sentences:
summary += str(sent)
summary += '\n'
return summary
def test_dictionary_without_stop_words(self):
summarizer = LsaSummarizer()
summarizer.stop_words = ["stop", "Halt", "SHUT", "HmMm"]
document = build_document(
("stop halt shut hmmm", "Stop Halt Shut Hmmm",),
("StOp HaLt ShUt HmMm", "STOP HALT SHUT HMMM",),
("Some relevant sentence", "Some moRe releVant sentEnce",),
)
expected = frozenset(["some", "more", "relevant", "sentence"])
dictionary = summarizer._create_dictionary(document)
self.assertEqual(expected, frozenset(dictionary.keys()))
def test_issue_5_svd_converges(self):
"""Source: https://github.com/miso-belica/sumy/issues/5"""
pytest.skip("Can't reproduce the issue.")
parser = PlaintextParser.from_string(
load_resource("articles/svd_converges.txt"),
Tokenizer("english")
)
summarizer = LsaSummarizer(Stemmer("english"))
summarizer.stop_words = get_stop_words("english")
sentences = summarizer(parser.document, 20)
self.assertEqual(len(sentences), 20)
def lsaReferenceSummary(path): sentencesList=[] parser = PlaintextParser.from_file(path, Tokenizer(LANGUAGE)) stemmer = Stemmer(LANGUAGE) summarizer = LsaSummarizer(stemmer) summarizer.stop_words = get_stop_words(LANGUAGE) for sentence in summarizer(parser.document, SENTENCES_COUNT): #print(sentence._text) sentencesList.append(sentence._text) return sentencesList
def test_single_sentence(self):
document = build_document(("I am the sentence you like", ))
summarizer = LsaSummarizer()
summarizer.stopwords = (
"I",
"am",
"the",
)
sentences = summarizer(document, 10)
self.assertEqual(len(sentences), 1)
self.assertEqual(to_unicode(sentences[0]),
"I am the sentence you like")
def summarize_text(text):
language = "english"
# Create a parser from the string
parser = PlaintextParser.from_string(text, Tokenizer(language))
summarizer = LsaSummarizer(Stemmer(language))
summarizer.stop_words = sumy.utils.get_stop_words(language)
summary_text = ""
for sentence in summarizer(parser.document, 5):
summary_text += str(sentence) + " "
return summary_text
def summarizeLSA():
from sumy.summarizers.lsa import LsaSummarizer
print('\nLSA')
summarizer = LsaSummarizer()
# Summarize the document with 2 sentences
summary = summarizer(parser_string.document, 2)
print('\nSummarizing text from string:')
for sent in summary:
print(sent)
print('\nSummarizing text from .txt file:')
summary = summarizer(parser_file.document, 2)
for sent in summary:
print(sent)
def test_document(self):
document = build_document(
("I am the sentence you like", "Do you like me too",),
("This sentence is better than that above", "Are you kidding me",)
)
summarizer = LsaSummarizer()
summarizer.stopwords = (
"I", "am", "the", "you", "are", "me", "is", "than", "that", "this",
)
sentences = summarizer(document, 2)
self.assertEqual(len(sentences), 2)
self.assertEqual(to_unicode(sentences[0]), "I am the sentence you like")
self.assertEqual(to_unicode(sentences[1]), "This sentence is better than that above")
def lsa_summarize(original_text):
from sumy.summarizers.lsa import LsaSummarizer
from sumy.nlp.tokenizers import Tokenizer
from sumy.parsers.plaintext import PlaintextParser
parser = PlaintextParser.from_string(original_text, Tokenizer('english'))
lsa_summarizer = LsaSummarizer()
lsa_summary = lsa_summarizer(parser.document, 5)
# Printing the summary
for sentence in lsa_summary:
print(sentence)
def summarize(srt_file, n_sentences, language="english"):
# generate segmented summary
parser = PlaintextParser.from_string(srt_to_txt(srt_file),
Tokenizer(language))
stemmer = Stemmer(language)
summarizer = LsaSummarizer(stemmer)
summarizer.stop_words = get_stop_words(language)
segment = []
for sentence in summarizer(parser.document, n_sentences):
index = int(re.findall("\(([0-9]+)\)", str(sentence))[0])
item = srt_file[index]
segment.append(srt_segment_to_range(item))
print(segment)
return segment
def LSA(rsc_file, dst_file, count):
language = "chinese"
parser = PlaintextParser.from_file(rsc_file,
Tokenizer(language),
encoding='utf-8')
stemmer = Stemmer(language) # 语言容器
summarizer = LsaSummarizer(stemmer) # LSA算法
summarizer.stop_words = get_stop_words(language)
with open(dst_file, 'w', encoding='utf-8') as f:
for sentence in summarizer(parser.document, count):
f.write(str(sentence))
f.write('\n')
print(sentence)
def evaluate_summary(file_name, input_dir, sent_count, lingua_franca_summary, show_summaries):
method_name = inspect.stack()[0][3]
try:
process_logger.debug("in "+ method_name +" method")
file_model_summary = open(input_dir + file_name +".model", "r")
model_summary = file_model_summary.read()
rouge_scores_dict = {}
rouge_scores = rouge_evaluation(lingua_franca_summary, model_summary)
rouge_scores_dict[">>LINGUA FRANCA"] = rouge_scores
file_summary = open("Test System Summary/" + file_name + "-" + "LINGUA FRANCA" + ".txt", "w")
file_summary.write(lingua_franca_summary)
LANGUAGE = "english"
parser = PlaintextParser.from_file(input_dir + file_name + ".txt", Tokenizer(LANGUAGE))
stemmer = Stemmer(LANGUAGE)
lsa_summarizer = LsaSummarizer(stemmer)
rouge_scores = sumy_summarizers("LSA", lsa_summarizer, parser.document, sent_count, model_summary, show_summaries, file_name)
rouge_scores_dict["LSA"] = rouge_scores
lex_summarizer = LexRankSummarizer(stemmer)
rouge_scores = sumy_summarizers("LEX RANK", lex_summarizer, parser.document, sent_count, model_summary, show_summaries, file_name)
rouge_scores_dict["LEX RANK"] = rouge_scores
luhn_summarizer = LuhnSummarizer(stemmer)
rouge_scores = sumy_summarizers("LUHN", luhn_summarizer, parser.document, sent_count, model_summary, show_summaries, file_name)
rouge_scores_dict["LUHN"] = rouge_scores
text_rank_summarizer = TextRankSummarizer(stemmer)
rouge_scores = sumy_summarizers("TEXT RANK", text_rank_summarizer, parser.document, sent_count, model_summary, show_summaries, file_name)
rouge_scores_dict["TEXT RANK"] = rouge_scores
sum_basic_summarizer = SumBasicSummarizer(stemmer)
rouge_scores = sumy_summarizers("SUM BASIC", sum_basic_summarizer, parser.document, sent_count, model_summary, show_summaries, file_name)
rouge_scores_dict["SUM BASIC"] = rouge_scores
kl_summarizer = KLSummarizer(stemmer)
rouge_scores = sumy_summarizers("KL SUM", kl_summarizer, parser.document, sent_count, model_summary, show_summaries, file_name)
rouge_scores_dict["KL SUM"] = rouge_scores
# score_reader(rouge_scores_dict)
df_rouge, summarizer_list = process_rouge_scores(rouge_scores_dict)
return df_rouge, summarizer_list
except Exception as Ex:
error_logger.error("Exception occurred in " + method_name + "| Exception:" + str(Ex))
return None
def summarize(self, paragraphs):
"""Summarize content with the sumy library"""
content = '\n\n'.join(paragraphs)
stemmer = Stemmer(LANGUAGE)
summarizer = LsaSummarizer(stemmer)
tokenizer = Tokenizer(LANGUAGE)
parser = PlaintextParser.from_string(content, tokenizer)
sentences = []
for sentence in summarizer(parser.document, '10%'):
sentences.append([str(sentence)])
return sentences
def slititle(sena):
# Creates a temporary text file since sumy works that way
with open('Titlefile___.txt', 'w') as tefil:
tefil.write(sena)
thefile = "Titlefile___.txt" #name of the plain-text file
parser = PlaintextParser.from_file(thefile, Tokenizer("english"))
summarizer = LsaSummarizer()
summary = summarizer(parser.document,
1) # Reduce the document to 1 sentence
os.remove('Titlefile___.txt')
return str(summary[0]).split('.')[0]
def build_summary_from_paper(paper_object: Paper, sections_selection=None) -> str:
"""
This function summarize texts corresponding to selected sections in a paper instance
:param paper_object: An instance of class Paper
:param sections_selection: List of section's to summarize
:return: A string containing article's summary
"""
paper_text = paper_object.get_sections_texts_str(sections_selection)
parser = PlaintextParser.from_string(paper_text, Tokenizer(LANG))
summarizer = LsaSummarizer()
summary_sentences = summarizer(parser.document, SENTENCES_COUNT)
summary = ' '.join(list(map(str, summary_sentences)))
return summary
def perform_sumy_summarization(data, gold_standard_data):
stemmer = Stemmer(LANGUAGE)
summarizers = [LsaSummarizer(stemmer), TextRankSummarizer(stemmer), LexRankSummarizer(stemmer)]
summarizer_names = ["Lsa", "TextRank", "LexRank"]
# print "SUMY Scores: "
# Read each sentence from 'data' and create a summary of it
for line, gs_line in zip(data, gold_standard_data):
# Only consider the content part of the text. Changed it from unicode to normal string
# summarized_text = line["content"].encode('ascii', 'ignore')
summarized_text = line["content"]
if gs_line == "\n":
continue
gold_standard = gs_line
# Read line by line instead of reading the entire file
parser = PlaintextParser.from_string(summarized_text, Tokenizer(LANGUAGE))
idx = 0
for summarizer in summarizers:
# Store the scores in a dictionary
output_scores[line["index"]] = []
sumy_dict_key = str(line["index"]) + summarizer_names[idx]
sumy_scores_dict[sumy_dict_key] = []
summarizer.stop_words = get_stop_words(LANGUAGE)
# print "SUMY with", summarizer
for sentence in summarizer(parser.document, SENTENCES_COUNT):
if line["index"] in output_scores:
output_scores[line["index"]] = []
if sumy_dict_key in sumy_scores_dict[sumy_dict_key]:
continue
# Store output in a dictionary in the form of a key-value pair
# Example --> 1: 'with the exception of the elderly and the youth'
output_scores[int(line["index"])].append({"sumy_rouge_unigrams":
calculate_rouge_n_score(line["index"], gold_standard, str(sentence), 1)})
output_scores[int(line["index"])].append({"sumy_rouge_bigrams":
calculate_rouge_n_score(line["index"], gold_standard, str(sentence), 2)})
output_scores[int(line["index"])].append({"sumy_rouge_l":
calculate_rouge_l_score(line["index"], gold_standard, str(sentence))})
output_scores[int(line["index"])].append({"sumy_rouge_s":
calculate_rouge_s_score(line["index"], gold_standard,str(sentence), 2)})
sumy_scores_dict[sumy_dict_key].append(calculate_rouge_n_score(line["index"], gold_standard, str(sentence), 1))
sumy_scores_dict[sumy_dict_key].append(calculate_rouge_n_score(line["index"], gold_standard, str(sentence), 2))
sumy_scores_dict[sumy_dict_key].append(calculate_rouge_l_score(line["index"], gold_standard, str(sentence)))
sumy_scores_dict[sumy_dict_key].append(calculate_rouge_s_score(line["index"], gold_standard,str(sentence), 2))
idx += 1
def test_real_example(self):
"""Source: http://www.prevko.cz/dite/skutecne-pribehy-deti"""
parser = PlaintextParser.from_string(
load_resource("snippets/prevko.txt"), Tokenizer("czech"))
summarizer = LsaSummarizer(Stemmer("czech"))
summarizer.stop_words = get_stop_words("czech")
sentences = summarizer(parser.document, 2)
self.assertEqual(len(sentences), 2)
self.assertEqual(
to_unicode(sentences[0]),
"Jednalo se o případ chlapce v 6. třídě, který měl problémy s učením."
)
self.assertEqual(
to_unicode(sentences[1]),
"Nedopadl bohužel dobře a tak musel opakovat 6. třídu, "
"což se chlapci ani trochu nelíbilo.")
def summarization(text, alg="lexrank"):
parser = PlaintextParser.from_string("".join(text), Tokenizer("japanese"))
if alg == "lexrank":
summarizer = LexRankSummarizer()
elif alg == "textrank":
summarizer = TextRankSummarizer()
elif alg == "lsa":
summarizer = LsaSummarizer()
else:
raise Exception("IllegalArgumentException")
summarizer.stop_words = [" "]
abst = summarizer(document=parser.document, sentences_count=5)
abst = [x for x in map(lambda x: "".join(x.words), abst)]
return abst
def get_algorithm(algorithm_name):
print('parameter:')
print(algorithm_name)
summarizer = None
if algorithm_name == algorithms.LAS.value:
summarizer = LsaSummarizer()
print(algorithms.LAS.value)
elif algorithm_name == algorithms.LexRank.value:
summarizer = LexRankSummarizer()
print(algorithms.LexRank.value)
elif algorithm_name == algorithms.Luhn.value:
summarizer = LuhnSummarizer()
print(algorithms.Luhn.value)
else:
summarizer = TextRankSummarizer()
print(algorithms.TextRank.value)
return summarizer
def summarize_sumy(doc, case):
summary = ""
file_doc = open("temp.txt", "w", encoding='utf-8')
file_doc.write(doc)
file_doc.close()
parser = PlaintextParser.from_file("temp.txt", Tokenizer(LANGUAGE))
stemmer = Stemmer(LANGUAGE)
if case == 1:
summarizer = LexRankSummarizer(stemmer)
else:
summarizer = LsaSummarizer(stemmer)
summarizer.stop_words = get_stop_words(LANGUAGE)
for sentence in summarizer(parser.document, SENTENCES_COUNT):
summary += str(sentence) + ' '
return summary
def create_lsa_summary():
for filename in os.listdir(TREES_SOURCE_PATH):
if filename.endswith('.xml'):
name = re.search('topic_(.*)\.xml', filename)
input_path = create_input(name.group(1))
with open(SUM_PATH + name.group(1) + '_LSA_Group5.txt',
'w',
encoding='utf8') as summary:
summary.write("====================== General Summary of " +
name.group(1) + " ======================\n")
parser = PlaintextParser.from_file(input_path,
Tokenizer("english"))
summarizer = LsaSummarizer()
s = summarizer(parser.document, 4)
for sentence in s:
summary.write(str(sentence) + '\n')
def build_summary_from_papers_list(papers_list: list, sections_to_summarize: list) -> str:
"""
Merge the texts corresponding to sections selected for generate a summary from different papers
:param papers_list: A list of papers instances
:param sections_to_summarize: A list of sections to summarize
:return: A str object corresponding to selected sections texts summary
"""
sections_texts = list(
map(lambda paper: paper.get_sections_texts_str(sections_to_summarize), parse_papers_list(papers_list))
)
text = ' '.join(sections_texts)
parser = PlaintextParser.from_string(text, Tokenizer(LANG))
summarizer = LsaSummarizer()
summary = ' '.join(list(map(str, summarizer(parser.document, SENTENCES_COUNT))))
return summary
def lsa_summarizer(file):
parser = PlaintextParser.from_file(file, Tokenizer("english"))
summarizer = LsaSummarizer()
text = open(file, 'r').read()
chosen_sentences = summarizer(parser.document, model_utilities.summary_length(text))
summary = ""
for sentence_tuple in chosen_sentences:
line = ""
for i in range(len(sentence_tuple.words)):
if i == len(sentence_tuple.words) - 1:
line += sentence_tuple.words[i] + "."
else:
line += sentence_tuple.words[i] + " "
summary += line + " "
return summary
def dada_summarize(content: str, title: str = "") -> dict :
response = dict()
content = process_content(content)
title = process_title(title)
# textrank [need newline to split sentence]
response["textrank"] = summarize(content)
# textteaser [need newline to split sentence]
cnt = int(len(content.split('\n'))*0.3)
response['textteaser'] = "\n".join(tt.summarize(title, content, count=cnt))
### sumy
parser = PlaintextParser.from_string(content, tokenizer)
# LSA
summarizer = LsaSummarizer(stemmer)
sentences = [str(i) for i in summarizer(parser.document, SENTENCES_COUNT)]
response['lsa'] = "\n".join(sentences)
# textrank2
summarizer = TextRankSummarizer(stemmer)
sentences = [str(i) for i in summarizer(parser.document, SENTENCES_COUNT)]
response['textrank2'] = "\n".join(sentences)
# lexrank
summarizer = LexRankSummarizer(stemmer)
sentences = [str(i) for i in summarizer(parser.document, SENTENCES_COUNT)]
response['lexrank'] = "\n".join(sentences)
# ruduction
summarizer = ReductionSummarizer(stemmer)
sentences = [str(i) for i in summarizer(parser.document, SENTENCES_COUNT)]
response['reduction'] = "\n".join(sentences)
#kl-sum
summarizer = KLSummarizer(stemmer)
sentences = [str(i) for i in summarizer(parser.document, SENTENCES_COUNT)]
response['kl-sum'] = "\n".join(sentences)
# bert
response['bert'] = model(content, ratio=0.4)
return response
def choose_summarizer(self, summarizer_string: str):
logging.debug("Changing summarizer to: {}".format(summarizer_string))
if summarizer_string == "LexRank": # LexRank
self.summarizer = LexRankSummarizer(stemmer)
elif summarizer_string == "TextRank": # TextRank
self.summarizer = TextRankSummarizer(stemmer)
elif summarizer_string == "Luhn": # Luhn
self.summarizer = LuhnSummarizer(stemmer)
elif summarizer_string == "LSA": # LSA
self.summarizer = LsaSummarizer(stemmer)
elif summarizer_string == "SumBasic": # SumBasic
self.summarizer = SumBasicSummarizer(stemmer)
# allow summarizer to take stop words into account
self.summarizer.stop_words = get_stop_words(LANGUAGE)
def get_summarizers(self, names):
"""Retrieves sumy summarizers algorithms
Parameters:
names (list): list of summarizer algorithm names
Returns:
dict:summarizers
"""
summarizers = {}
for name in names:
if name == "random":
from sumy.summarizers.random import RandomSummarizer
summarizers["random"] = RandomSummarizer(null_stemmer)
elif name == "luhn":
from sumy.summarizers.luhn import LuhnSummarizer
summarizers["luhn"] = LuhnSummarizer(stemmer=null_stemmer)
elif name == "lsa":
from sumy.summarizers.lsa import LsaSummarizer
summarizers["lsa"] = LsaSummarizer(stemmer=null_stemmer)
elif name == "lexrank":
from sumy.summarizers.lex_rank import LexRankSummarizer
summarizers["lexrank"] = LexRankSummarizer(null_stemmer)
elif name == "textrank":
from sumy.summarizers.text_rank import TextRankSummarizer
summarizers["textrank"] = TextRankSummarizer(null_stemmer)
elif name == "sumbasic":
from sumy.summarizers.sum_basic import SumBasicSummarizer
summarizers["sumbasic"] = SumBasicSummarizer(null_stemmer)
elif name == "kl-sum":
from sumy.summarizers.kl import KLSummarizer
summarizers["kl-sum"] = KLSummarizer(null_stemmer)
elif name == "reduction":
from sumy.summarizers.reduction import ReductionSummarizer
summarizers["reduction"] = ReductionSummarizer(null_stemmer)
for _, summarizer in summarizers.items():
summarizer.stop_words = frozenset(
self.stop_words._get_stop_words(custom_stop_words=[]))
return summarizers
def sumy_summarizer(text, ratio, summarizer_type):
num_sent = int(len(text.split(".")) * ratio)
parser = PlaintextParser.from_string(text, Tokenizer("english"))
if ((summarizer_type == 'lexrank') or (summarizer_type == 'Lexrank')):
summarizer_instance = LexRankSummarizer()
elif ((summarizer_type == 'reduction')
or (summarizer_type == 'reduction')):
summarizer_instance = ReductionSummarizer()
elif ((summarizer_type == 'lsa') or (summarizer_type == 'LSA')):
summarizer_instance = LsaSummarizer()
elif ((summarizer_type == 'luhn') or (summarizer_type == 'Luhn')):
summarizer_instance = LuhnSummarizer()
elif ((summarizer_type == 'KL') or (summarizer_type == 'kl')):
summarizer_instance = KLSummarizer()
summary_values = summarizer_instance(parser.document, num_sent)
final_summary = []
for sent in summary_values:
final_summary.append(str(sent))
summary_values = convert_to_string(final_summary)
return summary_values
def summarize(self, summarizer_type, max_sentences):
# TextRank
if summarizer_type == "textrank":
self.result_list = summarize(self.document,
ratio=0.3,
word_count=None,
split=True)
# PyTextRank
elif summarizer_type == "lsa":
parser = HtmlParser.from_string(self.document,
None,
tokenizer=Tokenizer("english"))
stemmer = Stemmer("english")
summarizer = LsaSummarizer(stemmer)
summarizer.stop_words = get_stop_words("english")
summarized_sentence_list = summarizer(parser.document,
max_sentences)
self.result_list = [
str(sentence) for sentence in summarized_sentence_list
]
def extract_components(bill_id, url):
url = url + "/text"
browser = RoboBrowser(user_agent='Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36', history=True)
browser.open(url)
content = browser.find("section", class_="legis-body")
if content is None:
return []
sections = content.find_all("section", class_="little-level")
section_data = content.text
parser = PlaintextParser.from_string(section_data, Tokenizer("english"))
summarizer = LsaSummarizer()
num_sentences = 10 if len(sections) > 10 else len(sections)
summary = summarizer(parser.document, num_sentences)
return list(set(summary))
'''
def summary_benchmarks(sentences_string):
'''
:param sentences_string: all sentences as one string, has been tokenized
:return:
'''
parser = PlaintextParser.from_string(sentences_string,
Tokenizer("english"))
print('=========== Basic Sum ============')
summarizer = SumBasicSummarizer()
summary = summarizer(parser.document,
3) # Summarize the document with 5 sentences
for sentence in summary:
print sentence
print('=========== LSA ============')
summarizer = LsaSummarizer()
summary = summarizer(parser.document,
3) # Summarize the document with 5 sentences
for sentence in summary:
print sentence
print('===========LexRank============')
summarizer = LexRankSummarizer()
summary = summarizer(parser.document,
3) # Summarize the document with 5 sentences
for sentence in summary:
print sentence
print('===========KL Divergence============')
summarizer = KLSummarizer()
summary = summarizer(parser.document,
3) # Summarize the document with 5 sentences
for sentence in summary:
print sentence
print('===========Luhn============')
summarizer = LuhnSummarizer()
summary = summarizer(parser.document,
3) # Summarize the document with 5 sentences
for sentence in summary:
print sentence
