def klsum(parser, sentence_count):
summarizer_6 = KLSummarizer(Stemmer(language))
summarizer_6.stop_words = get_stop_words(language)
summary_6 = summarizer_6(parser.document, sentence_count)
temp = ''
for sentence in summary_6:
temp = temp + str(sentence)
return (temp)
def __init__():
LANGUAGE = "english"
SENTENCES_COUNT = 1
stemmer = Stemmer(LANGUAGE)
lsaSummarizer = Lsa(stemmer)
lsaSummarizer.stop_words = get_stop_words(LANGUAGE)
luhnSummarizer = Luhn(stemmer)
luhnSummarizer.stop_words = get_stop_words(LANGUAGE)
# edmundsonSummarizer.bonus_words = get_bonus_words
lexrankSummarizer = LexRank(stemmer)
lexrankSummarizer.stop_words = get_stop_words(LANGUAGE)
textrankSummarizer = TxtRank(stemmer)
textrankSummarizer.stop_words = get_stop_words(LANGUAGE)
sumbasicSummarizer = SumBasic(stemmer)
sumbasicSummarizer.stop_words = get_stop_words(LANGUAGE)
klSummarizer = KL(stemmer)
klSummarizer.stop_words = get_stop_words(LANGUAGE)
parser = HtmlParser.from_string(text, 0, Tokenizer(LANGUAGE))
allvariations = []
for sentence in lsaSummarizer(parser.document, SENTENCES_COUNT):
# print("Summarizing text via LSA: ")
print((str(sentence)))
allvariations.append(sentence)
for sentence in luhnSummarizer(parser.document, SENTENCES_COUNT):
#print("Summarizing text via Luhn: ")
print(str(sentence))
allvariations.append(sentence)
for sentence in lexrankSummarizer(parser.document, SENTENCES_COUNT):
#print("Summarizing text via Lexrank: ")
print(str(sentence))
allvariations.append(sentence)
for sentence in textrankSummarizer(parser.document, SENTENCES_COUNT):
#print("Summarizing text via Textrank: ")
print(str(sentence))
allvariations.append(sentence)
for sentence in sumbasicSummarizer(parser.document, SENTENCES_COUNT):
#print("Summarizing text via Sumbasic: ")
print(str(sentence))
allvariations.append(sentence)
for sentence in klSummarizer(parser.document, SENTENCES_COUNT):
#print("Summarizing text via klSum: ")
print(str(sentence))
allvariations.append(sentence)
return allvariations
def kl_summarizer(text, stemmer, language, sentences_count):
parser = PlaintextParser.from_string(text, Tokenizer(language))
summarizer_luhn = KLSummarizer(stemmer)
summarizer_luhn.stop_words = get_stop_words(language)
sentences = []
for sentence in summarizer_luhn(parser.document, sentences_count):
a = sentence
sentences.append(str(a))
return "\n".join(sentences)
def klReferenceSummary(path):
sentencesList = []
parser = PlaintextParser.from_file(path, Tokenizer(LANGUAGE))
stemmer = Stemmer(LANGUAGE)
summarizer = KLSummarizer(stemmer)
summarizer.stop_words = get_stop_words(LANGUAGE)
for sentence in summarizer(parser.document, SENTENCES_COUNT):
#print(sentence._text)
sentencesList.append(sentence._text)
return sentencesList
def run_sumy(text, algo='KL', sent_count=3):
parser = PlaintextParser.from_string(text, Tokenizer("english"))
stemmer = Stemmer("english")
if algo == 'KL':
summarizer = KLSummarizer(stemmer)
elif algo == 'LexRank':
summarizer = LexRankSummarizer(stemmer)
summarizer.stop_words = get_stop_words("english")
summary_list = summarizer(parser.document, sent_count)
return summary_list
def klReferenceSummary(path): sentencesList=[] parser = PlaintextParser.from_file(path, Tokenizer(LANGUAGE)) stemmer = Stemmer(LANGUAGE) summarizer = KLSummarizer(stemmer) summarizer.stop_words = get_stop_words(LANGUAGE) for sentence in summarizer(parser.document, SENTENCES_COUNT): #print(sentence._text) sentencesList.append(sentence._text) return sentencesList
def KL(rsc_file, dst_file, count):
language = "chinese"
parser = PlaintextParser.from_file(rsc_file,
Tokenizer(language),
encoding='utf-8')
stemmer = Stemmer(language) # 语言容器
summarizer = KLSummarizer(stemmer) # LSA算法
summarizer.stop_words = get_stop_words(language)
with open(dst_file, 'w', encoding='utf-8') as f:
for sentence in summarizer(parser.document, count):
f.write(str(sentence))
f.write('\n')
print(sentence)
def run_sumy(text, algo='KL', sent_count=6):
# time0 = time.time()
parser = PlaintextParser.from_string(text, Tokenizer("english"))
# time1 = time.time()
stemmer = Stemmer("english")
# time2 = time.time()
if algo == 'KL':
summarizer = KLSummarizer(stemmer)
elif algo == 'LexRank':
summarizer = LexRankSummarizer(stemmer)
summarizer.stop_words = get_stop_words("english")
# time3 = time.time()
summary_list = summarizer(parser.document, sent_count)
# time4 = time.time()
# print('Parse time: {} \t Stem time: {} \t Stop words time: {} \t Summarizer time: {}'.format(time1, time2, time3, time4))
return summary_list
def _initGenericSummarizer(self):
langauage = "english"
stemmer = SumyStemmer(langauage)
if self.algorithm == ALGORITHM_KL:
summarizer = SumyKLSummarizer(stemmer)
elif self.algorithm == ALGORITHM_LSA:
summarizer = SumyLsaSummarizer(stemmer)
elif self.algorithm == ALGORITHM_TEXTRANK:
summarizer = SumyTextRankSummarizer(stemmer)
elif self.algorithm == ALGORITHM_LUHN:
summarizer = SumyLuhnSummarizer(stemmer)
elif self.algorithm == ALGORITHM_BASIC:
summarizer = SumySumBasicSummarizer(stemmer)
else:
summarizer = SumyLexRankSummarizer(stemmer)
summarizer.stop_words = SumyStopWords(langauage)
return summarizer, langauage
def kl_summarize(self):
summarizer = KLSummarizer()
summarizer.stop_words = self.stop_words
summary_tuple = (summarizer(self.parser.document, 4))
kl_summary = " ".join(map(str, summary_tuple))
return kl_summary
def summarizer(stop_words):
summarizer = KLSummarizer()
summarizer.stop_words = stop_words
return summarizer
from __future__ import absolute_import
from __future__ import division, print_function, unicode_literals
from sumy.parsers.html import HtmlParser
from sumy.parsers.plaintext import PlaintextParser
from sumy.nlp.tokenizers import Tokenizer
from sumy.summarizers.lsa import LsaSummarizer
from sumy.summarizers.kl import KLSummarizer
from sumy.nlp.stemmers import Stemmer
from sumy.utils import get_stop_words
import sys
LANGUAGE = "english"
SENTENCES_COUNT = int(sys.argv[2])
text_file = sys.argv[1]
if __name__ == "__main__":
parser = PlaintextParser.from_file(text_file, Tokenizer(LANGUAGE))
stemmer = Stemmer(LANGUAGE)
summarizer = KLSummarizer(stemmer)
summarizer.stop_words = get_stop_words(LANGUAGE)
for sentence in summarizer(parser.document, SENTENCES_COUNT):
print(sentence)
from sumy.parsers.html import HtmlParser
from sumy.parsers.plaintext import PlaintextParser
from sumy.nlp.tokenizers import Tokenizer
from sumy.summarizers.kl import KLSummarizer as Summarizer
from sumy.nlp.stemmers import Stemmer
from sumy.utils import get_stop_words
LANGUAGE = "english"
SENTENCES_COUNT = "5"
if __name__ == "__main__":
directory = "~/dropbox/17-18/573/AQUAINT/nyt/2000/"
# TODO: Get list of files and loop each file
filename = "20000101_NYT"
process_file = "doc.txt" # directory + filename
url = "file://home/unclenacho/school/573/src/doc.txt"
parser = HtmlParser.from_file(process_file, None, Tokenizer(LANGUAGE))
# parser = PlaintextParser.from_file(process_file, Tokenizer(LANGUAGE))
stemmer = Stemmer(LANGUAGE)
summarizer = Summarizer(stemmer)
summarizer.stop_words = get_stop_words(LANGUAGE)
for sentence in summarizer(parser.document, SENTENCES_COUNT):
print(sentence)
def __summarize(self, parser):
summarizer = KLSummarizer(Stemmer(self.__language))
summarizer.stop_words = get_stop_words(self.__language)
final_sentences = summarizer(parser.document, self.__sentences_count)
return self.__join_sentences(final_sentences)
def summarizer(stop_words):
summarizer = KLSummarizer()
summarizer.stop_words = stop_words
return summarizer
def klsum(doc, refsum):
stemmer = Stemmer("english")
summarizer = KLSummarizer(stemmer)
summarizer.stop_words = get_stop_words("english")
summary = summarize(doc, summarizer)
return evaluate(summary, refsum)
def klsumm(doc):
stemmer = Stemmer("english")
summarizer = KLSummarizer(stemmer)
summarizer.stop_words = get_stop_words("english")
summary = summarize(doc, summarizer)
return summary
def build_kl(parser, language):
summarizer = KLSummarizer(Stemmer(language))
summarizer.stop_words = get_stop_words(language)
return summarizer
