def summary():
# 텍스트 변수에 받아오기
text = open('./static/text_files/{}'.format(TEXTFILENAME),
mode='rt',
encoding='utf-8')
text = text.read()
model = Summarizer('distilbert-base-uncased')
resp = model(text)
# make summary txt
with open(
'static/text_files/{}'.format(
TEXTFILENAME.replace('.txt', '_summary.txt')),
'w') as txt_file:
txt_file.write(resp)
return render_template('SummaryPage.html', data=resp)
def summarize_text(self, text, **kwargs):
"""Summarize text
:params text: Text to summarize
:params kwargs: kwargs to pass to tokenizer
"""
if self.summarizer is None:
summarizer_kwargs = {
"custom_model": self.model,
"custom_tokenizer": self.tokenizer
}
# TODO: would be cool to enable this feature
# if self.use_coreference_handling:
# summarizer_kwargs["sentence_handler"] = CoreferenceHandler(
# greedyness=0.4)
self.summarizer = Summarizer(**summarizer_kwargs)
return self.summarizer(text, **kwargs)
def bertExtSum():
from summarizer import Summarizer
model = Summarizer()
get_para_summary = open('sports_test.txt',
encoding="utf8",
errors='ignore').read()
from gensim.parsing.preprocessing import remove_stopwords
filtered_sentence = remove_stopwords(get_para_summary)
# print(filtered_sentence)
result = model(filtered_sentence, min_length=20)
summary = "".join(result)
return (summary)
def bert_summarizer(source_url_list, kyc_doc):
"""
:param source_url_list: list of valid urls to scrape
:param kyc_doc: profile summary of client from documentum
:return: similarity score , polarity
"""
corpus = gather_content_data(url_list=source_url_list)
model = Summarizer()
result = model(corpus,
min_length=10,
ratio=0.5,
algorithm='gmm',
max_length=200)
full = ''.join(result)
similarity_score = calculate_similarity_score(full, kyc_doc)
testimonial = TextBlob(full)
polarity = testimonial.sentiment.polarity
return full, similarity_score, polarity
def text_summary(algorithm, text, ratio):
''' Generate summary using one of the three implemented
algorithm
Args:
text: text to summarize
ratio: ratio of the original text to summarize into
Returns:
summarized text using input algorithm
'''
if (algorithm == 'gensim'):
return summarize(text=text, ratio=ratio)
if (algorithm == 'spacy'):
doc = nlp(text)
return spacy_summary(doc, ratio)
if (algorithm == 'bert'):
model = Summarizer()
result = model(text, ratio)
return ''.join(result)
def bertsum(filename, out_max_length, speaker_talk, speaker_num, total):
from summarizer import Summarizer
with open(
"./static/text_files/{}".format(
filename.replace('.txt', '_dr_sum_file.txt')), 'w') as f:
f.write(" More than " + str(speaker_num) +
" people participated... <br>\n")
for i in range(speaker_num):
text = ". ".join(speaker_talk[i])
f.write("speaker " + str(i + 1) + " :<br> ")
model = Summarizer('distilbert-base-uncased')
output = model(text, max_length=out_max_length)
f.write(output + "\n")
out_text = ". ".join(total)
total_output = model(out_text, max_length=out_max_length)
f.write("\nsummary : " + total_output)
return
def summarizer(argumet, ratio_t=0.5, summarizer_ext=False):
"""
Parameters
----------
argumet : String
The text from the conversation/meeting. It will be processed
ratio_t : float, optional
It specifies how much of the original text we wish to keep.
The default is 0.5.
summarizer_ext : Boolean, optional
If True will summarize further the keynotes on Organizations and people.
The default is False.
Returns
-------
full : String
The summary of our text(argument variable).
dates : DataObject
The key dates from the text(argument variable).
date_tuple: List
Contains a pair of dates and the associated text for them
"""
model = Summarizer()
result = model(argumet, ratio=ratio_t, min_length=60, max_length=160)
full = ''.join(result)
dates1 = re.findall(r'\d+\S\d+\S\d+', argumet)
dates2 = re.findall(r'[A-Z]\w+\s\d+', argumet)
dates = dates1 + dates2
a_list = nltk.tokenize.sent_tokenize(argumet)
date_tuple = []
sent = ""
for dat in dates:
dat = str(dat)
for element in a_list:
if dat in element:
sent = sent + " " + element
date_tuple.append([dat, sent])
if summarizer_ext:
for index in range(len(date_tuple)):
date_tuple[index][1] = model(date_tuple[index][1])
return full, dates, date_tuple
class SummarizationTest(unittest.TestCase):
summarizer = Summarizer('distilbert-base-uncased')
PASSAGE = '''
The Chrysler Building, the famous art deco New York skyscraper, will be sold for a small fraction of its previous sales price.
The deal, first reported by The Real Deal, was for $150 million, according to a source familiar with the deal.
Mubadala, an Abu Dhabi investment fund, purchased 90% of the building for $800 million in 2008.
Real estate firm Tishman Speyer had owned the other 10%.
The buyer is RFR Holding, a New York real estate company.
Officials with Tishman and RFR did not immediately respond to a request for comments.
It's unclear when the deal will close.
The building sold fairly quickly after being publicly placed on the market only two months ago.
The sale was handled by CBRE Group.
The incentive to sell the building at such a huge loss was due to the soaring rent the owners pay to Cooper Union, a New York college, for the land under the building.
The rent is rising from $7.75 million last year to $32.5 million this year to $41 million in 2028.
Meantime, rents in the building itself are not rising nearly that fast.
While the building is an iconic landmark in the New York skyline, it is competing against newer office towers with large floor-to-ceiling windows and all the modern amenities.
Still the building is among the best known in the city, even to people who have never been to New York.
It is famous for its triangle-shaped, vaulted windows worked into the stylized crown, along with its distinctive eagle gargoyles near the top.
It has been featured prominently in many films, including Men in Black 3, Spider-Man, Armageddon, Two Weeks Notice and Independence Day.
The previous sale took place just before the 2008 financial meltdown led to a plunge in real estate prices.
Still there have been a number of high profile skyscrapers purchased for top dollar in recent years, including the Waldorf Astoria hotel, which Chinese firm Anbang Insurance purchased in 2016 for nearly $2 billion, and the Willis Tower in Chicago, which was formerly known as Sears Tower, once the world's tallest.
Blackstone Group (BX) bought it for $1.3 billion 2015.
The Chrysler Building was the headquarters of the American automaker until 1953, but it was named for and owned by Chrysler chief Walter Chrysler, not the company itself.
Walter Chrysler had set out to build the tallest building in the world, a competition at that time with another Manhattan skyscraper under construction at 40 Wall Street at the south end of Manhattan. He kept secret the plans for the spire that would grace the top of the building, building it inside the structure and out of view of the public until 40 Wall Street was complete.
Once the competitor could rise no higher, the spire of the Chrysler building was raised into view, giving it the title.
'''
def test_summary_creation(self):
res = self.summarizer(self.PASSAGE)
self.assertEqual(res, "The Chrysler Building, the famous art deco New York skyscraper, will be sold for a small fraction of its previous sales price. Mubadala, an Abu Dhabi investment fund, purchased 90% of the building for $800 million in 2008. Officials with Tishman and RFR did not immediately respond to a request for comments. He kept secret the plans for the spire that would grace the top of the building, building it inside the structure and out of view of the public until 40 Wall Street was complete.")
def test_summary_larger_ratio(self):
res = self.summarizer(self.PASSAGE, ratio=0.5)
self.assertEqual(res, "The Chrysler Building, the famous art deco New York skyscraper, will be sold for a small fraction of its previous sales price. Mubadala, an Abu Dhabi investment fund, purchased 90% of the building for $800 million in 2008. Real estate firm Tishman Speyer had owned the other 10%. Officials with Tishman and RFR did not immediately respond to a request for comments. The building sold fairly quickly after being publicly placed on the market only two months ago. The rent is rising from $7.75 million last year to $32.5 million this year to $41 million in 2028. Meantime, rents in the building itself are not rising nearly that fast. It is famous for its triangle-shaped, vaulted windows worked into the stylized crown, along with its distinctive eagle gargoyles near the top. It has been featured prominently in many films, including Men in Black 3, Spider-Man, Armageddon, Two Weeks Notice and Independence Day. He kept secret the plans for the spire that would grace the top of the building, building it inside the structure and out of view of the public until 40 Wall Street was complete.")
def test_cluster_algorithm(self):
res = self.summarizer(self.PASSAGE, algorithm='gmm')
self.assertEqual(res, "The Chrysler Building, the famous art deco New York skyscraper, will be sold for a small fraction of its previous sales price. Mubadala, an Abu Dhabi investment fund, purchased 90% of the building for $800 million in 2008. The incentive to sell the building at such a huge loss was due to the soaring rent the owners pay to Cooper Union, a New York college, for the land under the building. Once the competitor could rise no higher, the spire of the Chrysler building was raised into view, giving it the title.")
def test_do_not_use_first(self):
res = self.summarizer(self.PASSAGE, ratio=0.1, use_first=False)
self.assertIsNotNone(res)
def summarize_one(pdf, trim, texts, sums, keys, lang):
""" summarizer for one document
"""
if pdf[-4:].lower() != ".pdf": return None
name = pdf[trim:-4]
tname0 = texts + name
tname = texts + name + ".txt"
sname = sums + name + ".txt"
kname = keys + name + ".txt"
ensure_path(tname)
try:
print('START processing:', pdf)
if not (pdf2txt(pdf, tname)):
print('Unable to convert from PDF, skipping file!')
return None
clean_text_file(tname, lang=lang)
nlp = Summarizer(lang=lang)
nlp.from_file(tname0)
kws, _, sents, _ = nlp.info()
ktext = "\n".join(kws)
ensure_path(kname)
string2file(ktext, kname)
stext = "\n".join(sents)
ensure_path(sname)
string2file(stext, sname)
print('WRITTEN TO', sname, kname)
text = "\n".join(
['FILE:', pdf, '\nSUMMARY:', stext, '\nKEYWORDS:', ktext, '\n'])
print('DONE processing:', pdf)
return text
except IndexError:
print('ERROR:', sys.exc_info()[0])
print('Processing failed on:', pdf)
return None
except ValueError:
return None
except RuntimeError:
return None
def __init__(self,
config,
lang=None,
custom_model=None,
custom_tokenizer=None,
tokenizer=None,
verbose=None,
**kwargs):
self.config = config
self.verbose = verbose
self.language = getSpacyLang(lang)
if not (custom_model and custom_tokenizer):
custom_model, custom_tokenizer = self.get_pretrained_language(lang)
self.model = Summarizer(language=self.language,
custom_model=custom_model,
custom_tokenizer=custom_tokenizer,
**kwargs)
def summarise(text):
model = Summarizer('distilbert-base-uncased')
#import time
#start = time.time()
resp = model(text)
#end = time.time()
#print(f'Response Time: {end-start}')
print(f'Summary: {resp}')
newfile = open(r"C:\Users\prasad\Desktop\SKT_writ\SKT_writ_summ.txt", 'w+')
newfile.write(resp)
def convert_mp4_to_audio_and_summarize_transcript(directory, file_path):
video_filename = file_path
audio_filename = video_filename[:-4] + ".wav"
os.system('ffmpeg -y -i {} -acodec pcm_s16le -ar 16000 {}'.format(
video_filename, audio_filename))
txt2timestamp, timestamp2txt = speech2text.speech_recognize_continuous_from_file(
audio_filename,
f'{directory}/secrets.json',
)
body = [k for k in txt2timestamp]
body = ' '.join(body)
raw_list = body.split('.')
model = Summarizer()
result = model(body, min_length=60)
brief = ''.join(result)
print(brief)
brief_list = brief.split('.')
brief_list = [f'{sentence}.' for sentence in brief_list]
print('original length: ', len(body))
print('summary length: ', len(brief))
print('original/summary: ', len(body) / len(brief))
print('original sentence count: ', len(raw_list))
print('summary sentence count: ', len(brief_list))
# azure tells me the timestamp of each utterance, not sentence, so I account for that here
timestamps = []
for sentence in brief_list:
for utterance in txt2timestamp.keys():
# utterance can be 1 or more sentences
if sentence in utterance:
timestamps.append(txt2timestamp[utterance])
timestamp_and_text = [{
"timestamp": time,
"text": text
} for time, text in zip(timestamps, brief_list)]
# print(timestamp_and_text)
return [timestamp_and_text, body]
def run_classification_summary(df, classifier_config):
"""Runs the various classification algorithms outputting a summary dataframe.
Args:
df: pandas dataframe containing the error information we wish to classify and summarize
classifier_config: config_pb2 proto specified by the configuration file
Returns:
pandas dataframe that summarizes the information obtained from the classification algorithms
run on the input dataframe
"""
# Running our classifiers
error_code_matcher = ErrorCodeMatcher(df, classifier_config)
error_code_matcher.match_informative_errors()
k_means_classifier = KMeansClusterer(df, classifier_config)
k_means_classifier.cluster_errors()
# Running the summarizer
summarizer = Summarizer(df, classifier_config)
return summarizer.generate_summary()
def bertSum(self):
# Load model, model config and tokenizer via Transformers
with open(
'/home/lab05/A1B4/wow/static/kcbert_large/kcbert_base_config.bin',
'rb') as f:
custom_config = pickle.load(f)
custom_config.output_hidden_states = True
with open(
'/home/lab05/A1B4/wow/static/kcbert_large/kcbert_base_tokenizer.bin',
'rb') as f:
custom_tokenizer = pickle.load(f)
# custom_tokenizer = AutoTokenizer.from_pretrained('beomi/kcbert-base')
# with open('/home/lab05/A1B4/wow/static/kcbert_large/kcbert_base_model.bin', 'rb') as f:
# custom_model = pickle.load(f)
custom_model = AutoModel.from_pretrained('beomi/kcbert-base',
config=custom_config)
summarizer_model = Summarizer(custom_model=custom_model,
custom_tokenizer=custom_tokenizer)
summaries = summarizer_model(self.paragraph,
num_sentences=self.probe_num)
return summaries
def __init__(self, summarizer, batch_size=1):
"""
:param summarizer: SummarizerModel value
:param batch_size : [int] batch size for summarizer input (for T5 and BART)
"""
super().__init__()
self.summarizer = summarizer
self.batch_size = batch_size
print("Loading model : ", str(summarizer))
if self.summarizer == SummarizerModel.BERT_SUM:
self.model = Summarizer()
if self.summarizer == SummarizerModel.T5:
self.tokenizer = T5Tokenizer.from_pretrained('t5-small')
self.model = T5ForConditionalGeneration.from_pretrained('t5-small')
self.model.eval()
if torch.cuda.is_available():
self.model.cuda()
self.decoding_strategy = T5_DECODING_STRAT
print("Use for decoding strategy :", self.decoding_strategy)
if self.summarizer == SummarizerModel.BART:
self.tokenizer = BartTokenizer.from_pretrained('bart-large-cnn')
self.model = BartForConditionalGeneration.from_pretrained('bart-large-cnn')
self.model.eval()
if torch.cuda.is_available():
self.model.cuda()
self.decoding_strategy = BART_DECODING_STRAT
print("Use for decoding strategy :", self.decoding_strategy)
if self.summarizer == SummarizerModel.PYSUM:
self.model = AutoAbstractor()
self.model.tokenizable_doc = SimpleTokenizer()
self.model.delimiter_list = ['.', '\n']
self.doc_filtering = TopNRankAbstractor()
if self.summarizer == SummarizerModel.KW:
self.model = keywords
async def summarize_cmd(message: discord.Message, url: str):
"""[summary]
Args:
message (discord.Message): [description]
url (str): [description]
"""
global scraped_content
global send_raw_cmds
# scraping must be done first before any analytics commands
if len(scraped_content) == 0:
await scrape_cmd(message=message, url=url)
# only display printout message for single urls
if type(url) != list:
await message.channel.send(content=f'**Summarizing {url}...**')
# summarization code
model = Summarizer()
result = model(scraped_content, ratio=0.25)
full = ''.join(result)
# handling keyword queries
if type(url) == list:
with open(file='keyword_search.txt', mode='a') as f:
f.write(
"**Summarizing collected text from websites together**\n\n" +
full + '\n\n')
f.close()
return
if 'summarize' in send_raw_cmds:
await message.channel.send(content=full)
else:
with open(file='summarize.txt', mode='w') as f:
f.write(scraped_content)
f.close()
await message.channel.send(file=discord.File(fp='summarize.txt'))
def test_dirty_text_summary(self):
test_text = '最新 科技 http: 新闻和创业 公司信息 ”∆˙∫˚˜ ˜µ∆∫˙© ∆∆˚µ˚' + self.test_text[
0]
summarizer = Summarizer(test_text)
summarizer.parse()
text = summarizer.summarize()
keywords = summarizer.keywords()
key_noun_phrases = summarizer.key_noun_phrases()
self.assertEqual(text, [
'This is happening in the city of Tianjin, about an hours drive south of '
'Beijing, within a gleaming office building that belongs to iFlytek, one of '
'Chinas rapidly rising artificial-intelligence companies.',
'Beyond guarded gates, inside a glitzy showroom, the US president is on a '
'large TV screen heaping praise on the Chinese company.',
'This is happening in the city of Tianjin, about an hours drive south of '
'Beijing, within a gleaming office building that belongs to iFlytek, one of '
'Chinas rapidly rising artificial-intelligence companies.',
'Beyond guarded gates, inside a glitzy showroom, the US president is on a '
'large TV screen heaping praise on the Chinese company.',
'However, AI itself could change all that.',
'A more advanced chip industry will help China realize its dream of becoming '
'a true technology superpower.',
'China wont be playing catch-up with these new chips, as it has done with '
'more conventional chips for decades.',
'Chinas chip ambitions have geopolitical implications, too.',
'A successful chip industry would make China more economically competitive '
'and independent.'
])
self.assertEqual(keywords, [
'chip', 'china', 'ai', 'company', 'iflytek', 'technology',
'algorithm', 'microchip', 'time', 'silicon', 'advanced',
'industry', 'beijing', 'belongs', 'inside'
])
self.assertEqual(key_noun_phrases, [
'chinese company', 'tsinghua unigroup', 'donald trump',
'gleaming office building', 'artificial-intelligence companies'
])
def __init__(self, priority):
'''
Initializes the text rank or bert summarizers
based on priority
priority should be either accuracy or speed
based on which we can choose either a
low speed high accuracy summarizer - bert
high speed lower accuracy summarizer - textRank
'''
self.text_summarizer = None
if priority == "accuracy":
handler = CoreferenceHandler(greedyness=.4)
self.text_summarizer = Summarizer(
model='distilbert-base-uncased',
sentence_handler=handler)
elif priority == "speed":
self.text_summarizer = summarize
else:
raise IncorrectInputError(
"priority must be either accuracy or speed")
self.entity_list = list()
self.text = None
def get_keywords_summ(list_sentences, cphrases, logPrint=False):
## 1. Remove '\n','\t', etc and make it readable.
for s in list_sentences:
s = " ".join(s.replace("\n", " ").split())
# Run through Summarizer
summarizer = Summarizer()
result = summarizer.summarize(list_sentences, cphrases)
# above gets lists of {'word': 'decision', 'count': 1}, for each sentence it is given.
# convert {'word': 'decision', 'count': 1} to form (str)word:(int)count for each word.
'''
if(logPrint):
print("\tConvert pre-processed words to dictionary layout:")
print("\t\t{ 'word':count }")
'''
cphrase_pp = ""
sentences = ""
list_sentences = list()
for i in range(0, len(result)):
sentences = ""
word_stat_line = result[i][0]
for j in range(0, len(word_stat_line)):
if i >= 1:
sentences = sentences + " " + str(
word_stat_line[j]['word']) + ":" + str(
word_stat_line[j]['count'])
else:
cphrase_pp = cphrase_pp + " " + str(
word_stat_line[j]['word']) + ":" + str(
word_stat_line[j]['count'])
if i > 0:
sentences.strip()
list_sentences.append(sentences.lstrip())
return (cphrase_pp.lstrip(), list_sentences)
def run():
parser = argparse.ArgumentParser(
description='Process and summarize lectures')
parser.add_argument('-path',
dest='path',
default=None,
help='File path of lecture')
parser.add_argument('-model',
dest='model',
default='bert-large-uncased',
help='')
parser.add_argument('-hidden',
dest='hidden',
default=-2,
help='Which hidden layer to use from Bert')
parser.add_argument('-reduce-option',
dest='reduce_option',
default='mean',
help='How to reduce the hidden layer from bert')
parser.add_argument('-greedyness',
dest='greedyness',
help='Greedyness of the NeuralCoref model',
default=0.45)
args = parser.parse_args()
if not args.path:
raise RuntimeError("Must supply text path.")
# with open(args.path) as d:
# text_data = d.read()
model = Summarizer(model=args.model,
hidden=args.hidden,
reduce_option=args.reduce_option)
input_data = pd.read_csv(args.path)
input_data['summary'] = input_data['Article'].map(model)
input_data.to_csv("Summarization_results.csv")
def main():
# Parse the JSON arguments
try:
config_args = parse_args()
except:
print("Add a config file using \'--config file_name.json\'")
exit(1)
# Create the experiment directories
_, config_args.summary_dir, config_args.checkpoint_dir = create_experiment_dirs(
config_args.experiment_dir)
# Reset the default Tensorflow graph
tf.reset_default_graph()
# Tensorflow specific configuration
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# Data loading
data = DataLoader(config_args.batch_size, config_args.shuffle)
print("Loading Data...")
config_args.img_height, config_args.img_width, config_args.num_channels, \
config_args.train_data_size, config_args.test_data_size = data.load_data()
print("Data loaded\n\n")
# Model creation
print("Building the model...")
model = MobileNet(config_args)
print("Model is built successfully\n\n")
# Summarizer creation
summarizer = Summarizer(sess, config_args.summary_dir)
# Train class
trainer = Train(sess, model, data, summarizer)
def summarizeUTNews():
print('UT NEWS SUMMARIZER START')
all_data = db.collection(u'UTNews').document(u'Documents').collection(u'Unsummarized').get()
# take text for each article, summarize it and add it to summarized collection
#print('next is ', next(all_data))
model = Summarizer()
# TODO: REMEMBER TO CHANGE THE RANGE BELOW
for doc in all_data[:]:
# gets raw, unsummarized article
article = doc.get('text')
# summarizes the articles into 2 sentences
result = model(article, num_sentences = 1)
summary = "".join(result)
summary = u'{}'.format(summary)
new_data = {
'category': doc.get('category'),
'title': doc.get('title'),
'text': summary,
'url': doc.get('url')
}
db.collection(u'UTNews').document(u'Documents').collection(u'Summarized').add(new_data)
print('----------finished--------')
print('END')
def run():
parser = argparse.ArgumentParser(
description='Process and summarize lectures')
parser.add_argument('-path',
dest='path',
default=None,
help='File path of lecture')
parser.add_argument('-model',
dest='model',
default='bert-base-uncased',
help='')
parser.add_argument('-hidden',
dest='hidden',
default=-2,
help='Which hidden layer to use from Bert')
parser.add_argument('-reduce-option',
dest='reduce_option',
default='mean',
help='How to reduce the hidden layer from bert')
parser.add_argument('-greedyness',
dest='greedyness',
help='Greedyness of the NeuralCoref model',
default=0.45)
args = parser.parse_args()
if not args.path:
raise RuntimeError("Must supply text path.")
with open(args.path) as d:
text_data = d.read()
model = Summarizer(model=args.model,
hidden=args.hidden,
reduce_option=args.reduce_option)
print(model(text_data))
def photo_upload():
photoform = PhotoForm()
if request.method == 'POST' and photoform.validate_on_submit():
photo = photoform.photo.data
filename = secure_filename(photo.filename)
photo.save(os.path.join(
app.config['UPLOAD_FOLDER'], filename
))
model = Summarizer('bert-base-uncased')
file = open('app/static/uploads/'+filename)
str = file.read()
str.encode('utf-8').strip()
file.close()
start = time.time()
resp = model(str)
end = time.time()
tot_time = end-start
return render_template('display_photo.html', filename=filename, summ=resp, tt=tot_time)
flash_errors(photoform)
return render_template('photo_upload.html', form=photoform)
def __init__(self): self.model=Summarizer()
from summarizer import Summarizer
model = Summarizer()
def textSumamryPredictor(text):
result = model(text)
full = ''.join(result)
return full
print(
textSumamryPredictor('''Science in Everyday Life
Science is a great blessing to mankind. Nothing better has happened in the history of man than advent of science in human life. The world into which science came was a world of ignorance, suffering and hardship. Science has come to relieve us to sufferings, to remove our ignorance and to lighter our toil.
Science is a faithful servant of man. It serves us in all walks of life. It is our servant in the home, in the field and in the factory. Science has transformed our daily life. Gone are the days when only the rich men could afford luxuries. Science has made them cheap and has brought them within the reach of everybody. Science has produced goods on a large market. These are sold at cheap rates in every market. Books, music and all other forms of entertainment have been brought to our door. Radio, television, cinema help us in passing our time and also provide education to us.
Science is our most faithful medical attendant. It shows every care for our health. Because of science we are cured of many diseases. It has given us the power to reduce epidemics. No longer are cholera, plague and small pox the scourge of mankind. Science has helped in reducing the death rate and has also enhanced the living age of humans.
Science has reduced distance and made travelling a pleasure. Science has annihilated time and space. Trains roar through deserts, jungles and mountains while aeroplanes fly across thousands of kilometers in a matter of hours. The work of months and years can now be completed in hours.
Science is the greatest blessing to the poor housewife. A thousand devices have been placed at her disposal to lighter her toil. There is electricity run kitchens in which cooking is pleasure. There is no dirt, no smoke and cooking with the help of gas and electricity can be done in the twinkling of an eye. Electricity helps her in washing and pressing clothes and even in cleaning floors.
Science has provided us with computers and machines which have greatly increased our efficiency. We are better connected to people today and information is only a click of the mouse away. Man no longer needs to do the back breaking job of digging into mines with bare hands or tilling the soil with animals. Every factory is a standing tribute from the care and comforts that science has brought into our life.
Science educates us in many ways. Large printing presses produce number of books at cheap rates. News is brought to us from every corner of the world through the newspaper, radios and television.
However science has done a great disservice to mankind in the field of armaments. Weapons of mass destruction, nuclear weapons and sophisticated armament have endangered our lives and threaten to destroy the world. However it is upto us whether we will destroy our world or make is more beautiful and comfortable with the help of science.'''
))
from summarizer import Summarizer
from transformers import *
model_list = ["distilbert-base-uncased", 'allenai/scibert_scivocab_uncased']
for m in model_list:
print("Caching model", m)
model = Summarizer(model=m)
from linebot import LineBotApi, WebhookHandler
from linebot.exceptions import InvalidSignatureError
from linebot.models import (
MessageEvent, TextMessage, TextSendMessage, SourceUser, SourceGroup,
SourceRoom, TemplateSendMessage, ConfirmTemplate, MessageTemplateAction,
ButtonsTemplate, URITemplateAction, PostbackTemplateAction,
CarouselTemplate, CarouselColumn, PostbackEvent, StickerMessage,
StickerSendMessage, LocationMessage, LocationSendMessage, ImageMessage,
VideoMessage, AudioMessage, UnfollowEvent, FollowEvent, JoinEvent,
LeaveEvent, BeaconEvent)
app = Flask(__name__)
line_bot_api = LineBotApi(CHANNEL_ACCESS_TOKEN)
handler = WebhookHandler(CHANNEL_SECRET)
summarizer = Summarizer()
static_tmp_path = os.path.join(os.path.dirname(__file__), 'static', 'tmp')
translator = Translator()
def make_static_tmp_dir():
try:
os.makedirs(static_tmp_path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(static_tmp_path):
pass
else:
raise
@app.route('/')
def __init__(self):
self.summarizer = Summarizer()
# import config
import torch
import flask
from flask import Flask, request, render_template
import json
from transformers import T5Tokenizer, T5ForConditionalGeneration, T5Config
from summarizer import Summarizer
BART_PATH = 'bart-large'
T5_PATH = 't5-base'
# BART_PATH = 'model/bart'
# T5_PATH = 'model/t5'
app = Flask(__name__)
bart_model = Summarizer()
t5_model = T5ForConditionalGeneration.from_pretrained(T5_PATH)
t5_tokenizer = T5Tokenizer.from_pretrained(T5_PATH)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
def bart_summarize(input_text, num_beams=4, num_words=50):
input_text = str(input_text)
result = bart_model(input_text, min_length=50,max_length=100)
output = ''.join(result)
return output
