def summarize(path_stage1, path_stage2, path_stage3, wordlimit,
phraselimit):
print('Generating summary...')
#Calculate a significance weight for each sentence, using MinHash to approximate a Jaccard distance from key phrases determined by TextRank
kernel = pytextrank.rank_kernel(path_stage2)
try:
os.remove(path_stage3)
except OSError:
pass
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
#print(pytextrank.pretty_print(s._asdict()))
#Summarize essay based on most significant sentences and key phrases
phrases = ", ".join(
set([
p
for p in pytextrank.limit_keyphrases(path_stage2,
phrase_limit=phraselimit)
]))
sent_iter = sorted(pytextrank.limit_sentences(path_stage3,
word_limit=wordlimit),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
s.append(pytextrank.make_sentence(sent_text))
graf_text = " ".join(s)
#print("**excerpts:** %s\n\n**keywords:** %s" % (graf_text, phrases,))
return graf_text, phrases
def stage3(path_stage1, path_stage2, path_stage3):
#Stage 3
kernel = pytextrank.rank_kernel(path_stage2)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
def summarize(self, _id, content_text, word_limit):
self.logger.log("_id: " + _id)
self.logger.log("word_limit: " + str(word_limit))
# File names
path_stage0 = 'process/' + _id + '.json'
path_stage1 = 'process/' + _id + '_o1.json'
path_stage2 = 'process/' + _id + '_o2.json'
path_stage3 = 'process/' + _id + '_o3.json'
path_stage4 = 'process/' + _id + '_o4.json'
# Create input file
with open(path_stage0, 'w') as outfile:
json.dump({"id": "123", "text": content_text}, outfile)
# Statistical Parsing - Stage 1
# Perform statistical parsing/tagging on a document in JSON format
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(
pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
# Ranked Keyphrases - Stage 2
# Collect and normalize the key phrases from a parsed document
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
# Extractive Summarization - Stage 3
# Calculate a significance weight for each sentence, using MinHash to approximate a Jaccard distance from key phrases determined by TextRank
kernel = pytextrank.rank_kernel(path_stage2)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
# Final Output - Stage 4
# Summarize a document based on most significant sentences and key phrases
phrases = ", ".join(
set([
p for p in pytextrank.limit_keyphrases(path_stage2,
phrase_limit=12)
]))
sent_iter = sorted(pytextrank.limit_sentences(path_stage3,
word_limit=word_limit),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
s.append(pytextrank.make_sentence(sent_text))
graf_text = " ".join(s)
return {'excerpts': graf_text, 'keywords': phrases}
def pytrankSummarize(filename):
"""
This is another TextRank algorithm. It works in four stages, each feeding its output to the next
1. Part-of-Speech Tagging and lemmatization are performed for every sentence in the document.
2. Key phrases are extracted along with their counts, and are normalized.
3. Calculates a score for each sentence by approximating jaccard distance between the sentence and key phrases.
4. Summarizes the document based on most significant sentences and key phrases.
"""
import pytextrank
jsonText = createJSON(filename)
path_stage0 = jsonText
path_stage1 = "o1.json"
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
path_stage2 = "o2.json"
graph, ranks = pytextrank.text_rank(path_stage1)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
path_stage3 = "o3.json"
kernel = pytextrank.rank_kernel(path_stage2)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
phrases = ", ".join(
set([
p
for p in pytextrank.limit_keyphrases(path_stage2, phrase_limit=12)
]))
sent_iter = sorted(pytextrank.limit_sentences(path_stage3, word_limit=50),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
s.append(pytextrank.make_sentence(sent_text))
graf_text = " ".join(s)
print("")
print("####### From PyTextRank #######")
print("**excerpts:** %s\n\n**keywords:** %s" % (
graf_text,
phrases,
))
def topSentences(strlen):
kernel = pytextrank.rank_kernel('temp3.json')
i = 0
summary = []
for s in pytextrank.top_sentences(kernel, 'temp2.json'):
summary.append(s.text)
i = i + 1
if i > strlen:
break
return summary
def calculate_sentence_significance(self, paragraph_output,
key_phrases_output,
top_sentences_output, top_n_sentences):
kernel = pytextrank.rank_kernel(key_phrases_output)
with open(top_sentences_output, 'w') as f:
counter = 0
for sentence in pytextrank.top_sentences(kernel, paragraph_output):
if counter < top_n_sentences:
f.write(pytextrank.pretty_print(sentence._asdict()))
f.write("\n")
else:
return
counter = counter + 1
def summarize_text(input_file):
# seriously f**k this API
path_stage0 = input_file
path_stage1 = 'stage1.txt'
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
# to view output in this notebook
#print(pytextrank.pretty_print(graf))
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
path_stage2 = 'stage2.txt'
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
# to view output in this notebook
#print(pytextrank.pretty_print(rl))
path_stage3 = 'stage3.txt'
kernel = pytextrank.rank_kernel(path_stage2)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
# to view output in this notebook
#print(pytextrank.pretty_print(s._asdict()))
phrases = ", ".join(
set([
p
for p in pytextrank.limit_keyphrases(path_stage2, phrase_limit=12)
]))
sent_iter = sorted(pytextrank.limit_sentences(path_stage3, word_limit=120),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
s.append(pytextrank.make_sentence(sent_text))
graf_text = " ".join(s)
#print("**excerpts:** %s\n\n**keywords:** %s" % (graf_text, phrases,))
return ' '.join(s)
def calculate_sentence_significance(self, paragraph_output, key_phrases_output, \
top_sentences_output, top_n_sentences):
"""
Calculate the significance of each sentence based on the ranking.
Ranking is determined by the top n sentences (filter)
"""
kernel = pytextrank.rank_kernel(key_phrases_output)
with open(top_sentences_output, 'w') as temp_file:
counter = 0
for sentence in pytextrank.top_sentences(kernel, paragraph_output):
if counter < top_n_sentences:
temp_file.write(pytextrank.pretty_print(sentence._asdict()))
temp_file.write("\n")
else:
return
counter = counter + 1
def stage_3():
cur_dir = os.path.dirname(__file__)
data_dir = stage_1_dir
ids = os.listdir(data_dir)
result_dir = stage_3_dir
if os.path.exists(result_dir):
shutil.rmtree(result_dir, ignore_errors=True)
os.mkdir(result_dir)
os.chdir(result_dir)
for cur_id in ids:
print(cur_id)
kernel = pytextrank.rank_kernel(stage_2_dir + '\\' + cur_id)
with codecs.open(cur_id, "w+", "utf_8_sig") as file:
for s in pytextrank.top_sentences(kernel,
stage_1_dir + '\\' + cur_id):
file.write(pytextrank.pretty_print(s._asdict()))
file.write("\n")
os.chdir(cur_dir)
def text_ranking(video_seg_id, book_segment):
"""
:param book_segment: book segment in json format
:return: key sentences and key phrases
"""
# os.chdir(video_path)
# creating directory to store segments for clean structure
if not os.path.exists('TextRank_data'):
os.mkdir('TextRank_data')
if not os.path.exists('TextRank_data/seg' + str(video_seg_id)):
os.mkdir('TextRank_data/seg' + str(video_seg_id))
subdir = 'TextRank_data/seg' + str(video_seg_id) + '/'
path_stage1 = subdir + "stage1.json"
path_stage2 = subdir + "stage2_key_ph.json"
path_stage3 = subdir + "stage3_imp_sent.json"
"""Perform statistical parsing/tagging on a document in JSON format"""
parse_book_seg = pytextrank.parse_doc([book_segment])
with open(path_stage1, 'w') as f:
for graf in parse_book_seg:
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
graph, ranks = pytextrank.text_rank(path_stage1)
"""Collect and normalize the key phrases from a parsed document"""
key_phrases = list(pytextrank.normalize_key_phrases(path_stage1, ranks))
with open(path_stage2, 'w') as f:
for rl in key_phrases:
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
kernel = pytextrank.rank_kernel(path_stage2)
"""Calculate a significance weight for each sentence,
using MinHash to approximate a Jaccard distance from key phrases determined by TextRank"""
key_sentences = list(pytextrank.top_sentences(kernel, path_stage1))
with open(path_stage3, 'w') as f:
for s in key_sentences:
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
return key_sentences, key_phrases
def retrieveSentences(content, word_limit):
currpath = os.getcwd()
folder = os.path.join(currpath, str(uuid.uuid4()))
os.mkdir(folder)
fname = str(uuid.uuid4())
with open("{0}/{1}.json".format(folder, fname), "w") as f:
f.write(json.dumps({"id": fname, "text": content}))
f.close()
path_stage0 = "{0}/{1}.json".format(folder, fname)
path_stage1 = "{0}/o1.json".format(folder)
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
f.close()
path_stage2 = "{0}/o2.json".format(folder)
graph, ranks = pytextrank.text_rank(path_stage1)
#pytextrank.render_ranks(graph, ranks)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
f.close()
kernel = pytextrank.rank_kernel(path_stage2)
path_stage3 = "{0}/o3.json".format(folder)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
f.close()
sent_iter = sorted(pytextrank.limit_sentences(path_stage3,
word_limit=word_limit),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
s.append(pytextrank.make_sentence(sent_text))
graf_text = " ".join(s)
shutil.rmtree(folder)
return s
def rank_bill(bill):
bill_id = bill['bill_id']
with open(prefix + '/{}_stage1'.format(bill_id), 'w') as f:
for graf in parse_doc([bill]):
f.write(pretty_print(graf._asdict()))
f.write('\n')
path_stage1 = prefix + '/{}_stage1'.format(bill_id)
graph, ranks = text_rank(path_stage1)
render_ranks(graph, ranks)
for rl in normalize_key_phrases(path_stage1, ranks):
output = pretty_print(rl._asdict())
with open(prefix + '/{}_stage2'.format(bill_id), 'w') as f:
f.write(output)
path_stage1 = prefix + '/{}_stage1'.format(bill_id)
path_stage2 = prefix + '/{}_stage2'.format(bill_id)
kernel = rank_kernel(path_stage2)
with open(prefix + '/{}_stage3'.format(bill_id), 'w') as f:
for s in top_sentences(kernel, path_stage1):
f.write(pretty_print(s._asdict()))
def calculate_sentence_significance(self, paragraph_output, key_phrases_output, \
top_sentences_output, top_n_sentences):
"""
Calculate the significance of each sentence based on the ranking.
Ranking is determined by the top n sentences (filter)
Parameters
==========
paragraph_output:
tagged and parsed JSON document as text file
key_phrases_output:
text file (JSON) into which key phrases are stored
top_sentences_output:
text file (JSON) into which top scored sentences are written
top_n_sentences:
top n sentences to return based on scores
Return
======
Nothing, writes top n sentences into a text file (JSON) by the
name specified in top_sentences_output
"""
kernel = pytextrank.rank_kernel(key_phrases_output)
with open(top_sentences_output, 'w') as temp_file:
counter = 0
for sentence in pytextrank.top_sentences(kernel, paragraph_output):
if counter < top_n_sentences:
temp_file.write(pytextrank.pretty_print(
sentence._asdict()))
temp_file.write("\n")
else:
return
counter = counter + 1
path_stage1 = "o1.json"
path_stage2 = "o2.json"
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
#print(pytextrank.pretty_print(rl))
path_stage1 = "o1.json"
path_stage2 = "o2.json"
path_stage3 = "o3.json"
kernel = pytextrank.rank_kernel(path_stage2)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
# to view output in this notebook
print(pytextrank.pretty_print(s._asdict()))
path_stage2 = "o2.json"
path_stage3 = "o3.json"
phrases = ", ".join(
set([p
for p in pytextrank.limit_keyphrases(path_stage2, phrase_limit=20)]))
sent_iter = sorted(pytextrank.limit_sentences(path_stage3, word_limit=500),
def extract_phrasesfrom_textrank(corpus):
record_data = pd.DataFrame({'sentences': corpus})
record_data = pd.DataFrame({
'id': record_data.index.tolist(),
'text': record_data['sentences'].tolist()
})
tweet_items = []
for jdict in record_data.to_dict(orient='records'):
tweet_items.append(jdict)
new_df_tweet = pd.DataFrame(columns=['text', 'keywords'])
path_stage1 = "celebrity1_tweet.json"
path_stage2 = "celebrity2_tweet.json"
path_stage3 = "celebrity3_tweet.json"
for item in tweet_items:
items_new = [item]
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(items_new):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
kernel = pytextrank.rank_kernel(path_stage2)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
phrases = ", ".join(
set([
p for p in pytextrank.limit_keyphrases(path_stage2,
phrase_limit=5)
]))
sent_iter = sorted(pytextrank.limit_sentences(path_stage3,
word_limit=150),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
s.append(pytextrank.make_sentence(sent_text))
graf_text = " ".join(s)
new_df_tweet = new_df_tweet.append(
{
'text': item.get('text'),
'keywords': phrases
}, ignore_index=True)
celeb_list = [
'Bradley Cooper', 'Chris Kyle', 'Clint Eastwood', 'bradley cooper',
'bradley', 'cooper', 'chris kyle', 'chris', 'kyle', 'clint eastwood',
'clint', 'eastwood'
]
cleaned_df_tweet = pd.DataFrame(columns=['sentences', 'keywords'])
for index, row in new_df_tweet.iterrows():
if any(celeb in row['keywords'] for celeb in celeb_list):
cleaned_df_tweet = cleaned_df_tweet.append(
{
'sentences': row['text'],
'keywords': row['keywords']
},
ignore_index=True)
cleaned_df_tweet.to_csv(phrase_filepath,
sep=',',
encoding='utf-8',
index=False)
new_df_tweet.to_csv(all_phrasefile_path,
sep=',',
encoding='utf-8',
index=False)
return new_df_tweet, cleaned_df_tweet
from pytextrank import pretty_print, rank_kernel, top_sentences
import sys
## Stage 3:
## * calculate a significance weight for each sentence, using MinHash to
## * approximate Jaccard distance from key phrases determined by TextRank
##
## INPUTS: <stage1> <stage2>
## OUTPUT: JSON format `SummarySent(dist, idx, text)`
if __name__ == "__main__":
path_stage1 = sys.argv[1]
path_stage2 = sys.argv[2]
kernel = rank_kernel(path_stage2)
for s in top_sentences(kernel, path_stage1):
print(pretty_print(s._asdict()))
if len(rl_fake_json) == 0:
for file in find_files(stage_2_directory, "*.Stage2"):
if publisher in file and version in file: # ex. DC & paperback
rl_fake_json.extend(pickle.load(open(file, 'rb')))
else:
pass
pickle.dump(rl_fake_json, open(stage_2_out, 'wb'))
else:
rl_fake_json = pickle.load(open(stage_2_out, 'rb'))
#stage 3: get top sentences ("Calculate a significance weight for each sentence,
# using MinHash to approximate a Jaccard distance from key phrases determined by TextRank")
# -https://github.com/ceteri/pytextrank/blob/master/example.ipynb
kernel = pytextrank.rank_kernel(rl_fake_json)
sentences_fake_json = [] # stage 3 output
for s in pytextrank.top_sentences(kernel, fake_json_graph_dicts):
sentence_dict = s._asdict()
sentences_fake_json.append([sentence_dict])
print(pytextrank.pretty_print(sentence_dict))
stage_3_filename = "{publisher}_{version}_textRank.topSentences.".format(
version=version, publisher=publisher)
stage_3_out = os.path.join(directory, "Stage3Results",
"agglomerated", stage_3_filename)
pickle.dump(fake_json_graph_dicts, open(stage_3_out, 'wb'))
#stage 4: generate a summary of the entire set of books
phrases = ", ".join(
set([
with open(path_stage2, 'w') as f:
for rl in ptr.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % ptr.pretty_print(rl._asdict()))
print(ptr.pretty_print(rl))
# Stage 3
import networkx as nx
# import pylab as plt
nx.draw(graph, with_labels=True)
# plt.show()
path_stage3 = "../tests/pytextrank_dat/o3.json"
kernel = ptr.rank_kernel(path_stage2)
with open(path_stage3, 'w') as f:
for s in ptr.top_sentences(kernel, path_stage1):
f.write(ptr.pretty_print(s._asdict()))
f.write("\n")
print(ptr.pretty_print(s._asdict()))
# Stage 4
phrases = ", ".join(
set([p for p in ptr.limit_keyphrases(path_stage2, phrase_limit=12)]))
sent_iter = sorted(ptr.limit_sentences(path_stage3, word_limit=150),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
'minimal generating sets of solutions for all types of systems are given. ' + \
'These criteria and the corresponding algorithms for constructing a minimal ' + \
'supporting set of solutions can be used in solving all the considered types ' + \
'systems and systems of mixed types.'
someothertext = 'Amazon.com, Inc. is located in Seattle, WA and was founded July 5th, 1994 by Jeff Bezos, ' + \
'allowing customers to buy everything from books to blenders. Seattle is north of Portland and ' + \
'south of Vancouver, BC. Other notable Seattle - based companies are Starbucks and Boeing.'
docs = [{'text': sometext, 'id': 777}]
grafs = [{'graf': graf.graf} for graf in pytextrank.parse_doc(docs)]
graph, ranks = pytextrank.text_rank(grafs)
rank_list = [
rl._asdict()
for rl in pytextrank.normalize_key_phrases(grafs, ranks, skip_ner=False)
]
kernel = pytextrank.rank_kernel(rank_list)
sents = [s._asdict() for s in pytextrank.top_sentences(kernel, grafs)]
phrases = [
p[0] for p in pytextrank.limit_keyphrases(rank_list, phrase_limit=6)
]
sent_iter = sorted(pytextrank.limit_sentences(sents, word_limit=150),
key=lambda x: x[1])
sents = [pytextrank.make_sentence(sent_text) for sent_text, idx in sent_iter]
graf_text = ' '.join(sents)
print("\n**excerpts:** %s\n\n**keywords:** %s" % (
graf_text,
phrases,
))
#!/usr/bin/env python
# encoding: utf-8
from pytextrank import pretty_print, rank_kernel, top_sentences
import sys
## Stage 3:
## * calculate a significance weight for each sentence, using MinHash to
## * approximate Jaccard distance from key phrases determined by TextRank
##
## INPUTS: <stage1> <stage2>
## OUTPUT: JSON format `SummarySent(dist, idx, text)`
if __name__ == "__main__":
path_stage1 = sys.argv[1]
path_stage2 = sys.argv[2]
kernel = rank_kernel(path_stage2, force_encode=False)
for s in top_sentences(kernel, path_stage1, force_encode=False):
print(pretty_print(s._asdict()))
def insert_key_phrases_into_db(list_of_doc_dicts, doctype, collection):
'''
Takes in list of doc dictionaries and a doctype ('comment' or 'post'),
processes each doc with PyTextRank, obtains key phrases and
inserts key phrases into document in Mongodb as 'key_phrases' field.
'''
path_stage0 = 'stage0.json'
path_stage1 = 'stage1.json'
path_stage2 = 'stage2.json'
path_stage3 = 'stage3.json'
total_docs = len(list_of_doc_dicts)
failed_ids = []
for i, doc_dict in enumerate(list_of_doc_dicts):
if i % 50 == 0:
print(f'processing {i} of {total_docs} documents')
doc_dict['text'] = doc_dict['text'].split('\n_____\n\n')[0]
try:
with open(path_stage0, 'w') as f:
json.dump(doc_dict, f)
# Stage 1
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(
pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
# print(pytextrank.pretty_print(graf))
# Stage 2
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
# to view output in this notebook
# print(pytextrank.pretty_print(rl))
# Stage 3
kernel = pytextrank.rank_kernel(path_stage2)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
# to view output in this notebook
# print(pytextrank.pretty_print(s._asdict()))
# Stage 4
phrase_list = list(
set([
p for p in pytextrank.limit_keyphrases(path_stage2,
phrase_limit=15)
]))
phrases = ", ".join(phrase_list)
sent_iter = sorted(pytextrank.limit_sentences(path_stage3,
word_limit=150),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
s.append(pytextrank.make_sentence(sent_text))
graf_text = " ".join(s)
collection.update_one({f'{doctype}_id': {
'$eq': doc_dict['id']
}}, {'$set': {
'key_phrases': phrase_list
}})
except:
failed_ids.append(doc_dict['id'])
print('failed on ', doc_dict['id'])
continue
