def execute_stage_one(path_stage0):
path_stage1 = path_stage0.split('::')[0] + ".stage1.output.dat"
with open(os.path.join(PATH_PREFIX, path_stage1), 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(
os.path.join(PATH_PREFIX, path_stage0))):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
return path_stage1
def text_rank(json_request):
pattern = re.compile("TI - (.*?)\\r|AB - (.*?)\\r")
matches = re.findall(pattern, json_request['ris'])
all_inputs = []
for section in matches:
all_inputs.append((''.join([word + ' ' for word in section])).strip())
input_json = {}
input_json['id'] = "0"
input_json['text'] = '.'.join(all_inputs)
with open('ris_extracted.json', 'w') as output:
json.dump(input_json, output)
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter('ris_extracted.json')):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
graph, ranks = pytextrank.text_rank(path_stage1)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
phrases = list([p for p in pytextrank.limit_keyphrases(path_stage2, phrase_limit=20)])
return phrases
def generateGraph(text, outputfile, outputdir, plotGraph=False):
print('Generating Graph...')
#Start by doing statistical parsing/tagging for
temp_file = os.path.join(outputdir, 'temp.json')
path_stage1 = os.path.join(outputdir,
outputfile.split("_")[0] + '_o1.json')
txtToJson.textTojson(text, temp_file)
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(temp_file)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
#Collect and Normalize the key sentences from the parsed doc
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
#path_stage2 = path_stage1.replace('o1', 'o2')
path_stage2 = os.path.join(outputdir, outputfile)
try:
os.remove(outputfile)
except OSError:
pass
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
#print(pytextrank.pretty_print(rl))
try:
os.remove(temp_file)
except OSError:
pass
if plotGraph:
matplotlib.rcParams['figure.figsize'] = (15.0, 15.0)
networkx.draw_networkx(graph)
plt.show()
nx.draw(graph, with_labels=True)
plt.show()
def one(text):
path_stage0 = "tempfile.json"
path_stage1 = "o1.json"
path_stage2 = "o2.json"
f = open("tempfile.json", "w")
f.write("{\"id\":\"777\", \"text\":\"" + text + "\"}")
f.close()
with open(path_stage1, 'w') as f:
for graf in parse_doc(json_iter(path_stage0)):
f.write("%s\n" % pretty_print(graf._asdict()))
graph, ranks = text_rank(path_stage1)
render_ranks(graph, ranks)
outputs = []
with open(path_stage2, 'w') as f:
for rl in normalize_key_phrases(path_stage1, ranks):
ans = "%s\n" % pretty_print(rl._asdict())
output = ast.literal_eval(ans)
outputs.append((output["text"], output["rank"]))
os.remove("tempfile.json")
return outputs
# text = "The earliest recorded model for planetary motions proposed by Ptolemy about 2000 years ago was a ‘geocentric’ model in which all celestial objects, stars, the sun and the planets, all revolved around the earth."
# print (one("The earliest recorded model for planetary motions proposed by Ptolemy about 2000 years ago was a ‘geocentric’ model in which all celestial objects, stars, the sun and the planets, all revolved around the earth."))
def summarize(self, _id, content_text, word_limit):
self.logger.log("_id: " + _id)
self.logger.log("word_limit: " + str(word_limit))
# File names
path_stage0 = 'process/' + _id + '.json'
path_stage1 = 'process/' + _id + '_o1.json'
path_stage2 = 'process/' + _id + '_o2.json'
path_stage3 = 'process/' + _id + '_o3.json'
path_stage4 = 'process/' + _id + '_o4.json'
# Create input file
with open(path_stage0, 'w') as outfile:
json.dump({"id": "123", "text": content_text}, outfile)
# Statistical Parsing - Stage 1
# Perform statistical parsing/tagging on a document in JSON format
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(
pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
# Ranked Keyphrases - Stage 2
# Collect and normalize the key phrases from a parsed document
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
# Extractive Summarization - Stage 3
# Calculate a significance weight for each sentence, using MinHash to approximate a Jaccard distance from key phrases determined by TextRank
kernel = pytextrank.rank_kernel(path_stage2)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
# Final Output - Stage 4
# Summarize a document based on most significant sentences and key phrases
phrases = ", ".join(
set([
p for p in pytextrank.limit_keyphrases(path_stage2,
phrase_limit=12)
]))
sent_iter = sorted(pytextrank.limit_sentences(path_stage3,
word_limit=word_limit),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
s.append(pytextrank.make_sentence(sent_text))
graf_text = " ".join(s)
return {'excerpts': graf_text, 'keywords': phrases}
def pred_net(sample_case):
import numpy as np, keras
from pathlib import Path
from spacy import displacy
from PIL import Image
import json, pytextrank, networkx as nx
import matplotlib.pyplot as plt
path_stage0 = "o0.json"
path_stage1 = "o1.json"
file_dic = {"id": 0, "text": sample_case}
loaded_file_dic = json.loads(json.dumps(file_dic))
with open(path_stage0, 'w') as outfile:
json.dump(loaded_file_dic, outfile)
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
print(pytextrank.pretty_print(graf._asdict()))
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
nx.draw(graph, with_labels=True)
plt.savefig("sample_case.png", dpi=200, format='png', bbox_inches='tight')
plt.close()
im = Image.open("sample_case.png").convert('L').resize((300, 200))
sample_image = np.array([np.array(im)])
sample_image = sample_image.reshape(sample_image.shape[0],
sample_image.shape[1],
sample_image.shape[2], 1)
model = keras.models.load_model("graph_conv_autoencoder.hdf5")
y_pred = model.predict(sample_image)
labels = [
'Major Depressive Disorder',
'Attention Deficit Hyperactivity Disorder',
'Oppositional Defiant Disorder', 'Conduct Disorder',
'Pervasive Developmental Disorder',
'Intellectual Disability (Mental Retardation)', 'Psychotic Disorder',
'Adjustment Disorder', 'Mood Disorder', 'General Anxiety Disorder',
'Social Anxiety Disorder', 'Seasonal Affective Disorder',
'Substance Abuse', 'Autism Spectrum Disorder'
]
max1 = labels[np.argmax(y_pred)]
with open('external_resources.json') as data_file:
for v in json.load(data_file):
if v['diagnosis'] == max1:
about1, treatment1 = v['about'], v['treatment']
return (max1, about1, treatment1)
def perform_statistical_parsing_tagging(self, text_file, paragraph_output):
"""
Perform statistical parsing and tagging of
sentences in the text (aka JSON document)
"""
with open(paragraph_output, 'w') as temp_file:
for paragraph in pytextrank.parse_doc(pytextrank.json_iter(text_file)):
temp_file.write("%s\n" % pytextrank.pretty_print(paragraph._asdict()))
def pytrankSummarize(filename):
"""
This is another TextRank algorithm. It works in four stages, each feeding its output to the next
1. Part-of-Speech Tagging and lemmatization are performed for every sentence in the document.
2. Key phrases are extracted along with their counts, and are normalized.
3. Calculates a score for each sentence by approximating jaccard distance between the sentence and key phrases.
4. Summarizes the document based on most significant sentences and key phrases.
"""
import pytextrank
jsonText = createJSON(filename)
path_stage0 = jsonText
path_stage1 = "o1.json"
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
path_stage2 = "o2.json"
graph, ranks = pytextrank.text_rank(path_stage1)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
path_stage3 = "o3.json"
kernel = pytextrank.rank_kernel(path_stage2)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
phrases = ", ".join(
set([
p
for p in pytextrank.limit_keyphrases(path_stage2, phrase_limit=12)
]))
sent_iter = sorted(pytextrank.limit_sentences(path_stage3, word_limit=50),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
s.append(pytextrank.make_sentence(sent_text))
graf_text = " ".join(s)
print("")
print("####### From PyTextRank #######")
print("**excerpts:** %s\n\n**keywords:** %s" % (
graf_text,
phrases,
))
def summarize_text(input_file):
# seriously f**k this API
path_stage0 = input_file
path_stage1 = 'stage1.txt'
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
# to view output in this notebook
#print(pytextrank.pretty_print(graf))
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
path_stage2 = 'stage2.txt'
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
# to view output in this notebook
#print(pytextrank.pretty_print(rl))
path_stage3 = 'stage3.txt'
kernel = pytextrank.rank_kernel(path_stage2)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
# to view output in this notebook
#print(pytextrank.pretty_print(s._asdict()))
phrases = ", ".join(
set([
p
for p in pytextrank.limit_keyphrases(path_stage2, phrase_limit=12)
]))
sent_iter = sorted(pytextrank.limit_sentences(path_stage3, word_limit=120),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
s.append(pytextrank.make_sentence(sent_text))
graf_text = " ".join(s)
#print("**excerpts:** %s\n\n**keywords:** %s" % (graf_text, phrases,))
return ' '.join(s)
def _get_keywords(path_stage0, path_stage2):
# Stage 1: parse doc
path_stage1 = 'o1.json'
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
# Stage 2: rank words
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
result_dict = dict()
with open(path_stage2, 'w') as f2:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
_ro = rl._asdict()
ro = dict()
ro[_ro['text']] = _ro['rank']
#f2.write("%s\n" % pytextrank.pretty_print(ro))
result_dict[_ro['text']] = _ro['rank']
return result_dict
def perform_statistical_parsing_tagging(self, text_file, paragraph_output):
"""
Perform statistical parsing and tagging of
sentences in the text (aka JSON document)
Parameters
==========
text_file:
file containing the input text (as JSON) to perform
statistical parsing and tagging on
paragraph_output:
output file into which the results are written (as a JSON file)
Return
======
Nothing, writes results to a text file (as JSON)
"""
with open(paragraph_output, 'w') as temp_file:
for paragraph in pytextrank.parse_doc(
pytextrank.json_iter(text_file)):
temp_file.write("%s\n" %
pytextrank.pretty_print(paragraph._asdict()))
def retrieveSentences(content, word_limit):
currpath = os.getcwd()
folder = os.path.join(currpath, str(uuid.uuid4()))
os.mkdir(folder)
fname = str(uuid.uuid4())
with open("{0}/{1}.json".format(folder, fname), "w") as f:
f.write(json.dumps({"id": fname, "text": content}))
f.close()
path_stage0 = "{0}/{1}.json".format(folder, fname)
path_stage1 = "{0}/o1.json".format(folder)
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
f.close()
path_stage2 = "{0}/o2.json".format(folder)
graph, ranks = pytextrank.text_rank(path_stage1)
#pytextrank.render_ranks(graph, ranks)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
f.close()
kernel = pytextrank.rank_kernel(path_stage2)
path_stage3 = "{0}/o3.json".format(folder)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
f.close()
sent_iter = sorted(pytextrank.limit_sentences(path_stage3,
word_limit=word_limit),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
s.append(pytextrank.make_sentence(sent_text))
graf_text = " ".join(s)
shutil.rmtree(folder)
return s
def insert_key_phrases_into_db(list_of_doc_dicts, doctype, collection):
'''
Takes in list of doc dictionaries and a doctype ('comment' or 'post'),
processes each doc with PyTextRank, obtains key phrases and
inserts key phrases into document in Mongodb as 'key_phrases' field.
'''
path_stage0 = 'stage0.json'
path_stage1 = 'stage1.json'
path_stage2 = 'stage2.json'
path_stage3 = 'stage3.json'
total_docs = len(list_of_doc_dicts)
failed_ids = []
for i, doc_dict in enumerate(list_of_doc_dicts):
if i % 50 == 0:
print(f'processing {i} of {total_docs} documents')
doc_dict['text'] = doc_dict['text'].split('\n_____\n\n')[0]
try:
with open(path_stage0, 'w') as f:
json.dump(doc_dict, f)
# Stage 1
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(
pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
# print(pytextrank.pretty_print(graf))
# Stage 2
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
# to view output in this notebook
# print(pytextrank.pretty_print(rl))
# Stage 3
kernel = pytextrank.rank_kernel(path_stage2)
with open(path_stage3, 'w') as f:
for s in pytextrank.top_sentences(kernel, path_stage1):
f.write(pytextrank.pretty_print(s._asdict()))
f.write("\n")
# to view output in this notebook
# print(pytextrank.pretty_print(s._asdict()))
# Stage 4
phrase_list = list(
set([
p for p in pytextrank.limit_keyphrases(path_stage2,
phrase_limit=15)
]))
phrases = ", ".join(phrase_list)
sent_iter = sorted(pytextrank.limit_sentences(path_stage3,
word_limit=150),
key=lambda x: x[1])
s = []
for sent_text, idx in sent_iter:
s.append(pytextrank.make_sentence(sent_text))
graf_text = " ".join(s)
collection.update_one({f'{doctype}_id': {
'$eq': doc_dict['id']
}}, {'$set': {
'key_phrases': phrase_list
}})
except:
failed_ids.append(doc_dict['id'])
print('failed on ', doc_dict['id'])
continue
# Create dictionary to feed into json file
file_dic = {"id": 0, "text": sample_text}
file_dic = json.dumps(file_dic)
loaded_file_dic = json.loads(file_dic)
# Create test.json and feed file_dic into it.
with open('test.json', 'w') as outfile:
json.dump(loaded_file_dic, outfile)
path_stage0 = "test.json"
path_stage1 = "o1.json"
# Extract keyword using pytextrank
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
#print(pytextrank.pretty_print(graf._asdict()))
path_stage1 = "o1.json"
path_stage2 = "o2.json"
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
#print(pytextrank.pretty_print(rl))
path_stage1 = "o1.json"
#!/Users/redhouaneabdellaoui/anaconda/envs/surya/bin/python python
# -*- coding: utf-8 -*-
import pytextrank as ptr
# Stage 1
path_stage0 = "../tests/pytextrank_dat/mih.json"
path_stage1 = "../tests/pytextrank_dat/o1.json"
with open(path_stage1, 'w') as f:
for graf in ptr.parse_doc(ptr.json_iter(path_stage0)):
f.write("%s\n" % ptr.pretty_print(graf._asdict()))
print(ptr.pretty_print(graf))
# Stage 2
path_stage2 = "../tests/pytextrank_dat/o2.json"
graph, ranks = ptr.text_rank(path_stage1)
ptr.render_ranks(graph, ranks)
with open(path_stage2, 'w') as f:
for rl in ptr.normalize_key_phrases(path_stage1, ranks):
f.write("%s\n" % ptr.pretty_print(rl._asdict()))
print(ptr.pretty_print(rl))
# Stage 3
import networkx as nx
# import pylab as plt
nx.draw(graph, with_labels=True)
def stage1(path_stage0, path_stage1):
#Stage 1
with open(path_stage1, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
import pytextrank
import sys
import os
path_dir = "/Users/masterman/Downloads/pytextrank-master/"
path_stage0 = path_dir + "dat/mih.json"
path_stage1 = path_dir + "o1.json"
with open(path_stage1, 'w') as f:
grafs = pytextrank.parse_doc(pytextrank.json_iter(path_stage0))
for graf in grafs:
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
# to view output in this notebook
# print(pytextrank.pretty_print(graf))
# path_stage1 = path_dir + "o1.json"
path_stage2 = path_dir + "o2.json"
graph, ranks = pytextrank.text_rank(grafs)
pytextrank.render_ranks(graph, ranks)
with open(path_stage2, 'w') as f:
for rl in pytextrank.normalize_key_phrases(grafs, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
# to view output in this notebook
print(pytextrank.pretty_print(rl))
import networkx as nx
import pylab as plt
nx.draw(graph, with_labels=True)
def parseSentence():
with open('temp2.json', 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter('temp1.json')):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
from pytextrank import json_iter, parse_doc, pretty_print
import sys
## Stage 1:
## * perform statistical parsing/tagging on a document in JSON format
##
## INPUTS: <stage0>
## OUTPUT: JSON format `ParsedGraf(id, sha1, graf)`
if __name__ == "__main__":
path_stage0 = sys.argv[1]
for graf in parse_doc(json_iter(path_stage0)):
print(pretty_print(graf._asdict()))
def perform_statistical_parsing_tagging(self, text_file, paragraph_output):
with open(paragraph_output, 'w') as f:
for paragraph in pytextrank.parse_doc(
pytextrank.json_iter(text_file)):
f.write("%s\n" % pytextrank.pretty_print(paragraph._asdict()))
def do_pytextrank(data):
for item in data:
for subItem in data[item]:
print('###############')
print('description:', subItem['description'])
# using pytextrank
# reference https://github.com/ceteri/pytextrank/issues/18
# raw input
subItemJSON = {'id': subItem['id'], 'text': subItem['description']}
subItemJSON = json.dumps(subItemJSON)
with open('sub_item.json', 'w') as outFile:
outFile.write(subItemJSON)
# stage 1
with open('stage1_output.json', 'w') as outFile:
for graf in pytextrank.parse_doc(
pytextrank.json_iter('sub_item.json')):
outFile.write("%s\n" %
pytextrank.pretty_print(graf._asdict()))
# stage 2
graph, ranks = pytextrank.text_rank('stage1_output.json')
pytextrank.render_ranks(graph, ranks)
rlLists = []
print('key phrases:')
with open('stage2_output.json', 'w') as outFile:
for rl in pytextrank.normalize_key_phrases(
'stage1_output.json', ranks):
rlList = eval(pytextrank.pretty_print(rl))
rlLists.append(rlList)
print(rlList)
# cleanup
os.system(
'rm -f sub_item.json stage1_output.json stage2_output.json graph.dot'
)
# input filter results based on pos
# this is a heuristic
filteredRlLists = [x for x in rlLists if 'nn' not in x[-2]]
if (len(filteredRlLists) == 0):
# invalid case
continue
else:
[heuristic, iOItem] = do_heuristic(subItem, filteredRlLists)
print('heuristic:', heuristic)
print('i/o input:', iOItem)
# input filter results based on pos
# this is a heuristic
filteredRlLists = [x for x in rlLists if 'nn' in x[-2]]
if (len(filteredRlLists) == 0):
# invalid case
continue
else:
[heuristic, iOItem] = do_heuristic(subItem, filteredRlLists)
print('heuristic:', heuristic)
print('i/o output:', iOItem)
print('###############')
#!/usr/bin/env python
# encoding: utf-8
from pytextrank import json_iter, parse_doc, pretty_print
import sys
## Stage 1:
## * perform statistical parsing/tagging on a document in JSON format
##
## INPUTS: <stage0>
## OUTPUT: JSON format `ParsedGraf(id, sha1, graf)`
if __name__ == "__main__":
path_stage0 = sys.argv[1]
for graf in parse_doc(json_iter(path_stage0), force_encode=False):
print(pretty_print(graf._asdict()))
if os.path.exists(OUTOUT_DIRECTORY):
shutil.rmtree(OUTOUT_DIRECTORY)
os.makedirs(OUTOUT_DIRECTORY)
print('Saving tweets to json...')
with open(TWEETS_JSON, 'w', encoding='utf8') as outfile:
json.dump({'id': '777', 'text': '. '.join(user_tweets)}, outfile, ensure_ascii=False)
print('Saving tweets to json - Done')
print('Performing statistical parsing/tagging on tweets...')
with open(STATISTICAL_PARSING_OUTPUT, 'w') as f:
for graf in pytextrank.parse_doc(pytextrank.json_iter(TWEETS_JSON)):
f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
print('Performing statistical parsing/tagging on tweets... - Done')
print('Collect and normalizing the key phrases from the parsed document...')
graph, ranks = pytextrank.text_rank(STATISTICAL_PARSING_OUTPUT)
pytextrank.render_ranks(graph, ranks)
with open(KEY_PHRASES_NORMALIZATION_OUTPUT, 'w') as f:
for rl in pytextrank.normalize_key_phrases(STATISTICAL_PARSING_OUTPUT, ranks):
f.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
print('Collect and normalizing the key phrases from the parsed document... - Done')
print("Summarizing tweets based on key phrases...")
phrases = ", ".join(set([p for p in pytextrank.limit_keyphrases(KEY_PHRASES_NORMALIZATION_OUTPUT, phrase_limit=MAX_SUBJECTS_TO_SHOW)]))
import pytextrank
import sys
path_in = "input/in.json"
path_stage1 = "test_data/s1.json"
path_stage2 = "test_data/s2.json"
path_stage3 = "test_data/s3.json"
# stage1 : generate graph
with open(path_stage1, 'w') as file:
json_iterator = pytextrank.json_iter(path_in)
for g in pytextrank.parse_doc(json_iterator):
file.write("%s\n" % pytextrank.pretty_print(g._asdict()))
#print(pytextrank.pretty_print(g))
# stage2 : key words
graph, ranks = pytextrank.text_rank(path_stage1)
pytextrank.render_ranks(graph, ranks)
with open(path_stage2, 'w') as file:
for rl in pytextrank.normalize_key_phrases(path_stage1, ranks):
file.write("%s\n" % pytextrank.pretty_print(rl._asdict()))
#print(pytextrank.pretty_print(rl))
# # stage3 : summarize sentence
kernel = pytextrank.rank_kernel(path_stage2)
with open(path_stage3, 'w') as file:
for s in pytextrank.top_sentences(kernel, path_stage1):
file.write(pytextrank.pretty_print(s._asdict()))
file.write("\n")
#print(pytextrank.pretty_print(s._asdict()))
# stage4 : summarize based on most significant sentence & keywords
