def execute_stage_one(path_stage0): path_stage1 = path_stage0.split('::')[0] + ".stage1.output.dat" with open(os.path.join(PATH_PREFIX, path_stage1), 'w') as f: for graf in pytextrank.parse_doc(pytextrank.json_iter( os.path.join(PATH_PREFIX, path_stage0))): f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) return path_stage1
def text_rank(json_request): pattern = re.compile("TI - (.*?)\\r|AB - (.*?)\\r") matches = re.findall(pattern, json_request['ris']) all_inputs = [] for section in matches: all_inputs.append((''.join([word + ' ' for word in section])).strip()) input_json = {} input_json['id'] = "0" input_json['text'] = '.'.join(all_inputs) with open('ris_extracted.json', 'w') as output: json.dump(input_json, output) with open(path_stage1, 'w') as f: for graf in pytextrank.parse_doc(pytextrank.json_iter('ris_extracted.json')): f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) graph, ranks = pytextrank.text_rank(path_stage1) with open(path_stage2, 'w') as f: for rl in pytextrank.normalize_key_phrases(path_stage1, ranks): f.write("%s\n" % pytextrank.pretty_print(rl._asdict())) phrases = list([p for p in pytextrank.limit_keyphrases(path_stage2, phrase_limit=20)]) return phrases
def generateGraph(text, outputfile, outputdir, plotGraph=False): print('Generating Graph...') #Start by doing statistical parsing/tagging for temp_file = os.path.join(outputdir, 'temp.json') path_stage1 = os.path.join(outputdir, outputfile.split("_")[0] + '_o1.json') txtToJson.textTojson(text, temp_file) with open(path_stage1, 'w') as f: for graf in pytextrank.parse_doc(pytextrank.json_iter(temp_file)): f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) #Collect and Normalize the key sentences from the parsed doc graph, ranks = pytextrank.text_rank(path_stage1) pytextrank.render_ranks(graph, ranks) #path_stage2 = path_stage1.replace('o1', 'o2') path_stage2 = os.path.join(outputdir, outputfile) try: os.remove(outputfile) except OSError: pass with open(path_stage2, 'w') as f: for rl in pytextrank.normalize_key_phrases(path_stage1, ranks): f.write("%s\n" % pytextrank.pretty_print(rl._asdict())) #print(pytextrank.pretty_print(rl)) try: os.remove(temp_file) except OSError: pass if plotGraph: matplotlib.rcParams['figure.figsize'] = (15.0, 15.0) networkx.draw_networkx(graph) plt.show() nx.draw(graph, with_labels=True) plt.show()
def one(text): path_stage0 = "tempfile.json" path_stage1 = "o1.json" path_stage2 = "o2.json" f = open("tempfile.json", "w") f.write("{\"id\":\"777\", \"text\":\"" + text + "\"}") f.close() with open(path_stage1, 'w') as f: for graf in parse_doc(json_iter(path_stage0)): f.write("%s\n" % pretty_print(graf._asdict())) graph, ranks = text_rank(path_stage1) render_ranks(graph, ranks) outputs = [] with open(path_stage2, 'w') as f: for rl in normalize_key_phrases(path_stage1, ranks): ans = "%s\n" % pretty_print(rl._asdict()) output = ast.literal_eval(ans) outputs.append((output["text"], output["rank"])) os.remove("tempfile.json") return outputs # text = "The earliest recorded model for planetary motions proposed by Ptolemy about 2000 years ago was a ‘geocentric’ model in which all celestial objects, stars, the sun and the planets, all revolved around the earth." # print (one("The earliest recorded model for planetary motions proposed by Ptolemy about 2000 years ago was a ‘geocentric’ model in which all celestial objects, stars, the sun and the planets, all revolved around the earth."))
def summarize(self, _id, content_text, word_limit): self.logger.log("_id: " + _id) self.logger.log("word_limit: " + str(word_limit)) # File names path_stage0 = 'process/' + _id + '.json' path_stage1 = 'process/' + _id + '_o1.json' path_stage2 = 'process/' + _id + '_o2.json' path_stage3 = 'process/' + _id + '_o3.json' path_stage4 = 'process/' + _id + '_o4.json' # Create input file with open(path_stage0, 'w') as outfile: json.dump({"id": "123", "text": content_text}, outfile) # Statistical Parsing - Stage 1 # Perform statistical parsing/tagging on a document in JSON format with open(path_stage1, 'w') as f: for graf in pytextrank.parse_doc( pytextrank.json_iter(path_stage0)): f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) # Ranked Keyphrases - Stage 2 # Collect and normalize the key phrases from a parsed document graph, ranks = pytextrank.text_rank(path_stage1) pytextrank.render_ranks(graph, ranks) with open(path_stage2, 'w') as f: for rl in pytextrank.normalize_key_phrases(path_stage1, ranks): f.write("%s\n" % pytextrank.pretty_print(rl._asdict())) # Extractive Summarization - Stage 3 # Calculate a significance weight for each sentence, using MinHash to approximate a Jaccard distance from key phrases determined by TextRank kernel = pytextrank.rank_kernel(path_stage2) with open(path_stage3, 'w') as f: for s in pytextrank.top_sentences(kernel, path_stage1): f.write(pytextrank.pretty_print(s._asdict())) f.write("\n") # Final Output - Stage 4 # Summarize a document based on most significant sentences and key phrases phrases = ", ".join( set([ p for p in pytextrank.limit_keyphrases(path_stage2, phrase_limit=12) ])) sent_iter = sorted(pytextrank.limit_sentences(path_stage3, word_limit=word_limit), key=lambda x: x[1]) s = [] for sent_text, idx in sent_iter: s.append(pytextrank.make_sentence(sent_text)) graf_text = " ".join(s) return {'excerpts': graf_text, 'keywords': phrases}
def pred_net(sample_case): import numpy as np, keras from pathlib import Path from spacy import displacy from PIL import Image import json, pytextrank, networkx as nx import matplotlib.pyplot as plt path_stage0 = "o0.json" path_stage1 = "o1.json" file_dic = {"id": 0, "text": sample_case} loaded_file_dic = json.loads(json.dumps(file_dic)) with open(path_stage0, 'w') as outfile: json.dump(loaded_file_dic, outfile) with open(path_stage1, 'w') as f: for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)): f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) print(pytextrank.pretty_print(graf._asdict())) graph, ranks = pytextrank.text_rank(path_stage1) pytextrank.render_ranks(graph, ranks) nx.draw(graph, with_labels=True) plt.savefig("sample_case.png", dpi=200, format='png', bbox_inches='tight') plt.close() im = Image.open("sample_case.png").convert('L').resize((300, 200)) sample_image = np.array([np.array(im)]) sample_image = sample_image.reshape(sample_image.shape[0], sample_image.shape[1], sample_image.shape[2], 1) model = keras.models.load_model("graph_conv_autoencoder.hdf5") y_pred = model.predict(sample_image) labels = [ 'Major Depressive Disorder', 'Attention Deficit Hyperactivity Disorder', 'Oppositional Defiant Disorder', 'Conduct Disorder', 'Pervasive Developmental Disorder', 'Intellectual Disability (Mental Retardation)', 'Psychotic Disorder', 'Adjustment Disorder', 'Mood Disorder', 'General Anxiety Disorder', 'Social Anxiety Disorder', 'Seasonal Affective Disorder', 'Substance Abuse', 'Autism Spectrum Disorder' ] max1 = labels[np.argmax(y_pred)] with open('external_resources.json') as data_file: for v in json.load(data_file): if v['diagnosis'] == max1: about1, treatment1 = v['about'], v['treatment'] return (max1, about1, treatment1)
def perform_statistical_parsing_tagging(self, text_file, paragraph_output): """ Perform statistical parsing and tagging of sentences in the text (aka JSON document) """ with open(paragraph_output, 'w') as temp_file: for paragraph in pytextrank.parse_doc(pytextrank.json_iter(text_file)): temp_file.write("%s\n" % pytextrank.pretty_print(paragraph._asdict()))
def pytrankSummarize(filename): """ This is another TextRank algorithm. It works in four stages, each feeding its output to the next 1. Part-of-Speech Tagging and lemmatization are performed for every sentence in the document. 2. Key phrases are extracted along with their counts, and are normalized. 3. Calculates a score for each sentence by approximating jaccard distance between the sentence and key phrases. 4. Summarizes the document based on most significant sentences and key phrases. """ import pytextrank jsonText = createJSON(filename) path_stage0 = jsonText path_stage1 = "o1.json" with open(path_stage1, 'w') as f: for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)): f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) path_stage2 = "o2.json" graph, ranks = pytextrank.text_rank(path_stage1) with open(path_stage2, 'w') as f: for rl in pytextrank.normalize_key_phrases(path_stage1, ranks): f.write("%s\n" % pytextrank.pretty_print(rl._asdict())) path_stage3 = "o3.json" kernel = pytextrank.rank_kernel(path_stage2) with open(path_stage3, 'w') as f: for s in pytextrank.top_sentences(kernel, path_stage1): f.write(pytextrank.pretty_print(s._asdict())) f.write("\n") phrases = ", ".join( set([ p for p in pytextrank.limit_keyphrases(path_stage2, phrase_limit=12) ])) sent_iter = sorted(pytextrank.limit_sentences(path_stage3, word_limit=50), key=lambda x: x[1]) s = [] for sent_text, idx in sent_iter: s.append(pytextrank.make_sentence(sent_text)) graf_text = " ".join(s) print("") print("####### From PyTextRank #######") print("**excerpts:** %s\n\n**keywords:** %s" % ( graf_text, phrases, ))
def summarize_text(input_file): # seriously f**k this API path_stage0 = input_file path_stage1 = 'stage1.txt' with open(path_stage1, 'w') as f: for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)): f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) # to view output in this notebook #print(pytextrank.pretty_print(graf)) graph, ranks = pytextrank.text_rank(path_stage1) pytextrank.render_ranks(graph, ranks) path_stage2 = 'stage2.txt' with open(path_stage2, 'w') as f: for rl in pytextrank.normalize_key_phrases(path_stage1, ranks): f.write("%s\n" % pytextrank.pretty_print(rl._asdict())) # to view output in this notebook #print(pytextrank.pretty_print(rl)) path_stage3 = 'stage3.txt' kernel = pytextrank.rank_kernel(path_stage2) with open(path_stage3, 'w') as f: for s in pytextrank.top_sentences(kernel, path_stage1): f.write(pytextrank.pretty_print(s._asdict())) f.write("\n") # to view output in this notebook #print(pytextrank.pretty_print(s._asdict())) phrases = ", ".join( set([ p for p in pytextrank.limit_keyphrases(path_stage2, phrase_limit=12) ])) sent_iter = sorted(pytextrank.limit_sentences(path_stage3, word_limit=120), key=lambda x: x[1]) s = [] for sent_text, idx in sent_iter: s.append(pytextrank.make_sentence(sent_text)) graf_text = " ".join(s) #print("**excerpts:** %s\n\n**keywords:** %s" % (graf_text, phrases,)) return ' '.join(s)
def _get_keywords(path_stage0, path_stage2): # Stage 1: parse doc path_stage1 = 'o1.json' with open(path_stage1, 'w') as f: for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)): f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) # Stage 2: rank words graph, ranks = pytextrank.text_rank(path_stage1) pytextrank.render_ranks(graph, ranks) result_dict = dict() with open(path_stage2, 'w') as f2: for rl in pytextrank.normalize_key_phrases(path_stage1, ranks): _ro = rl._asdict() ro = dict() ro[_ro['text']] = _ro['rank'] #f2.write("%s\n" % pytextrank.pretty_print(ro)) result_dict[_ro['text']] = _ro['rank'] return result_dict
def perform_statistical_parsing_tagging(self, text_file, paragraph_output): """ Perform statistical parsing and tagging of sentences in the text (aka JSON document) Parameters ========== text_file: file containing the input text (as JSON) to perform statistical parsing and tagging on paragraph_output: output file into which the results are written (as a JSON file) Return ====== Nothing, writes results to a text file (as JSON) """ with open(paragraph_output, 'w') as temp_file: for paragraph in pytextrank.parse_doc( pytextrank.json_iter(text_file)): temp_file.write("%s\n" % pytextrank.pretty_print(paragraph._asdict()))
def retrieveSentences(content, word_limit): currpath = os.getcwd() folder = os.path.join(currpath, str(uuid.uuid4())) os.mkdir(folder) fname = str(uuid.uuid4()) with open("{0}/{1}.json".format(folder, fname), "w") as f: f.write(json.dumps({"id": fname, "text": content})) f.close() path_stage0 = "{0}/{1}.json".format(folder, fname) path_stage1 = "{0}/o1.json".format(folder) with open(path_stage1, 'w') as f: for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)): f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) f.close() path_stage2 = "{0}/o2.json".format(folder) graph, ranks = pytextrank.text_rank(path_stage1) #pytextrank.render_ranks(graph, ranks) with open(path_stage2, 'w') as f: for rl in pytextrank.normalize_key_phrases(path_stage1, ranks): f.write("%s\n" % pytextrank.pretty_print(rl._asdict())) f.close() kernel = pytextrank.rank_kernel(path_stage2) path_stage3 = "{0}/o3.json".format(folder) with open(path_stage3, 'w') as f: for s in pytextrank.top_sentences(kernel, path_stage1): f.write(pytextrank.pretty_print(s._asdict())) f.write("\n") f.close() sent_iter = sorted(pytextrank.limit_sentences(path_stage3, word_limit=word_limit), key=lambda x: x[1]) s = [] for sent_text, idx in sent_iter: s.append(pytextrank.make_sentence(sent_text)) graf_text = " ".join(s) shutil.rmtree(folder) return s
def insert_key_phrases_into_db(list_of_doc_dicts, doctype, collection): ''' Takes in list of doc dictionaries and a doctype ('comment' or 'post'), processes each doc with PyTextRank, obtains key phrases and inserts key phrases into document in Mongodb as 'key_phrases' field. ''' path_stage0 = 'stage0.json' path_stage1 = 'stage1.json' path_stage2 = 'stage2.json' path_stage3 = 'stage3.json' total_docs = len(list_of_doc_dicts) failed_ids = [] for i, doc_dict in enumerate(list_of_doc_dicts): if i % 50 == 0: print(f'processing {i} of {total_docs} documents') doc_dict['text'] = doc_dict['text'].split('\n_____\n\n')[0] try: with open(path_stage0, 'w') as f: json.dump(doc_dict, f) # Stage 1 with open(path_stage1, 'w') as f: for graf in pytextrank.parse_doc( pytextrank.json_iter(path_stage0)): f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) # print(pytextrank.pretty_print(graf)) # Stage 2 graph, ranks = pytextrank.text_rank(path_stage1) pytextrank.render_ranks(graph, ranks) with open(path_stage2, 'w') as f: for rl in pytextrank.normalize_key_phrases(path_stage1, ranks): f.write("%s\n" % pytextrank.pretty_print(rl._asdict())) # to view output in this notebook # print(pytextrank.pretty_print(rl)) # Stage 3 kernel = pytextrank.rank_kernel(path_stage2) with open(path_stage3, 'w') as f: for s in pytextrank.top_sentences(kernel, path_stage1): f.write(pytextrank.pretty_print(s._asdict())) f.write("\n") # to view output in this notebook # print(pytextrank.pretty_print(s._asdict())) # Stage 4 phrase_list = list( set([ p for p in pytextrank.limit_keyphrases(path_stage2, phrase_limit=15) ])) phrases = ", ".join(phrase_list) sent_iter = sorted(pytextrank.limit_sentences(path_stage3, word_limit=150), key=lambda x: x[1]) s = [] for sent_text, idx in sent_iter: s.append(pytextrank.make_sentence(sent_text)) graf_text = " ".join(s) collection.update_one({f'{doctype}_id': { '$eq': doc_dict['id'] }}, {'$set': { 'key_phrases': phrase_list }}) except: failed_ids.append(doc_dict['id']) print('failed on ', doc_dict['id']) continue
# Create dictionary to feed into json file file_dic = {"id": 0, "text": sample_text} file_dic = json.dumps(file_dic) loaded_file_dic = json.loads(file_dic) # Create test.json and feed file_dic into it. with open('test.json', 'w') as outfile: json.dump(loaded_file_dic, outfile) path_stage0 = "test.json" path_stage1 = "o1.json" # Extract keyword using pytextrank with open(path_stage1, 'w') as f: for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)): f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) #print(pytextrank.pretty_print(graf._asdict())) path_stage1 = "o1.json" path_stage2 = "o2.json" graph, ranks = pytextrank.text_rank(path_stage1) pytextrank.render_ranks(graph, ranks) with open(path_stage2, 'w') as f: for rl in pytextrank.normalize_key_phrases(path_stage1, ranks): f.write("%s\n" % pytextrank.pretty_print(rl._asdict())) #print(pytextrank.pretty_print(rl)) path_stage1 = "o1.json"
#!/Users/redhouaneabdellaoui/anaconda/envs/surya/bin/python python # -*- coding: utf-8 -*- import pytextrank as ptr # Stage 1 path_stage0 = "../tests/pytextrank_dat/mih.json" path_stage1 = "../tests/pytextrank_dat/o1.json" with open(path_stage1, 'w') as f: for graf in ptr.parse_doc(ptr.json_iter(path_stage0)): f.write("%s\n" % ptr.pretty_print(graf._asdict())) print(ptr.pretty_print(graf)) # Stage 2 path_stage2 = "../tests/pytextrank_dat/o2.json" graph, ranks = ptr.text_rank(path_stage1) ptr.render_ranks(graph, ranks) with open(path_stage2, 'w') as f: for rl in ptr.normalize_key_phrases(path_stage1, ranks): f.write("%s\n" % ptr.pretty_print(rl._asdict())) print(ptr.pretty_print(rl)) # Stage 3 import networkx as nx # import pylab as plt nx.draw(graph, with_labels=True)
def stage1(path_stage0, path_stage1): #Stage 1 with open(path_stage1, 'w') as f: for graf in pytextrank.parse_doc(pytextrank.json_iter(path_stage0)): f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
import pytextrank import sys import os path_dir = "/Users/masterman/Downloads/pytextrank-master/" path_stage0 = path_dir + "dat/mih.json" path_stage1 = path_dir + "o1.json" with open(path_stage1, 'w') as f: grafs = pytextrank.parse_doc(pytextrank.json_iter(path_stage0)) for graf in grafs: f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) # to view output in this notebook # print(pytextrank.pretty_print(graf)) # path_stage1 = path_dir + "o1.json" path_stage2 = path_dir + "o2.json" graph, ranks = pytextrank.text_rank(grafs) pytextrank.render_ranks(graph, ranks) with open(path_stage2, 'w') as f: for rl in pytextrank.normalize_key_phrases(grafs, ranks): f.write("%s\n" % pytextrank.pretty_print(rl._asdict())) # to view output in this notebook print(pytextrank.pretty_print(rl)) import networkx as nx import pylab as plt nx.draw(graph, with_labels=True)
def parseSentence(): with open('temp2.json', 'w') as f: for graf in pytextrank.parse_doc(pytextrank.json_iter('temp1.json')): f.write("%s\n" % pytextrank.pretty_print(graf._asdict()))
from pytextrank import json_iter, parse_doc, pretty_print import sys ## Stage 1: ## * perform statistical parsing/tagging on a document in JSON format ## ## INPUTS: <stage0> ## OUTPUT: JSON format `ParsedGraf(id, sha1, graf)` if __name__ == "__main__": path_stage0 = sys.argv[1] for graf in parse_doc(json_iter(path_stage0)): print(pretty_print(graf._asdict()))
def perform_statistical_parsing_tagging(self, text_file, paragraph_output): with open(paragraph_output, 'w') as f: for paragraph in pytextrank.parse_doc( pytextrank.json_iter(text_file)): f.write("%s\n" % pytextrank.pretty_print(paragraph._asdict()))
def do_pytextrank(data): for item in data: for subItem in data[item]: print('###############') print('description:', subItem['description']) # using pytextrank # reference https://github.com/ceteri/pytextrank/issues/18 # raw input subItemJSON = {'id': subItem['id'], 'text': subItem['description']} subItemJSON = json.dumps(subItemJSON) with open('sub_item.json', 'w') as outFile: outFile.write(subItemJSON) # stage 1 with open('stage1_output.json', 'w') as outFile: for graf in pytextrank.parse_doc( pytextrank.json_iter('sub_item.json')): outFile.write("%s\n" % pytextrank.pretty_print(graf._asdict())) # stage 2 graph, ranks = pytextrank.text_rank('stage1_output.json') pytextrank.render_ranks(graph, ranks) rlLists = [] print('key phrases:') with open('stage2_output.json', 'w') as outFile: for rl in pytextrank.normalize_key_phrases( 'stage1_output.json', ranks): rlList = eval(pytextrank.pretty_print(rl)) rlLists.append(rlList) print(rlList) # cleanup os.system( 'rm -f sub_item.json stage1_output.json stage2_output.json graph.dot' ) # input filter results based on pos # this is a heuristic filteredRlLists = [x for x in rlLists if 'nn' not in x[-2]] if (len(filteredRlLists) == 0): # invalid case continue else: [heuristic, iOItem] = do_heuristic(subItem, filteredRlLists) print('heuristic:', heuristic) print('i/o input:', iOItem) # input filter results based on pos # this is a heuristic filteredRlLists = [x for x in rlLists if 'nn' in x[-2]] if (len(filteredRlLists) == 0): # invalid case continue else: [heuristic, iOItem] = do_heuristic(subItem, filteredRlLists) print('heuristic:', heuristic) print('i/o output:', iOItem) print('###############')
#!/usr/bin/env python # encoding: utf-8 from pytextrank import json_iter, parse_doc, pretty_print import sys ## Stage 1: ## * perform statistical parsing/tagging on a document in JSON format ## ## INPUTS: <stage0> ## OUTPUT: JSON format `ParsedGraf(id, sha1, graf)` if __name__ == "__main__": path_stage0 = sys.argv[1] for graf in parse_doc(json_iter(path_stage0), force_encode=False): print(pretty_print(graf._asdict()))
if os.path.exists(OUTOUT_DIRECTORY): shutil.rmtree(OUTOUT_DIRECTORY) os.makedirs(OUTOUT_DIRECTORY) print('Saving tweets to json...') with open(TWEETS_JSON, 'w', encoding='utf8') as outfile: json.dump({'id': '777', 'text': '. '.join(user_tweets)}, outfile, ensure_ascii=False) print('Saving tweets to json - Done') print('Performing statistical parsing/tagging on tweets...') with open(STATISTICAL_PARSING_OUTPUT, 'w') as f: for graf in pytextrank.parse_doc(pytextrank.json_iter(TWEETS_JSON)): f.write("%s\n" % pytextrank.pretty_print(graf._asdict())) print('Performing statistical parsing/tagging on tweets... - Done') print('Collect and normalizing the key phrases from the parsed document...') graph, ranks = pytextrank.text_rank(STATISTICAL_PARSING_OUTPUT) pytextrank.render_ranks(graph, ranks) with open(KEY_PHRASES_NORMALIZATION_OUTPUT, 'w') as f: for rl in pytextrank.normalize_key_phrases(STATISTICAL_PARSING_OUTPUT, ranks): f.write("%s\n" % pytextrank.pretty_print(rl._asdict())) print('Collect and normalizing the key phrases from the parsed document... - Done') print("Summarizing tweets based on key phrases...") phrases = ", ".join(set([p for p in pytextrank.limit_keyphrases(KEY_PHRASES_NORMALIZATION_OUTPUT, phrase_limit=MAX_SUBJECTS_TO_SHOW)]))
import pytextrank import sys path_in = "input/in.json" path_stage1 = "test_data/s1.json" path_stage2 = "test_data/s2.json" path_stage3 = "test_data/s3.json" # stage1 : generate graph with open(path_stage1, 'w') as file: json_iterator = pytextrank.json_iter(path_in) for g in pytextrank.parse_doc(json_iterator): file.write("%s\n" % pytextrank.pretty_print(g._asdict())) #print(pytextrank.pretty_print(g)) # stage2 : key words graph, ranks = pytextrank.text_rank(path_stage1) pytextrank.render_ranks(graph, ranks) with open(path_stage2, 'w') as file: for rl in pytextrank.normalize_key_phrases(path_stage1, ranks): file.write("%s\n" % pytextrank.pretty_print(rl._asdict())) #print(pytextrank.pretty_print(rl)) # # stage3 : summarize sentence kernel = pytextrank.rank_kernel(path_stage2) with open(path_stage3, 'w') as file: for s in pytextrank.top_sentences(kernel, path_stage1): file.write(pytextrank.pretty_print(s._asdict())) file.write("\n") #print(pytextrank.pretty_print(s._asdict())) # stage4 : summarize based on most significant sentence & keywords