def summarize3(txt, cuttor=None): # TODO how to do this better 21/08/13 13:07:22 # You can replace this with "import numpy", and of cause you have to # install the lib numpy name = "numpy" numpy = __import__(name, fromlist=[]) if cuttor: tmp_cuttor = cuttor else: tmp_cuttor = Cuttor() tmp_cuttor.set_stage1_regex( re.compile('(\d+)|([a-zA-Z]+)', re.I | re.U)) sentences = [] #TODO do it better 21/08/13 12:36:08 for s, need in tmp_cuttor.cut_to_sentence(txt): if need: sentences.append(s) normalized_sentences = [s.lower() for s in sentences] top_n_words = extract_keywords(txt, N, tmp_cuttor) scored_sentences = __score_sentences(normalized_sentences, top_n_words, tmp_cuttor) avg = numpy.mean([s[1] for s in scored_sentences]) std = numpy.std([s[1] for s in scored_sentences]) mean_scored = [(sent_idx, score) for (sent_idx, score) in scored_sentences if score > avg + 0.5 * std] mean_scored_summary = [sentences[idx] for (idx, score) in mean_scored] return ', '.join(mean_scored_summary) + '.'
def summarize1(original_text, summary_size=8, cuttor=None): if cuttor: tmp_cuttor = cuttor else: tmp_cuttor = Cuttor() tmp_cuttor.set_stage1_regex( re.compile('(\d+)|([a-zA-Z]+)', re.I | re.U)) words_sorted = extract_keywords(original_text, 16, cuttor) summary_set = {} sentences = [] for s, need in tmp_cuttor.cut_to_sentence(original_text): if need: sentences.append(s) for word in words_sorted: matching_sentence = __search_word(sentences, word) if matching_sentence <> '': summary_set[matching_sentence] = 1 if len(summary_set) >= summary_size: break summary = [] for s in sentences: if s in summary_set: summary.append(s) return ', '.join(summary) + '.'
def summarize1(original_text, summary_size = 8, cuttor = None): if cuttor: tmp_cuttor = cuttor else: tmp_cuttor = Cuttor() tmp_cuttor.set_stage1_regex(re.compile('(\d+)|([a-zA-Z]+)', re.I|re.U)) words_sorted = extract_keywords(original_text, 16, cuttor) summary_set = {} sentences = [] for s,need in tmp_cuttor.cut_to_sentence(original_text): if need: sentences.append(s) for word in words_sorted: matching_sentence = __search_word(sentences, word) if matching_sentence <> '': summary_set[matching_sentence] = 1 if len(summary_set) >= summary_size: break summary = [] for s in sentences: if s in summary_set: summary.append(s) return ', '.join(summary)+'.'
def summarize3(txt, cuttor=None): # TODO how to do this better 21/08/13 13:07:22 # You can replace this with "import numpy", and of cause you have to # install the lib numpy name = "numpy" numpy = __import__(name, fromlist=[]) if cuttor: tmp_cuttor = cuttor else: tmp_cuttor = Cuttor() tmp_cuttor.set_stage1_regex(re.compile('(\d+)|([a-zA-Z]+)', re.I|re.U)) sentences = [] #TODO do it better 21/08/13 12:36:08 for s,need in tmp_cuttor.cut_to_sentence(txt): if need: sentences.append(s) normalized_sentences = [s.lower() for s in sentences] top_n_words = extract_keywords(txt, N, tmp_cuttor) scored_sentences = __score_sentences(normalized_sentences, top_n_words, tmp_cuttor) avg = numpy.mean([s[1] for s in scored_sentences]) std = numpy.std([s[1] for s in scored_sentences]) mean_scored = [(sent_idx, score) for (sent_idx, score) in scored_sentences if score > avg + 0.5 * std] mean_scored_summary=[sentences[idx] for (idx, score) in mean_scored] return ', '.join(mean_scored_summary) + '.'
def summarize2(txt, cuttor=None): if cuttor: tmp_cuttor = cuttor else: tmp_cuttor = Cuttor() tmp_cuttor.set_stage1_regex(re.compile('(\d+)|([a-zA-Z]+)', re.I|re.U)) sentences = [] #TODO do it better 21/08/13 12:36:08 for s,need in tmp_cuttor.cut_to_sentence(txt): if need: sentences.append(s) normalized_sentences = [s.lower() for s in sentences] top_n_words = extract_keywords(txt, N_2, tmp_cuttor) scored_sentences = __score_sentences(normalized_sentences, top_n_words, tmp_cuttor) top_n_scored = sorted(scored_sentences, key=lambda s: s[1])[-TOP_SENTENCES:] top_n_scored = sorted(top_n_scored, key=lambda s: s[0]) top_n_summary=[sentences[idx] for (idx, score) in top_n_scored] return ', '.join(top_n_summary) + '.'
def summarize3(txt, cuttor=None): # Remove numpy and calc mean,std by own 21/08/13 13:07:22 if cuttor: tmp_cuttor = cuttor else: tmp_cuttor = Cuttor() tmp_cuttor.set_stage1_regex(re.compile('(\d+)|([a-zA-Z]+)', re.I|re.U)) sentences = [] #TODO do it better 21/08/13 12:36:08 for s,need in tmp_cuttor.cut_to_sentence(txt): if need: sentences.append(s) normalized_sentences = [s.lower() for s in sentences] top_n_words = extract_keywords(txt, N_3, tmp_cuttor) scored_sentences = __score_sentences(normalized_sentences, top_n_words, tmp_cuttor) avg,std = _mean_std([s[1] for s in scored_sentences]) mean_scored = [(sent_idx, score) for (sent_idx, score) in scored_sentences if score > avg + 0.5 * std] mean_scored_summary=[sentences[idx] for (idx, score) in mean_scored] return ', '.join(mean_scored_summary) + '.'