def _summ(self, text, n_sentences, model='textrank-g'): """ This function creates a summary for the specified model. If no model is given, textrank with gensim (textrank-g) is computed by default. Args: ----- text (string) >>> Text that we want to summarize. n_sentences (int) >>> Optional. Used by sumy algorithms. Number of sentences chosen for the summary. models (string) >>> Optional. Model used to summarize. Default is set to textrank-g. Returns: -------- String with the summary for the choosen model. """ lang = Summarization.get_lang(text) parser = PlaintextParser.from_string(text, Tokenizer(lang)) if (model == 'lexrank-s'): summarizer = LexRankSummarizer() summary = summarizer(parser.document, n_sentences) summary = [str(sentence) for sentence in summary] summary_p = ' '.join(summary) elif (model == 'textrank-s'): summarizer = TextRankSummarizer() summary = summarizer(parser.document, n_sentences) summary = [str(sentence) for sentence in summary] summary_p = ' '.join(summary) elif (model == 'lsa-s'): summarizer = LsaSummarizer() summary = summarizer(parser.document, n_sentences) summary = [str(sentence) for sentence in summary] summary_p = ' '.join(summary) elif (model == 'luhn-s'): summarizer = LuhnSummarizer() summary = summarizer(parser.document, n_sentences) summary = [str(sentence) for sentence in summary] summary_p = ' '.join(summary) else: total_number_of_sentences = Summarization.count_sentences(text) ratio = n_sentences / total_number_of_sentences summary_p = gensim_summarize(text, ratio=ratio).replace('\n', ' ') return summary_p return summary_p
def text_summary(select_func, title_holder): title_holder.markdown('# ' + select_func.split()[-1].upper()) st.info('Raw content') raw_content = st.empty() demo_checkbox = st.sidebar.checkbox('Show news demo') st.sidebar.subheader('Input your texts here') content = st.sidebar.text_area('Click outside textarea after input', 'Texts need to be summarized.') st.sidebar.subheader('Method to compute similarity') metric_radio = st.sidebar.radio('', ('Co-occur', 'Cosine')) if metric_radio == 'Cosine': metric = cosine st.sidebar.selectbox('Method to compute sentence embedding', ('SIF', 'Average Embedding')) else: metric = similarity_with_coocurr if demo_checkbox: content = news_demo if content != 'Texts need to be summarized.': raw_content.markdown(content) summary = process_pipe(content, metric, tokens_counter, fasttext) st.success(f'Summarized content with {metric.__name__}') st.write(summary) st.warning(f'Summarized content with gensim API - Unstable') try: st.write(gensim_summarize(content)) except ValueError as e: st.error(e) st.sidebar.subheader('Keywords') show_kw = st.sidebar.checkbox('Show Keywords') if show_kw: kw_length = st.sidebar.slider(label='Keywords Number', min_value=3, max_value=10) kw_method = st.sidebar.selectbox( 'Method to extract keywords', ('TextRank', 'TextRank API', 'Tf-idf API')) if kw_method == 'TextRank': keywords = extract_keyword(co_occurrence(content, window_size=2), topk=kw_length) elif kw_method == 'TextRank API': keywords = textrank(content, topK=kw_length) else: keywords = extract_tags(content, topK=kw_length) st.sidebar.info('\t'.join(keywords)) content = pretty_output(content, keywords) raw_content.markdown(content, unsafe_allow_html=True) bokeh_figure()
def gensim_textrank(self): hypes = [] refs = [] refs_dirty = [] for item in self.items: if item.ref: if item.quote.ntokens <= self.mean_ntokens or \ item.quote.nsents == 1: hypes.append(item.quote.cleaned_text) refs.append(item.ref.cleaned_text) refs_dirty.append(item.ref.original) # print(item.quote.cleaned_text) else: # for x in range(2, 11): # ratio = x * .1 hyp = gensim_summarize("\n ".join(item.quote.clean_sents), word_count=self.mean_ntokens) if len(hyp) > 0: hypes.append(hyp) refs.append(item.ref.cleaned_text) score = rouge.get_scores(hyp, item.ref.cleaned_text) # print(f"{item.uid}\n \t {score}\n \t Hyp: {hyp} \n \t Ref: {item.ref}") elif item.quote.ntokens / item.quote.nsents > self.mean_ntokens: hyp = gensim_summarize("\n ".join( item.quote.clean_sents), word_count=self.max_ntokens) if len(hyp) > 0: hypes.append(hyp) refs.append(item.ref.cleaned_text) else: print( f"No summary for {item.uid} | {int(item.quote.ntokens/item.quote.nsents)}\t| {item.quote.cleaned_text}" ) # break hypes = [x.lower() for x in hypes] refs = [x.lower() for x in refs] return rouge.get_scores(hypes, refs, avg=True)
def summarize(text_to_summarize): result = {'success': True, 'summary': '', 'error': ''} try: summary = gensim_summarize(text_to_summarize) if len(summary) == 0: raise Exception('Gensim needs more text in order to summarize.') result['summary'] = summary except Exception as exception: result['success'] = False result['error'] = str(exception) return result
def _createSummaryWithGensim(self, doc): # # We use Gensim TextRank for extractive text summarization. # TODO: Explore other algorithms and options, filter candidate sentences, # use spaCy sentencizer for better results etc... text = str( doc.text ) # "".join([str(s.text) for s in self.filter_for_summarize(doc.sents)]) summarySentences = gensim_summarize( text=text, # ratio=0.1, word_count=200, split=True, ) return summarySentences
def generate_summary_gensim(text, size_summary=200): """ Performs text summary from gensim summarizer Parameters ---------- text: list A list of sentences Returns ------- summary: list List with each item being a single sentence """ text = preprocess_corpus(text) text = ". ".join(text) return gensim_summarize(text, word_count=size_summary, split=True)
def summarize(self, text: str): return gensim_summarize(text, word_count=self.word_count)
def summarize(self, text, **kwargs): try: summarized = gensim_summarize(text, **kwargs) return summarized except: return text