def extract_keywords(text, lemmatizer): """ extract keywords using textRank algorithm, lemmatize them and return the top n percent keywords :param text: string :param lemmatizer: nltk WordNetLemmatizer :return: dictionary {keyword:score, ...} """ lemmatized_text = " ".join(lemmatize_all_pos_text(text, lemmatizer)) # use gensim.summarization.keywords to extract keywords keywords_scores_pair = keywords(lemmatized_text, lemmatize=True, scores=True) # lemmatized the keywords lemmatized_keywords_mapping = lemmatize_all_pos_pair(keywords_scores_pair, lemmatizer) lemmatized_keywords = lemmatized_keywords_mapping.keys() # lemmatize stopwords stop_words = set(lemmatize_all_pos_text(stopwords.words('english'), lemmatizer, tokenize=False)) # remove stopwords from keywords keywords_without_stopwords = [keyword for keyword in lemmatized_keywords if keyword not in stop_words] keywords_mapping = dict() for keyword in keywords_without_stopwords: keywords_mapping[keyword] = lemmatized_keywords_mapping[keyword] return keywords_mapping
def extract_top_n_percent_keywords(lemmatized_text, lemmatizer, top_n_percent): """ extract keywords using textRank algorithm, lemmatize them and return the top n percent keywords :param lemmatized_text: list of strings :param lemmatizer: nltk WordNetLemmatizer :param top_n_percent: int (0 <= top_n_percent <= 100) :return: list of strings (top n percent keywords) """ # use gensim.summarization.keywords to extract keywords # this function returns [keyword, scores] keyword_score_pair = keywords(lemmatized_text, lemmatize=True, scores=True) keywords_only = [x[0] for x in keyword_score_pair] # lemmatized the keywords lemmatized_keywords = lemmatize_all_pos(keywords_only, lemmatizer, tokenize=False) # pair the lemmatized keywords with their scores scores = [x[1] for x in keyword_score_pair] lemmatized_pairs = list(zip(lemmatized_keywords, scores)) # select top keywords # 100 - int(top_n_percent) percentile is the same as top_n_percent # e.g. 90 percentile returns the top 10% top_percentile = np.percentile(scores, 100 - int(top_n_percent)) # pair[0] is the keyword and pair[1] is the corresponding score top_keywords = [pair[0] for pair in lemmatized_pairs if pair[1] >= top_percentile] return top_keywords
def extract_keywords(text): ''' Returns the keywords from a given body of text. Requires more than one sentence as input. Parameters text: A body of text with multiple sentences ''' return keywords(text, words=5)
def set_textrank_keywords(self, n=5, ratio=1.0): try: kws = keywords(self.spaced, split=True, ratio=ratio, deacc=False) kws = [kw.replace(" ", "") for kw in kws] n = min(len(kws), n) self.textrank_keywords = kws[:n] except (IndexError, ZeroDivisionError) as e: print(e) print("setting textrank_keywords to empty") self.textrank_keywords = []
def get_keywords(text): words = 3 raw_keywords = [] while True: try: if words <= 0: return [] raw_keywords = keywords(text, scores=True, words=words) except: words -= 1 continue return raw_keywords
def predict(self, X: str, y=None) -> Union[List[Tuple[str, float]], List[str], str]: return keywords( X, ratio=self.ratio, words=self.n_keywords, split=self.split, scores=self.scores, pos_filter=self.pos_filter, lemmatize=False, deacc=self.deacc, )
def get_title(text): sentences = _clean_text_by_sentences(text) if len(sentences) == 0: return "" if len(sentences) == 1: return _format_results(sentences, False) sentence_tokenized = _tokenize_sentence(_format_results(sentences, True)) indices_delete = [] for i in range(len(sentence_tokenized)): #print(sen_word_count[i][1]) if sentence_tokenized[i][1] > 10: indices_delete.append(i) sentence_tokenized = [ i for j, i in enumerate(sentence_tokenized) if j not in indices_delete ] #print(len(sen_word_count)) if len(sentence_tokenized) == 0: return "" if len(sentence_tokenized) == 1: return sentence_tokenized[0][0] keyword_list = keywords(_format_results(sentences, False), words=4, split=True) #print(keyword_list) title_score = [0, 0] for sen_tuple in sentence_tokenized: #print(sen_tuple[0]) temp_score = 0 index = sentence_tokenized.index(sen_tuple) count = 0 for word_pos in sen_tuple[2]: if word_pos[0] in keyword_list: count += 1 temp_score = count / sen_tuple[1] #print(temp_score) #print(sen_tuple[1]) if temp_score > title_score[1]: title_score = [index, temp_score] return sentence_tokenized[title_score[0]][0]