def giveKeyword(text): from bs4 import BeautifulSoup text = BeautifulSoup(text, "lxml").get_text() from nltk.tokenize import RegexpTokenizer tokenizer = RegexpTokenizer(r'\w+') text = tokenizer.tokenize(text) #print text cleanText = "".join([ " " + i if not i.startswith("'") and i not in string.punctuation else i for i in text ]).strip() rakeobj = Rake("SmartStoplist.txt") keywords = rakeobj.run(cleanText) output = "" #take top two highest ranked (if there even are two) for keyword in keywords[0:2]: output = output + keyword[0] + "," return output #the highest ranked one...
def rake_extract_concepts(text, exclude, number): r = Rake('/usr/share/postgresql/10/tsearch_data/english.stop') concepts = r.run(text, minCharacters=2, maxWords=4, minFrequency=3) count = 0 for keyword, weight in concepts: if weight > 1 and count < number and keyword not in exclude: yield keyword, weight count += 1
def rake(text, top_n=10): # Clean the text from non-printable characters. text = ''.join(word for word in text if word in printable) # Uses all english stopwords and punctuation from NLTK. r = Rake(NLTKStopList()) return [keyphrase for (keyphrase, score) in r.run(text)[:top_n]]
def giveKeyword(text): from bs4 import BeautifulSoup text = BeautifulSoup(text,"lxml").get_text() from nltk.tokenize import RegexpTokenizer tokenizer = RegexpTokenizer(r'\w+') text = tokenizer.tokenize(text) #print text cleanText = "".join([" " + i if not i.startswith("'") and i not in string.punctuation else i for i in text]).strip() rakeobj = Rake("SmartStoplist.txt") keywords = rakeobj.run(cleanText) output = "" #take top two highest ranked (if there even are two) for keyword in keywords[0:2]: output = output + keyword[0] + ","; return output #the highest ranked one...
def get_doc_keywords(html, articleDom): ''' Search meta keyword tag for any predefined keywords Else use RAKE library to extract keywords from document content Return first five keywords ''' tree = fromstring(html) keywords = tree.xpath('//meta[@name="keywords"]/@content') if keywords: arr = keywords.split(',')[:5] # return first five keywords return [x.strip(' ') for x in arr] else: # Use RAKE to extract keywords from article contetnt from RAKE import Rake import operator node = fromstring(articleDom) text = node.text_content() extractor = Rake( "RAKE/stoplists/SmartStoplist.txt", 3, 3, 5) # min 3 chars, max 3 words, word appears min 5 times keywords = [x[0] for x in extractor.run(text)] keywords = keywords[:5] # get top five return [x.strip(' ') for x in keywords]
class KWGet(Driver): def setup(self): path = os.path.join(os.path.dirname(__file__), STOPWORDS) self._rake = Rake(path) def _get_keywords(self, text): buf = b64decode(text) keywords = self._rake.run(buf) if PRINT: print('KWGet: keywords=%s' % str(keywords)) return keywords @wrapper def put(self, *args, **kwargs): text = kwargs.get('content') if text: keywords = self._get_keywords(text) if keywords: return {'keywords': keywords}
class KWGet(Driver): def setup(self): path = os.path.join(os.path.dirname(__file__), STOPWORDS) self._rake = Rake(path) def _get_keywords(self, text): buf = b64decode(text) keywords = self._rake.run(buf) if PRINT: print('KWGet: keywords=%s' % str(keywords)) return keywords @wrapper def put(self, *args, **kwargs): text = kwargs.get('content') if text: keywords = self._get_keywords(text) if keywords: return {'keywords':keywords}
class Analyzer: '''analyzes text for sentiment and important terms''' def __init__(self): self.filter = Rake('stoplist.txt') def sentiment(self, text): '''transform sentiment into trinary value''' polarity = TextBlob(text).sentiment.polarity if polarity > 0.2: sentiment = "1" elif polarity < -0.2: sentiment = "-1" else: sentiment = "0" return sentiment def terms(self, text): '''get most important terms from text''' if text: terms = self.filter.run(text)[0][0] else: terms = '' return terms
def __init__(self): self.filter = Rake('stoplist.txt')
os.path.dirname( os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), 'NEL')) from elasticsearch import Elasticsearch import json import time import math import stringmatching_npm as stringmatch from dateutil import parser as dateparser from datetime import datetime from collections import Counter from RAKE import Rake import requests from markdown_extract import extractText rk = Rake('SmartStoplist.txt') def get_text_content(pkg): desc = pkg.get('desc', '') keywords = pkg.get('keywords', []) readme = pkg.get('readme', '') or '' if type(readme) != str and type(readme) != unicode: # print 'No Readme Found' readme = '' readmeText = extractText(readme) # print 'readme', readmeText parsedKeywords = rk.run(readmeText) # print 'rake', parsedKeywords results = [] for kw in keywords:
import spacy from RAKE import Rake from nltk.stem.porter import PorterStemmer NLP = spacy.load('en_core_web_sm') RAKE = Rake('stopwords.txt') PSTEMMER = PorterStemmer() def find_entities(sentence): doc = NLP(sentence) return [entity.text for entity in doc.ents] def extract_keywords(sentence): return [w for s, _ in RAKE.run(sentence) for w in s.split()] def stemmer(word): stemedword = PSTEMMER.stem(word) return stemedword def test(): with open('test_sentences.txt') as f: sentences = f.read().split('\n') for sentence in sentences: print(sentence) print(find_entities(sentence))
def test_instantiate(): rake = Rake(SmartStopList()) assert rake._Rake__stop_words_pattern rake = RAKE.Rake(RAKE.SmartStopList()) assert rake._Rake__stop_words_pattern
def setup(self): path = os.path.join(os.path.dirname(__file__), STOPWORDS) self._rake = Rake(path)
def get_keyword(text): rake = Rake("SmartStoplist.txt") if text == "": return "" keywords = rake.run(text) return keywords[0][0]
def generate_titles(file_name, random=False, use_rake=False, use_summa_text_rank=False, use_text_rank=False): logger.info("Opening file") text_file = open(file_name) logger.info("Reading file") raw_text = text_file.read().lower() # Remove Unicode characters. raw_text = raw_text.decode('unicode_escape').encode('ascii', 'ignore') #Convert raw text to word tokens logger.info("Tokenizing") tokens = nltk.word_tokenize(raw_text.translate(None, string.punctuation)) #Remove stopwords logger.info("Removing stopwords") stop_words = set(stopwords.words('english')) #NOTE: we need to include some more stopwords, as 'english' doesn't contain some stopwords # related to journal articles (e.g., "et" and "al" in "et al.") stop_words.update(ADDITIONAL_STOPWORDS) filtered_text = [word for word in tokens if word not in stop_words] #Create Corpus object for input text logger.info("Creating corpus object") input_text = Corpus(raw_text, tokens, filtered_text) input_text.stop_words = stop_words logger.info("Filtered words to use") logger.info("\t %s" % input_text.filtered_tokens[:5]) #NOTE: stopwords are removed before POS tags assigned, this could # potentially degrade POS tagging performance - may want to # switch this order #Demonstrate functions logger.info("Getting POS tags") input_text.pos_tags = pos_tagger(input_text) logger.info("\t %s" % input_text.pos_tags[:5]) logger.info("Finding all used parts of speech.") input_text.used_pos = set([tag[1] for tag in input_text.pos_tags]) logger.info(input_text.used_pos) logger.info("Getting stemmed words") input_text.stemmed_words = stem_tokens(input_text) logger.info("\t %s" % input_text.stemmed_words[:5]) # split the stemmed words into ~equal-sized groups logger.info("Splitting the stemmed words into groups") #logger.info("There are %s words in this group" % len(input_text.stemmed_words)) num_splits = 2 input_text.splits = split_tokens(input_text, num_splits) #for s in input_text.splits: # logger.info("%s %s\n\n" % (s,len(s))) logger.info("Getting word frequency and proximity") cutoff = 0.125 if len(input_text.filtered_tokens) < 250: cutoff = 0.35 #33 input_text.word_freq_proximity = stems_frequency_proximity( input_text, cutoff) #logger.info("\t %s" % (input_text.word_freq_proximity[u'becom'],)) logger.info("Mapping filtered words and their stemmed forms") input_text.filtered_word_and_bases, input_text.filtered_bases_and_words = stems_and_bases( input_text) #logger.info("\t %s" % input_text.filtered_word_and_bases[u'becom']) logger.info("Mapping POS tags and words") input_text.pos_tag_and_words = pos_tags_and_words(input_text) #logger.info("\t %s" % input_text.pos_tag_and_words['NNS'][:5]) logger.info("------ End Processing ------\n\n") ########################## if use_rake: logger.info("------ Begin Rake ------") """More information at: https://github.com/fabianvf/python-rake""" r = Rake(RAKE.SmartStopList()) #stop_words_list) sorted_keywords = r.run(input_text.raw_text) logger.info("Sorted keywords: %s" % sorted_keywords[:5]) logger.info("------ End Rake ------\n\n") if use_summa_text_rank: logger.info("------ Begin SummaTextRank ------") """More information at https://github.com/summanlp/textrank""" logger.info("Sentence(s) summary: %s" % summarizer.summarize(raw_text)) logger.info("Keywords: %s" % keywords.keywords(raw_text)) logger.info("------ End SummaTextRank ------\n\n") if use_text_rank: logger.info("------ Begin TextRank ------") """More information at https://github.com/davidadamojr/TextRank""" logger.info("Sentence(s) summary: %s " % textrank.extract_sentences(raw_text)) logger.info("Keywords: %s" % textrank.extract_key_phrases(raw_text)) logger.info("------ End TextRank ------\n\n") ########################## logger.info("------ Begin Weighting ------") logger.info("Calculating word weights") input_text.word_weights = get_word_weights(input_text, random) logger.info("Printing word weights") weight_thresh = -1 print_words_with_weight_above(weight_thresh, input_text.word_weights, input_text) logger.info("------ End Weighting ------\n\n") ########################## logger.info("------ Begin Building ------") titles = build_titles(input_text) logger.info("------ End Building ------\n\n") ########################## logger.info("Closing file") text_file.close() ########################## logger.info("------ Begin Ranking ------") #NOTE: the scores denote the title rankings relative to one another # 1 denotes the title with the highest rank and 0 denotes the # title with the lowest rank (determined by a combination of # summed word weights and average word weight) titles_ranked = order_titles(titles, input_text) logger.info("------ End Ranking ------\n\n") ########################## return titles_ranked
import csv from RAKE import Rake import email import os base = os.getcwd() + '/Data_set' topic_identifier_instance = Rake('Stop_list.txt') for folder in os.listdir(base + '/Data_set'): fold = base + '/Data_set/{}'.format(folder) for file in os.listdir(fold): mail = email.message_from_file(open(fold + '/{}'.format(file))) message_string = mail.get_payload() score_table = topic_identifier_instance.run(message_string) score_table.insert(0, ('topic', 'word_score')) print(score_table) writer = csv.writer( open( base + '/Result_data_set' + '/' + folder + '/' + file + '.csv', 'w+')) for row in score_table: writer.writerow(row)
def setup(self): path = os.path.join(os.path.dirname(__file__), STOPWORDS) self._rake = Rake(path)