def RemoveStopWords(self):
LOGGER.debug("text before StopWords processing: {}".format(
self.corpus))
self.corpus = self._remove_punct()
self.corpus = self._remove_stop_words()
LOGGER.debug("text after StopWords processing: {}".format(self.corpus))
def Stem(self, text):
LOGGER.debug("text before Stem processing: {}".format(text))
tokens = word_tokenize(text)
porter = PorterStemmer()
# vectorizing function to able to call on list of tokens
stem_words = np.vectorize(porter.stem)
self.corpus = stem_words(tokens)
LOGGER.debug("text after Stem processing: {}".format(self.corpus))
def Lemmatize(self):
LOGGER.debug("text before Lemm processing: {}".format(self.corpus))
wordnet_lemmatizer = WordNetLemmatizer()
# vectorizing function to able to call on list of tokens
lemmatize_words = np.vectorize(wordnet_lemmatizer.lemmatize)
self.corpus = lemmatize_words(self.corpus)
LOGGER.debug("text after Lemm processing: {}".format(self.corpus))
def Lemmatize(self):
"""
Spacy lemmatized much better than nltk,
one of the examples risen -> rise, only spacy handled that.
"""
LOGGER.debug("text before Lemm processing: {}".format(self.corpus))
new_corpus = []
nlp = spacy.load("en_core_web_sm")
for el in self.corpus:
new_corpus += [nlp(str(el))]
self.corpus = new_corpus.copy()
LOGGER.debug("text after Lemm processing: {}".format(self.corpus))