def stem(self, min_word_count=10): stemmer = Stemmer({w:n for (w,n) in self.vocab.items() if n >= min_word_count}) for mail in self.mails: mail.sents = [[stemmer.stem(w) for w in sent] for sent in mail.sents] self.stemmer = stemmer
def main(args): dl = DataLoader() stem = Stemmer('porter') # files is a list of files, which are lists of lines, which are lists of words files = [{element[0]: stem.stem(element[1]) for element in dl.load_data(file) if stem.stem(element[1])} for file in args] for file, arg in zip(files, args): print('Processing file {}...'.format(arg)) file = {k: list(v) for k, v in file.items()} print('Data Clusterer') test_clusterer(DataClusterer(list(file.values()), 'euclidean'), file) print('-'*64) print('Description Clusterer') test_clusterer(DescriptionClusterer(list(file.values()), 'cosine'), file)
def test_stem(self): """Checks the final stems.""" stemmer = Stemmer() output = file('output.txt') for word in file('voc.txt'): word = word.strip() stem = output.next().strip() self.failUnless(stemmer.stem(word) == stem, "Test failed for word \'%s\' stemmed "\ "to %s should have been %s"\ % (word, stemmer.stemmed, stem))
def predict(self, doc): # Prepare document doc = self.clean(doc) # Getting class with highly score score = [] for cat in self.C: probability = math.log10(self.DC[cat] / self.D) for word in doc.split(): if len(word) > 2: cur_word = Stemmer.stem(u'{}'.format(word)) probability += math.log10( (self.WiC[cat].get(cur_word, 0) + 1) / (len(self.W) + self.WC[cat])) score.append(probability) return self.C[score.index(max(score))]
def train(self, doc, category): # Prepare document doc = self.clean(doc) # Update classifier: # Update D self.D += 1 # Update C & DC if category not in self.C: self.C.append(category) self.DC[category] = 1 else: self.DC[category] += 1 for word in doc.split(): if len(word) > 2: # 'Normalize' word cur_word = Stemmer.stem(u'{}'.format(word)) # Update W if cur_word not in self.W: self.W.append(cur_word) # Update WC if category not in self.WC.keys(): self.WC[category] = 1 else: self.WC[category] += 1 # Update Wic if category not in self.WiC.keys(): self.WiC[category] = {} if cur_word not in self.WiC[category].keys(): self.WiC[category][cur_word] = 1 else: self.WiC[category][cur_word] += 1
from stopwordsremover import StopWordsRemover from texthandler import TextHandler from stemmer import Stemmer from tfidf import TFIDFHandler from searchhandler import SearchHandler # Text to be converted text = """ The 2019–20 coronavirus pandemic is an ongoing pandemic of coronavirus disease 2019 (COVID-19), caused by severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2). The outbreak was first noted in Wuhan, Hubei province, China, in December 2019. The World Health Organization (WHO) declared the outbreak to be a Public Health Emergency of International Concern on 30 January 2020 and recognized it as a pandemic on 11 March 2020. As of 6 April 2020, more than 1,270,000 cases of COVID-19 have been reported in over 200 countries and territories, resulting in approximately 69,400 deaths. More than 260,000 people have recovered. """ # Remove Stopwords or unnessary words such as is, a, and removed_stopwords_text = StopWordsRemover.remove(text) # Stem to reduce inflected words to their word stem, base or root form stemmed_text = Stemmer.stem(removed_stopwords_text) # Counts number of words has appeared in the document sanitized_text = TextHandler.WordCounter(stemmed_text) book1 = { "ID": '1', "Title": "Covid", "Subtitle": "viruses", "Author": "author 1", "RawText": text, "SanitizedText": sanitized_text, "RemovedStopWordsText": removed_stopwords_text, "TotalNoOfTerms": len(text.lower().split(" ")), "TFIDF": 0, } text2 = """ Artificial neural networks (ANN) or connectionist systems are computing systems vaguely inspired by the biological neural networks that constitute animal brains. Such systems "learn" to perform tasks by considering examples, generally without being programmed with task-specific rules. For example, in image recognition, they might learn to identify images that contain cats by analyzing example images that have been manually labeled as "cat" or "no cat" and using the results to identify cats in other images. They do this without any prior knowledge of cats, for example, that they have fur, tails, whiskers and cat-like faces. Instead, they automatically generate identifying characteristics from the examples that they process.
def query(self, value): self._query = Stemmer.stem( value) # Automatically Stem the input of the user
def _get_features(self, tokens, idx): stemmer = Stemmer() numbs = numbers.values() puncts = punctuations.values() token = stemmer.stem(tokens[idx]) feature_list = [] if not token: return feature_list for number in numbs: if number in list(token): feature_list.append("HAS_NUM") for punctuation in puncts: if punctuation in list(token): feature_list.append("PUNCTUATION") feature_list.append("WORD_" + token) if len(token) > 1: feature_list.append("SUF_" + token[-1:]) feature_list.append("PRE_" + token[:1]) if len(token) > 2: feature_list.append("SUF_" + token[-2:]) feature_list.append("PRE_" + token[:2]) if len(token) > 3: feature_list.append("SUF_" + token[-3:]) feature_list.append("PRE_" + token[:3]) if idx >= 1: previous_token = stemmer.stem(tokens[idx - 1]) if not previous_token: return feature_list for number in numbs: if number in list(previous_token): feature_list.append("HAS_NUM") for punctuation in puncts: if punctuation in list(previous_token): feature_list.append("PUNCTUATION") if len(previous_token) > 1: feature_list.append("SUF_" + previous_token[-1:]) feature_list.append("PRE_" + previous_token[:1]) if len(previous_token) > 2: feature_list.append("SUF_" + previous_token[-2:]) feature_list.append("PRE_" + previous_token[:2]) if len(previous_token) > 3: feature_list.append("SUF_" + previous_token[-3:]) feature_list.append("PRE_" + previous_token[:3]) feature_list.append("PREV_WORD_" + previous_token) if idx >= 2: previous_token = stemmer.stem(tokens[idx - 2]) if not previous_token: return feature_list for number in numbs: if number in list(previous_token): feature_list.append("HAS_NUM") for punctuation in puncts: if punctuation in list(previous_token): feature_list.append("PUNCTUATION") if len(previous_token) > 1: feature_list.append("SUF_" + previous_token[-1:]) feature_list.append("PRE_" + previous_token[:1]) if len(previous_token) > 2: feature_list.append("SUF_" + previous_token[-2:]) feature_list.append("PRE_" + previous_token[:2]) if len(previous_token) > 3: feature_list.append("SUF_" + previous_token[-3:]) feature_list.append("PRE_" + previous_token[:3]) feature_list.append("PREV_PREV_WORD_" + previous_token) if idx < len(tokens) - 1: next_token = stemmer.stem(tokens[idx + 1]) if not next_token: return feature_list for number in numbs: if number in list(next_token): feature_list.append("HAS_NUM") for punctuation in puncts: if punctuation in list(next_token): feature_list.append("PUNCTUATION") if len(next_token) > 1: feature_list.append("SUF_" + next_token[-1:]) feature_list.append("PRE_" + next_token[:1]) if len(next_token) > 2: feature_list.append("SUF_" + next_token[-2:]) feature_list.append("PRE_" + next_token[:2]) if len(next_token) > 3: feature_list.append("SUF_" + next_token[-3:]) feature_list.append("PRE_" + next_token[:3]) feature_list.append("NEXT_WORD_" + next_token) if idx < len(tokens) - 2: next_token = stemmer.stem(tokens[idx + 2]) if not next_token: return feature_list for number in numbs: if number in list(next_token): feature_list.append("HAS_NUM") for punctuation in puncts: if punctuation in list(next_token): feature_list.append("PUNCTUATION") if len(next_token) > 1: feature_list.append("SUF_" + next_token[-1:]) feature_list.append("PRE_" + next_token[:1]) if len(next_token) > 2: feature_list.append("SUF_" + next_token[-2:]) feature_list.append("PRE_" + next_token[:2]) if len(next_token) > 3: feature_list.append("SUF_" + next_token[-3:]) feature_list.append("PRE_" + next_token[:3]) feature_list.append("NEXT_NEXT_WORD_" + next_token) return feature_list
def pos_tag(self, sentence): stemmer = Stemmer() sent = stemmer.stem(sentence) sent = WordTokenizer(sent) tags = self.tag(sent) return tags
# In[100]: import re import numpy as np import pandas as pd from stemmer import Stemmer data = pd.read_excel("Gujarati_Dimensionality_Reduction.xlsx") input_text = input() s1 = "રોગ" if input_text.startswith(s1): flag = 1 stemmer = Stemmer() stemmed_text = stemmer.stem(input_text) stemmed_words = re.split(r'[;|,|\s]\s*', stemmed_text) diseases_dire = [] for i in range(len(stemmed_words)): for col in data['રોગ']: if (stemmed_words[i] == col and stemmed_words[i] != 'રોગ'): diseases_dire.append(stemmed_words[i]) print("રોગો મેળ ખાતા ફોર્મ ડેટાસેટ : ", diseases_dire) diseases_dire = np.array(diseases_dire) data2 = pd.read_excel("Gujarati_Dataset2.xlsx") Y = data2[data2.columns[0]].as_matrix()
d = m.__dict__ d["date"] = date.fromtimestamp(d["timestamp"]).strftime("%d %B %Y") with open(outf, "wt") as f: d["from"] = d.pop("sender") if m.sfbi: ce = d["contact-email"] ce = "\t".join(ce) if type(ce) is set else ce d["contact-email"] = ce.replace(" [dot] ", ".").replace("[at]", "@") cn = d["contact-nom"] d["contact-nom"] = "\t".join(cn) if type(cn) is set else cn if "lieu" in d: # TODO: A mettre dans le parser web: d["lieu"] = d["lieu"].replace("\n\n", "\n") f.write("!!! Ceci est le contenu d'une annonce sur le site sfbi.fr.\n") for field in fields: if field in d: f.write("* {}: {}\n".format(field, d[field])) f.write("-" * 60 + "\n") f.write(m.description) f.write("-" * 60 + "\n") for sent in m.sents: f.write(" ".join(stemmer.stem(word) for word in sent) + "\n")