def __init__(self, word2vec_provider: Word2VecProvider, emoji_provider: EmojiProvider): self._emoji_provider = emoji_provider self._repeat_replacer = RepeatReplacer() self._polarity_replacer = PolarityReplacer() self._replacement_patterns = NEGATION_REPLACEMENT_PATTERNS self._replacement_patterns.extend([ # remove urls (r'((www\.[^\s]+)|(https?://[^\s]+))', ''), # remove usernames (r'@[^\s]+', ''), # remove # from hashtags (r'#([^\s]+)', r'\1'), # leave only letters (r'[^a-zA-Z]+', ' '), # remove months (r'(\b\d{1,2}\D{0,3})?\b(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|' + r'aug(?:ust)?|sep(?:tember)?|oct(?:ober)?|(nov|dec)(?:ember)?)\D?(\d{1,2}(st|nd|rd|th)?)?(([,.\-\/])' + r'\D?)?((19[7-9]\d|20\d{2})|\d{2})*', '') ]) self._regexp_replacer = RegexpReplacer(self._replacement_patterns) self._stem_replacer = StemReplacer() self._word2vec_provider = word2vec_provider self._stopwords = stopwords.words('english') # drop negation words from stopwords self._stopwords.extend(['NEG_' + word for word in self._stopwords]) self._stopwords.extend(["'nt", "st", "nd", "rd", "th", "rt"]) self._stopwords.extend(self._emoji_provider.emoji)
def aquire_text(path): filename = os.path.join(path, 'picscontent.txt') replacer = RegexpReplacer() content = '' with open(filename, 'r', encoding="ISO-8859-1") as f: for line in f.readlines(): if len(line) < 2 or '.jpg' in line or '.png' in line: continue else: content = content + ' ' + line.strip() words = word_count(content) singlewords = [] for key in words.keys(): singlewords.append(key) content_update = replacer.replace(content) content_sent = sent_tokenize(content_update) words_content = [] for item in content_sent: words_content.append(nltk.pos_tag(word_tokenize(item))) #to generate word card word_list(words, path) #to classify word by tag class word_classify(words_content, words, path) # to get the vowel of words find_vowel(singlewords, path) # to get the distance of words word_distance(singlewords, path)
def clean(self, text): result = [] replacer = RegexpReplacer() text1 = replacer.replace_simple(text) text2 = replacer.replace(text1) sent_text = sent_tokenize(text2) for item in sent_text: if len(item) > 0: result.append(item) return (result)
def unset_apostrophe(self): """ This function needs to import RegexpReplacer. """ replacer = RegexpReplacer() self.unset_apostrophe_list = [] for element in self.words: temp_elem = replacer.replace(element) self.unset_apostrophe_list.append(temp_elem.replace('\'', '')) return self.unset_apostrophe_list
def tokenizer(raw): # print(stopwords.words('english')) stop_words = stopwords.words('english') symbols = ["'", '"', '`', '.', ',', '-', '!', '?', ':', ';', '(', ')', '--', '\'s', '\'', '\'re', '{', '}', 'ー'] replacer = RegexpReplacer() replaced_raw = replacer.replace(raw).lower() tokens = [word for word in word_tokenize(replaced_raw) if word not in stop_words + symbols] text = nltk.Text(word_tokenize(replaced_raw)) return tokens, text
def create_bag(df): '''Create a "bag of words" from the review text. Input: - Pandas dataframe containing review text. Output: - List of sentences. ''' replacer = RegexpReplacer() sentences = [] for review in df['text']: tmp = replacer.replace(review) tmp1 = tmp.strip() sentences.append(sent_tokenize(tmp1)) sentences = [ inner for sublist in sentences for inner in sublist ] return sentences
def obtain_analysis_objective(self, filename): replacer = RegexpReplacer() pathname = filename pos = pathname.rindex('/') self.objectivename = pathname[int(pos)+1:len(pathname)] f = open(pathname, 'r+') for line in f.readlines(): line = replacer.replace(line) self.objectiveoutstring += line
def tokenize(text): regex = re.compile(r'^[a-zA-Z]') replacer = RegexpReplacer() lemmatizer = WordNetLemmatizer() temp = replacer.replace(text) sent_temp = sent_tokenize(temp) word_temp = [word_tokenize(doc) for doc in sent_temp] wordlist = [item for sub in word_temp for item in sub] x = re.compile('[%s]' % re.escape(string.punctuation)) y = re.compile('^[a-zA-Z]') newwordlist = [] for word in wordlist: if (not bool(re.search(x, word))) and (word.lower() not in stops) and bool( re.search(y, word)): t = lemmatizer.lemmatize(word.lower()) newwordlist.append(t) return newwordlist
def initialization(self): replacer = RegexpReplacer() basicwordfile = '/Users/yanchunyang/Documents/highschools/scripts/highfreq.txt' g = open(basicwordfile, 'r+') self.basicwords = g.readline().strip().split(' ')
def __init__(self, text, settings): """ @param text: List of text @param settings: dictionary of booleans @type settings: C{dictionary} param['replaceContractions'] is True or False """ self.settings = settings self.text = text # lowercase all #self.text = [w.lower() for w in text] #replace contractions try: if settings['replaceContractions'] == True: replacer = RegexpReplacer() self.text = [replacer.replace(w) for w in self.text] except: print('failed to replace contractions') pass
def readfile(path, filename): replacer = RegexpReplacer() with codecs.open(os.path.join(path, filename), "r", encoding='utf-8', errors='ignore') as f: orgtext = f.read() orgtext1 = replacer.replace(orgtext) temp = [] parse_pattern = [] sent_text = sent_tokenize(orgtext1) grammar = "NP:{<DT>?<JJ>*<NN><IN>?<NN>*}" find = nltk.RegexpParser(grammar) for sent in sent_text: sent_word = nltk.pos_tag(word_tokenize(sent)) if find.parse(sent_word): parse_pattern.append(sent) temp.append(nltk.pos_tag(word_tokenize(sent))) print(len(parse_pattern)) print(parse_pattern[0:5])
def obtain_text(): text = [] replacer = RegexpReplacer() st = LancasterStemmer() stops = stopwords.words('english') path = '/Users/yanchunyang/Documents/highschools/subtext/' for filename in os.listdir(path): if '3' in filename: with open(os.path.join(path, filename), 'r') as f: text.append(replacer.replace(f.read())) word_content = [] for sub in text: temp = word_tokenize(sub) word_content.append( [st.stem(word) for word in temp if word not in stops]) dictionary = corpora.Dictionary(word_content) corpus = [dictionary.doc2bow(text) for text in word_content] get_tdidf_lda(corpus, dictionary)
def __init__(self, text): """ False @param text: List of text @param settings: dictionary of booleans @type settings: C{dictionary} param['replaceContractions'] is True or False """ self.text = text self.lemmatize = False self.porter_stem = False self.remove_numerals = False self.remove_punctuation = False self.remove_stops = False # lowercase all #self.text = [w.lower() for w in text] #replace contractions try: if self.replace_contractions is True: replacer = RegexpReplacer() self.text = [replacer.replace(w) for w in self.text] except Exception as e: print(('failed to replace contractions %s' % e))
class translate(object): def __init__(self, basic=basicwords, stop=stopwords_one): self.basic = basicwords self.stop = stopwords_one self.d = enchant.Dict("en_US") self.lemmatizer = WordNetLemmatizer() self.replacer = RegexpReplacer() def translate_to_chinese(self, string): translate_result = {} org_string = self.replacer.replace(string) sent_word = word_tokenize(org_string) for word in sent_word: word = self.lemmatizer.lemmatize(word) if (word.lower() not in self.stop) and ( word not in self.basic) and self.d.check(word): translate_result[word] = [] syns = wn.synsets(word) for item in syns: name = item.name() result = wn.synset(name).lemma_names('cmn') for subitem in result: translate_result[word].append(subitem) return translate_result
from replacers import RegexpReplacer from nltk.tokenize import RegexpTokenizer from nltk.corpus import wordnet from nltk.stem import LancasterStemmer filename = [] outputstring = "" secondoutput = "" wordlist = {} f = open("picturebooks.txt", 'r') flag = 0 replacer = RegexpReplacer() tokenizer = RegexpTokenizer("[\w']+") for line in f.readlines(): line = line.strip() line = replacer.replace(line) if not line: continue if 'pdf' in line: filename.append(line) elif 'www' in line: continue elif 'the end' in line.lower() or 'about the author' in line.lower() or 'more books' in line.lower(): continue
import re from nltk.tokenize import RegexpTokenizer from nltk.corpus import stopwords from nltk.stem import PorterStemmer from sklearn.feature_extraction.text import CountVectorizer from sklearn.cluster import KMeans, MiniBatchKMeans from replacers import RegexpReplacer url = 'data-pre-processing.csv' dataframe = pandas.read_csv(url) tokenizer = RegexpTokenizer("[\w']+") replacer = RegexpReplacer() english_stops = set(stopwords.words('english')) stemmer = PorterStemmer() vectorizer = CountVectorizer() corpus = [] print("Removing contraction - replacer.replace") print("Removing special chars - re.sub") print("Getting tokens from comment and iterating through it - tokenizer.tokenizer") print("Removing stopwords - word not in english_stops") print("Steeming words - steemer.stem") for videoId,author,date,content,classification in dataframe.values: comment = [] content = content.lower() # Replace contractions such as as I'm to I am
#!c:\Python27\python.exe #!/usr/bin/env python import os import cgitb cgitb.enable() import cgi, cgitb import re from nltk.corpus import stopwords from nltk.corpus import webtext from nltk.collocations import BigramCollocationFinder from nltk.metrics import BigramAssocMeasures from nltk.tokenize import word_tokenize from replacers import RegexpReplacer replacer = RegexpReplacer() import csv from nltk.corpus import wordnet from nltk.tokenize.punkt import PunktSentenceTokenizer from replacers import SpellingReplacer from nltk.stem import PorterStemmer stemmer = PorterStemmer() from nltk.tokenize.punkt import PunktSentenceTokenizer from nltk.corpus import stopwords english_stops = set(stopwords.words('english')) print "Content-Type: text/html" print print """ <html> <head> <title>Spam Fighter</title> <link href='/style.css' rel='stylesheet' type='text/css' /> </head>
from replacers import RegexpReplacer from replacers import RepeatReplacer from replacers import AntonymReplacer from replacers import SpellingReplacer # from pickle import dump # # output = open('t2.pkl', 'wb') # dump(t2, output, -1) # output.close() test = "DO NOT GO THERE !!!\n\n1. I knew it was questionbale when i brought in oil i purchased for them to change out. He said they don't do this, because they like to purchase it. In other words, he needed to mark up the price for the same oil.\n\n2. He told me that our Shocks were blown out and said that we can't drive too far. Normally, when your shocks are blown out, your ride will be like a bouncing ball. I closely monitored my drive and i did not have a bumpy ride that indicated blown out shocks. I took it to two separate mechanics and they tested the car and said if the shocks were bad, the car would bounce up and down. \n\nBasically, the owner lied about the shocks to get me to pay to fix them. \n\n3. One of my light bulbs is going out. I looked up the model # to replace them and i went to autozone to purchase the ones for my car. The owner said that these are the wrong headlights and I needed a more expensive set. Now, mind you- the model's I had were based on Lexus' recommendation. \n\nHe then said that it would cost over $300 dollars to change out the bulbs. The bulbs he recommend was about $80 bucks, which means over 200 of labor. \n\nHe will over exaggerate everything to get you to pay more. \n\n\nBtw, I sent my wife in to see if he would try to run up maintenance. \n\nI would not recommend this place at all. He is not goood." test = test.lower() regex_replacer = RegexpReplacer() repeat_replacer = RepeatReplacer() spell_replacer = SpellingReplacer() antonym_replacer = AntonymReplacer() test = regex_replacer.replace(test) # test = repeat_replacer.replace(test) # tokens = antonym_replacer.replace_negations(sentence) # tokens = repeat_replacer.replace(word) # print(test) sentences = nltk.sent_tokenize(test) # # print(sentences) stopwords = nltk.corpus.stopwords.words('english')
import nltk from replacers import RegexpReplacer replacer= RegexpReplacer() replacer.replace("Don't hesitate to ask questions") print(replacer.replace("She must've gone to the market but she didn't go"))
#convert sentences to all lowercase lowercase_sentences = [None] * len(sent) for i in range(0, len(sent)): lowercase_sentences[i] = sent[i].lower() #convert tokenized words to all lowercase lowercase_words = [None] * len(tokens) for i in range(0, len(tokens)): lowercase_words[i] = tokens[i].lower() #replacing words with regular expressiong, i.e., 'won't' with 'will not' #start with s, the untokenized text replacer = RegexpReplacer() replacedText = replacer.replace(s) print(replacedText[:1000]) a = "I'm art won't bar can't he isn't you won't and they've but would've and she's while you're good and i'd here I'd" a = replacer.replace(a) print(a) #edit words with repeating characters and then tokenize a test sentence #will probably use on forum posts forumPost = 'I just looooooove it. It is ooooooh so fun aaah oooookaaay whateverrrrr' repReplacer = RepeatReplacer() forumPostTokenized = word_tokenize(forumPost) for i in range(0, len(forumPostTokenized)): forumPostTokenized[i] = repReplacer.replace(forumPostTokenized[i])
def pass_replacer(sent): replace_sentence = [] replacer_object = RegexpReplacer() for sentence in sent: replace_sentence.append(replacer_object.replace(sentence)) return replace_sentence
from replacers import RegexpReplacer replacer = RegexpReplacer() replacer.replace("@anirudh24seven hi")
import nltk from nltk.tokenize import word_tokenize from replacers import RegexpReplacer replacer = RegexpReplacer() word_tokenize("Don't hesitate to ask questions") print(word_tokenize(replacer.replace("Don't hesitate to ask questions")))
(' S ', ''), ##the stragglers: 'S' and 'A' tags (' A ', ''), ("[ ]{2,10}", ' '), ##clearing out extra whitespaces (2-10 in a row) #("[A-Z]{1}", "\n\1") ("(?P<FIRST>[A-Z]{1})", "\n\g<FIRST>") ] ##importing a file and turning it into a useable string with open("SBARQ.txt", "r") as myfile: data = myfile.read().replace('\n', '') ##this takes a string, you have to turn your file into a string first class RegexpReplacer(object): def __init__(self, patterns=replacement_patterns): self.patterns = [(re.compile(regex), repl) for (regex, repl) in patterns] def replace(self, text): s = text for (pattern, repl) in self.patterns: s = re.sub(pattern, repl, s) return s ##now the actual function from replacers import RegexpReplacer replacer = RegexpReplacer() with open("Output.txt", "w") as text_file: text_file.write(replacer.replace(data))
def __init__(self, basic=basicwords, stop=stopwords_one): self.basic = basicwords self.stop = stopwords_one self.d = enchant.Dict("en_US") self.lemmatizer = WordNetLemmatizer() self.replacer = RegexpReplacer()
from replacers import RegexpReplacer from nltk.corpus import stopwords from nltk.stem import WordNetLemmatizer from nltk.tokenize import word_tokenize #reading the csv file and extracting the column of tweets into a list csv_file=sys.argv[1] df=pd.read_csv(csv_file) saved_column=df['text'] list1=list(saved_column) #print (list1) replacer=AntonymReplacer() rep1=RepeatReplacer() rep2=RegexpReplacer() for i in range(0,len(list1)): list1[i]=re.sub(r'[^\x00-\x7F]',r' ',list1[i]) #Replacing non-ascii characters with a space list1[i]=rep2.replace(list1[i]) #texts like can't are converted into can not list1[i]=list1[i].split() #Splitting each sentence into words #list1[i]=[w for w in list1[i] if (len(w)>2)] #String length of a word is more than 2 list1[i]=replacer.replace_negations(list1[i]) #Replaces the negative words with antonyms emo={} f=open('emotions.txt','r') for line in f: line=line.split(',') emo[line[0]]=line[1].rstrip() #print(emo) abb={}
import nltk from replacers import RegexpReplacer replacer = RegexpReplacer() replacer.replace("Don't hesitate to ask questions") print(replacer.replace("She must've gone to the market but she didn't go"))
class TweetFeatureExtractor(BaseEstimator, TransformerMixin): def __init__(self, word2vec_provider: Word2VecProvider, emoji_provider: EmojiProvider): self._emoji_provider = emoji_provider self._repeat_replacer = RepeatReplacer() self._polarity_replacer = PolarityReplacer() self._replacement_patterns = NEGATION_REPLACEMENT_PATTERNS self._replacement_patterns.extend([ # remove urls (r'((www\.[^\s]+)|(https?://[^\s]+))', ''), # remove usernames (r'@[^\s]+', ''), # remove # from hashtags (r'#([^\s]+)', r'\1'), # leave only letters (r'[^a-zA-Z]+', ' '), # remove months (r'(\b\d{1,2}\D{0,3})?\b(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|' + r'aug(?:ust)?|sep(?:tember)?|oct(?:ober)?|(nov|dec)(?:ember)?)\D?(\d{1,2}(st|nd|rd|th)?)?(([,.\-\/])' + r'\D?)?((19[7-9]\d|20\d{2})|\d{2})*', '') ]) self._regexp_replacer = RegexpReplacer(self._replacement_patterns) self._stem_replacer = StemReplacer() self._word2vec_provider = word2vec_provider self._stopwords = stopwords.words('english') # drop negation words from stopwords self._stopwords.extend(['NEG_' + word for word in self._stopwords]) self._stopwords.extend(["'nt", "st", "nd", "rd", "th", "rt"]) self._stopwords.extend(self._emoji_provider.emoji) @classmethod def _count_with_func(cls, tweet, func): count = 0 for word in tweet.split(' '): if func(word): count += 1 return count @classmethod def _count_occurrences(cls, tweet, letter): count = 0 for l in tweet: if l == letter: count += 1 return count @classmethod def _count_uppercase_words(cls, tweet): return cls._count_with_func(tweet, lambda word: word == word.upper()) @classmethod def count_exclamation(cls, tweet): return cls._count_occurrences(tweet, '!') @classmethod def count_question_marks(cls, tweet): return cls._count_occurrences(tweet, '!') def count_positive_emoji(self, tweet): return self._count_with_func( tweet, lambda word: self._emoji_provider.is_positive_emoji(word.strip())) def count_negative_emoji(self, tweet): return self._count_with_func( tweet, lambda word: self._emoji_provider.is_negative_emoji(word.strip())) def clean_tweet(self, tweet): tweet = tweet.lower() # transform html encoded symbols tweet = BeautifulSoup(tweet, 'lxml').get_text() tweet = self._regexp_replacer.replace(tweet) tweet = word_tokenize(tweet) # eg loooove -> love tweet = self._repeat_replacer.replace(tweet) # replace negations tweet = self._stem_replacer.replace(tweet) tweet = self._polarity_replacer.mark_negations(tweet) return " ".join( [word for word in tweet if word not in self._stopwords]).strip() def get_avg_word_similarity(self, tweet, main_word): current_similarities = set() for word in tweet.split(' '): sim = self._word2vec_provider.get_similarity( main_word, word.lower()) if sim is not None: current_similarities.add(sim) if len(current_similarities) == 0: return if len(current_similarities) == 1: return current_similarities.pop() # return np.mean(zscore(list(current_similarities))) # if len(current_similarities) == 1: # return current_similarities[0 ] current_similarities = list(current_similarities) max_sim = np.max(current_similarities) min_sim = np.min(current_similarities) # normalize to <0;1> return list( np.mean([((sim - min_sim) / (max_sim - min_sim)) for sim in current_similarities])) def get_word2vec_vector(self, tweet): current_word2vec = [] for word in tweet.split(' '): vec = self._word2vec_provider.get_vector(word.lower()) if vec is not None: current_word2vec.append(vec) if len(current_word2vec) == 0: return np.zeros(200) return np.array(current_word2vec).mean(axis=0) def fit(self, x, y=None): return self def transform(self, texts): features = np.recarray(shape=(len(texts), ), dtype=[('pos_emoji_count', float), ('neg_emoji_count', float), ('uppercase_word_count', float), ('exclamation_count', float), ('question_mark_count', float), ('clean_text', object), ('word2vec', np.ndarray)]) for i, text in enumerate(texts): features['pos_emoji_count'][i] = self.count_positive_emoji(text) features['neg_emoji_count'][i] = self.count_negative_emoji(text) features['uppercase_word_count'][i] = self._count_uppercase_words( text) features['exclamation_count'][i] = self.count_exclamation(text) features['question_mark_count'][i] = self.count_question_marks( text) features['clean_text'][i] = self.clean_tweet(text) features['word2vec'][i] = self.get_word2vec_vector(text) return features
import nltk from nltk.tokenize import word_tokenize from replacers import RegexpReplacer replacer=RegexpReplacer() word_tokenize("Don't hesitate to ask questions") print(word_tokenize(replacer.replace("Don't hesitate to ask questions")))
# Regex Replacer from replacers import RegexpReplacer replacer = RegexpReplacer() print replacer.replace("can't is a contraction") print replacer.replace("I should've done that thing I didn't do") # Before Tokenizing from nltk.tokenize import word_tokenize replacer = RegexpReplacer() print word_tokenize("can't is a contraction") print word_tokenize(replacer.replace("can't is a contraction")) # Replace repeating characters from replacers import RepeatReplacer replacer = RepeatReplacer() print replacer.replace('looooove') print replacer.replace('oooooh') print replacer.replace('goose') # Replacing snonnyms from a word map from replacers import WordReplacer replacer = WordReplacer({'bday': 'birthday'}) print replacer.replace('bday') print replacer.replace('happy') # Word replacing using synonym file from replacers import CsvWordReplacer
basicwordfile = 'highfreq.txt' g = open(basicwordfile, 'r+') basicwords = g.readline().strip().split(' ') g.close() stopwords_one = set(stopwords.words('english') + list(punctuation)) lemmatizer = WordNetLemmatizer() #f = open(os.path.join(path, "exp.txt"), 'w+') for filename in os.listdir(path): if '.txt' in filename: sys.stdout.write(filename) sys.stdout.write('\n') replacer = RegexpReplacer() with codecs.open(os.path.join(path, filename), "r", encoding='utf-8', errors='ignore') as f: orgtext = f.read() orgtext1 = replacer.replace(orgtext) sent_text = sent_tokenize(orgtext1) for sent in sent_text: sent_word = word_tokenize(sent) for word in sent_word: word = lemmatizer.lemmatize(word) if (word.lower() not in stopwords_one) and ( word not in basicwords) and d.check(word): sys.stdout.write(word) sys.stdout.write('\t')
#!c:\Python27\python.exe #!/usr/bin/env python import os import cgitb; cgitb.enable() import cgi, cgitb import re from nltk.corpus import stopwords from nltk.corpus import webtext from nltk.collocations import BigramCollocationFinder from nltk.metrics import BigramAssocMeasures from nltk.tokenize import word_tokenize from replacers import RegexpReplacer replacer = RegexpReplacer() import csv from nltk.corpus import wordnet from nltk.tokenize.punkt import PunktSentenceTokenizer from replacers import SpellingReplacer from nltk.stem import PorterStemmer stemmer = PorterStemmer() from nltk.tokenize.punkt import PunktSentenceTokenizer from nltk.corpus import stopwords english_stops = set(stopwords.words('english')) print "Content-Type: text/html" print print """ <html> <head> <title>Spam Fighter</title> <link href='/style.css' rel='stylesheet' type='text/css' /> </head> <body>
import csv import os import io import nltk import re from nltk.stem import WordNetLemmatizer from nltk.corpus.reader import CHILDESCorpusReader from nltk.corpus import wordnet as wn from replacers import RegexpReplacer corpus_root = nltk.data.find('corpora/childes/data-xml/Eng-UK-MOR/') replacer = RegexpReplacer() wordnet_lemmatizer = WordNetLemmatizer() path = 'Corpus/FolderByAge/' classeAberta = [ 'NN', 'NNS', 'NNP', 'NNPS', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', 'JJ', 'JJR', 'JJS', 'RB', 'RBR', 'RBS' ] # TODO tirar as onomatopeias?? POS = on def canonicalTag(palavra): j = nltk.tag.pos_tag([palavra]) # pal_pos = (j[0][0], j[0][1])
from replacers import RegexpReplacer, RepeatReplacer, WordReplacer, CsvWordReplacer # contraction replacer = RegexpReplacer() print(replacer.replace("can't is a contraction")) print(replacer.replace("I should't done that thing I didn't do")) # repeat letters replacer = RepeatReplacer() print(replacer.replace('looooove')) print(replacer.replace('oooooh')) print(replacer.replace('goose')) # synonyms replacer = WordReplacer({'bday': 'birthday'}) print(replacer.replace('bday')) print(replacer.replace('happy')) replacer = CsvWordReplacer('syn.csv') print(replacer.replace('bday')) print(replacer.replace('NLP')) print(replacer.replace('happy'))
def regex_replacer_document(document): from replacers import RegexpReplacer replacer = RegexpReplacer() return replacer.replace(document)
print(stemmerporter.stem('happiness')) print(lemmatizer.lemmatize('happiness')) print(stemmerporter.stem('believes') == 'believ') print(lemmatizer.lemmatize('believes') == 'belief') print(stemmerporter.stem('buses') == 'buse') print(lemmatizer.lemmatize('buses') == 'bus') print(stemmerporter.stem('bus') == 'bu') print('============================================') print('Replacing Words Matching Regular Expressions') print('============================================') replacer = RegexpReplacer() print(replacer.replace("can't is a contraction") == 'cannot is a contraction') print( replacer.replace("I should've done that thing I didn't do") == 'I should have done that thing I did not do') print( word_tokenize("can't is a contraction") == ['ca', "n't", 'is', 'a', 'contraction']) print( word_tokenize(replacer.replace("can't is a contraction")) == ['can', 'not', 'is', 'a', 'contraction']) print('=============================') print('Removing Repeating Characters') print('=============================')
import csv from posTag import POSTagger from replacers import RegexpReplacer from scores import Score replacer = RegexpReplacer() tagger = POSTagger() some_score = Score() original_words = [] splited_words = [] splited_words_pos = [] canonical_words_pos = [] with open('input/palavras_por_lista_artigos.csv') as csvfile: reader = csv.DictReader(csvfile, delimiter=';') for row in reader: original_words.append((row['list'], row['word'].strip())) splited_words = replacer.replace_all_list(original_words) canonical_words_pos = tagger.canonicalTag(splited_words) classeAberta = ['NN', 'NNS', 'NNP', 'NNPS', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', 'JJ', 'JJR', 'JJS', 'RB', 'RBR', 'RBS'] list = [] words = [] index = canonical_words_pos[0][0] # print(index)
tofind_sentiment_sentence = " ".join( [word.lower() for word in tofind_sentiment_sentence.split(" ")]) sentiment_wordslist = tofind_sentiment_sentence.split(" ") # conjunctive present removing after that keyword for conjuctive_word_index, conjuctive_word in enumerate(sentiment_wordslist): if conjuctive_word in firstclause_emotion: remove_sentence = 'after' if conjuctive_word in secondclause_emotion: remove_sentence = 'before' try: if (remove_sentence == 'after'): tofind_sentiment_sentence = " ".join( sentiment_wordslist[conjuctive_word_index:]) except: pass # print(tofind_sentiment_sentence) ob = RegexpReplacer() replaced_after = ob.replace(tofind_sentiment_sentence) #print(replaced_after) dickeyword_count = emotion_count( replaced_after) #keyword_analysis counter returned dicpharse_count = pharse_sentiment( replaced_after) #pharse_analysis counter returned print(dickeyword_count.most_common(3)) print(dicpharse_count.most_common(3)) print("Overall emotion is anger")