def test_replace_currency_symbols(): in_outs = [ ("$1.00 equals 100¢.", "_CUR_1.00 equals 100_CUR_."), ("How much is ¥100 in £?", "How much is _CUR_100 in _CUR_?"), ("My password is 123$abc฿.", "My password is 123_CUR_abc_CUR_."), ] for in_, out_ in in_outs: assert preprocessing.replace_currency_symbols(in_) == out_
def cleanTweet(raw_data): data = [txt.replace_urls(x, "") for x in raw_data] data = [txt.replace_emails(x, "") for x in data] data = [txt.replace_emojis(x, "") for x in data] data = [txt.replace_user_handles(x, "") for x in data] data = [txt.replace_phone_numbers(x, "") for x in data] data = [txt.replace_numbers(x, "") for x in data] data = [txt.replace_currency_symbols(x, "") for x in data] data = [txt.replace_hashtags(x, "") for x in data] return data
def checkLenTweet(raw_data): data = txt.replace_urls(raw_data, "") data = txt.replace_emails(data, "") data = txt.replace_emojis(data, "") data = txt.replace_user_handles(data, "") data = txt.replace_phone_numbers(data, "") data = txt.replace_numbers(data, "") data = txt.replace_currency_symbols(data, "") data = txt.replace_hashtags(data, "") if len(data) < 28: #Lunghezza media di un tweet return False else: return True
def preprocess_sentence(sent, lower=True): """Pre-process a sentence ( via ``textacy.preprocess` module ). Args: sent (str): text. lower (bool): whether to return a lowercase string. Returns: str """ # normalize unicode sent = preprocessing.normalize_unicode(sent) # deaccent sent = preprocessing.remove_accents(sent) # replace newline chars sent = re.sub("\n|\r", " ", sent) # unpack contractions sent = contractions.fix(sent) # replace emoji symbols sent = preprocessing.replace_emojis(sent) # replace hashtags sent = preprocessing.replace_hashtags(sent) # replace user handles sent = preprocessing.replace_user_handles(sent) # replace currency symbols sent = preprocessing.replace_currency_symbols(sent) # replace emails sent = preprocessing.replace_emails(sent) # replace URLs sent = preprocessing.replace_urls(sent) # remove punctuation sent = preprocessing.remove_punctuation(sent) # normalize whitespace sent = preprocessing.normalize_whitespace(sent) if lower: sent = sent.lower() return sent
def textacy_preprocess(sentence): """Preprocess text.""" sentence = preprocessing.normalize_hyphenated_words(sentence) sentence = preprocessing.normalize_quotation_marks(sentence) #sentence = preprocessing.normalize_repeating_chars(sentence) sentence = preprocessing.normalize_unicode(sentence) sentence = preprocessing.normalize_whitespace(sentence) sentence = preprocessing.remove_accents(sentence) sentence = preprocessing.remove_punctuation(sentence) sentence = preprocessing.replace_currency_symbols(sentence) sentence = preprocessing.replace_emails(sentence) sentence = preprocessing.replace_emojis(sentence) sentence = preprocessing.replace_hashtags(sentence) sentence = preprocessing.replace_numbers(sentence) sentence = preprocessing.replace_phone_numbers(sentence) sentence = preprocessing.replace_urls(sentence) sentence = preprocessing.replace_user_handles(sentence) return sentence
def preprocess_text(text, char_count_filter=True, stopwords=None, min_len=2, max_len=15): """ Pre-processing steps prior to spaCy nlp pipeline. Optional filtering of tokens based on character length. Parameters ---------- text : str char_count_filter : bool stopwords : iterable, None min_len : int max_len : int Returns ------- text : str pre-processed text """ # 1) convert to lower case for robust stop-word recognition text = text.lower() # 2) normalise text = preprocessing.normalize_quotation_marks(text) # text = preprocessing.normalize_repeating_chars(text) text = preprocessing.normalize_hyphenated_words(text) text = preprocessing.normalize_whitespace(text) # 3) replace text = preprocessing.replace_currency_symbols(text) text = preprocessing.replace_emails(text) text = preprocessing.replace_emojis(text) text = preprocessing.replace_hashtags(text) text = preprocessing.replace_numbers(text) text = preprocessing.replace_phone_numbers(text) text = preprocessing.replace_urls(text) text = preprocessing.replace_user_handles(text) # 4) remove text = preprocessing.remove_accents(text) text = preprocessing.remove_punctuation(text) text = re.sub("[^A-Za-z0-9]+", " ", text) # keep text and numbers # 5) optionally remove tokens based on length if char_count_filter & (stopwords is not None): # filter based on token length tokens = gensim.utils.simple_preprocess(doc=text, min_len=min_len, max_len=max_len) # filter case-specific words tokens = [token for token in tokens if token not in stopwords] # convert processed list of tokens back to one string text = " ".join(tokens) else: raise NotImplementedError("Not implemented.") return text
def clean_tweet(self, text): # FIXED UNICODE # text = preprocess.fix_bad_unicode(text) text = ftfy.fix_text(text) # GET TEXT ONLY FROM HTML text = BeautifulSoup(text, features='lxml').getText() # UN-PACK CONTRACTIONS text = preprocess.unpack_contractions(text) # REMOVE URL # text = preprocess.replace_urls(text) text = preprocessing.replace_urls(text) # REMOVE EMAILS # text = preprocess.replace_emails(text) text = preprocessing.replace_emails(text) # REMOVE PHONE NUMBERS # text = preprocess.replace_phone_numbers(text) text = preprocessing.replace_phone_numbers(text) # REMOVE NUMBERS # text = preprocess.replace_numbers(text) text = preprocessing.replace_numbers(text) # REMOVE CURRENCY # text = preprocess.replace_currency_symbols(text) text = preprocessing.replace_currency_symbols(text) # REMOVE ACCENTS # text = preprocess.remove_accents(text) text = preprocessing.remove_accents(text) # CONVERT EMOJIS TO TEXT words = text.split() reformed = [ self.SMILEY[word] if word in self.SMILEY else word for word in words ] text = " ".join(reformed) text = emoji.demojize(text) text = text.replace(":", " ") text = ' '.join(text.split()) # SPLIT ATTACHED WORDS text = ' '.join(re.findall('[A-Z][^A-Z]*', text)) # SPLIT UNDERSCORE WORDS text = text.replace('_', ' ') # REMOVE PUNCTUATION # text = preprocess.remove_punct(text) text = preprocessing.remove_punctuation(text) # Remove numbers text = re.sub(r'\d', '', text) # REMOVE WORDS LESS THAN 3 CHARACTERS text = re.sub(r'\b\w{1,2}\b', '', text) # NORMALIZE WHITESPACE # text = preprocess.normalize_whitespace(text) text = preprocessing.normalize_whitespace(text) return text