def normalize(text): text = text.decode('utf-8') text = re.sub(r'[a-zA-z]+://[^\s]*', '', text) text = re.sub(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', '', text) text = strip_accents_ascii(text) text = text.encode('utf-8') text = ' '.join(map(lambda x: x.lower(), TreebankWordTokenizer().tokenize(text))) return text
def clean_up(text): #Retira mencao a outros usuarios text = re.sub('@\S*', '', text) #Retira urls text = re.sub('http\S*', '', text) #Retira numeros text = re.sub('\d+', '', text) #Retira os termos 'user' e 'RT' text = re.sub('user', '', text) text = re.sub('RT', '', text) #Retira caracteres especiais e emojis text = re.sub(r'[^\w\s]', '', text) text = text.encode('ascii', 'ignore').decode('ascii') return text
def normalize(text): text = text.decode('utf-8') text = re.sub(r'[a-zA-z]+://[^\s]*', '', text) text = re.sub(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', '', text) text = text.encode('utf-8') return text