tweets_df = tweets_df[['tweet','sentiment']] tweets_df['clean_tweet'] = clean.clean(tweets_df['tweet']) tweets_df['clean_tweet'] = tweets_df['clean_tweet'].apply(lambda x: clean.tokenize(x)) docs2 = tweets_df['clean_tweet'] t2 = Tokenizer() t2.fit_on_texts(docs2) vocab_size2 = len(t2.word_index) + 1 #encode the documents encoded_docs2 = t2.texts_to_sequences(docs2)''' clean = CleanText() #clean() removes urls, emoticons and hashtags tweets['text'] = clean.clean(tweets['text']) #remove punctuations, stopwords, lemmatize and splits the sentences into tokens tweets['text'] = tweets['text'].apply(lambda x: clean.tokenize(x)) docs = tweets['text'] labels = tweets['sentiment'] le = LabelEncoder() labels_en = le.fit_transform(labels) #Negative: 0, Positive: 1 labels_en = keras.utils.to_categorical(np.asarray(labels_en)) #tokenizer t = Tokenizer() t.fit_on_texts(docs) vocab_size = len(t.word_index) + 1 #encode the documents encoded_docs = t.texts_to_sequences(docs)
date = st.sidebar.date_input('Enter Date Range:',[datetime.date(2019, 7, 6), datetime.date(2019, 7, 8)]) limit = st.sidebar.slider('Enter number of Tweets to scrape:',0,1000) lang = 'english' if st.button('Scrape Tweets'): with st.spinner('Scraping Tweets...'): tweets = query_tweets('videogames', begindate = date[0], enddate = date[1], limit = limit, lang = lang) df = pd.DataFrame(t.__dict__ for t in tweets) df = df[['timestamp','text','likes','retweets']] df = df.drop_duplicates(subset=['likes']) clean = CleanText() df['clean_text'] = clean.clean(df['text']) df['clean_text'] = df['clean_text'].apply(lambda x: clean.tokenize(x)) docs = df['clean_text'] #tokenizer t = Tokenizer() t.fit_on_texts(docs) vocab_size = len(t.word_index) + 1 #encode the documents encoded_docs = t.texts_to_sequences(docs) #pad docs to max length padded_docs = pad_sequences(encoded_docs, maxlen = 40, padding = 'post') labels_categorical = model_sentiment.predict(padded_docs) df['labels'] = np.argmax(labels_categorical, axis = 1)