def NaiveBayesClassifierModel(): #Get sample tweets file mdiProjectSampleTweetList = mdiProjectReadSampleFile( mdiProjectReadFiles("mdiProjectFiles", "SampleTrainingData.csv")) #Split into training and testing set mdiProjectTrainingSet, mdiProjectTestingSet = mdiProjectSplitTrainTest( mdiProjectSampleTweetList) #Get stop word list mdiProjectStopWordsList = mdiProjectStopWordList( mdiProjectReadFiles("mdiProjectFiles", "StopWords.txt")) #Get tweets and feature list mdiProjectTweetsList, mdiProjectFeatureList = mdiProjectGetFeatureListAndTweetListForTrainingSet( mdiProjectTrainingSet, mdiProjectStopWordsList) #Get training set which has feature list training_set = nltk.classify.util.apply_features(mdiProjectExtractFeatures, mdiProjectTweetsList) NBClassifier = nltk.NaiveBayesClassifier.train(training_set) #Write naive bayes classifer as pickle file with open("NaiveBayesClassifierModel.pkl", 'wb') as NaiveBayesClassifierModel_Pickle: pickle.dump(NBClassifier, NaiveBayesClassifierModel_Pickle, protocol=2)
def sentimentsAnalysisSingleTweetUsingNaiveBayes(): #Get the tweet from UI mdiProjectTweet = request.form.get("tweet") #Process the tweet porcessedTweet = mdiProjectProcessTweet(mdiProjectTweet) #Get all the stop words stopWordList = mdiProjectStopWordList( mdiProjectReadFiles("mdiProjectFiles", "StopWords.txt")) #Get the features vector for a single tweet featureVector = mdiProjectGetFeatureVectorForSingleTweet( porcessedTweet, stopWordList) #Get the feacture words featureWords = mdiProjectExtractFeatures(featureVector) #Get sentiments based on the feature words tweetSentiment = NaiveBayesClassifierModel.classify(featureWords) return render_template( "naiveByesSingleTweet.html", prediction_text="Sentiment of the tweet is {}".format(tweetSentiment))
def mdiProjectProcessSampleFileToTrainModel(): #Read sample tweets mdiProjectSampleTweetList = mdiProjectReadSampleFile( mdiProjectReadFiles("mdiProjectFiles", "SampleTrainingData.csv")) mdiProjectSampleTweetList = list(mdiProjectSampleTweetList) mdiProjectSampleTweetList = mdiProjectSampleTweetList[ 1:len(mdiProjectSampleTweetList)] #Create data frame out of the sample file tweets mdiProjectDataFrame = pd.DataFrame(mdiProjectSampleTweetList, columns=['sentiments', 'text']) mdiProjectTweetTextList = mdiProjectDataFrame['text'] mdiProjectTweetSentimentsList = mdiProjectDataFrame['sentiments'] #Clean tweet list mdiProjectTweetTextListClean = mdiProjectCleanTweet( mdiProjectTweetTextList) mdiProjectCleanedDataFrame = pd.DataFrame({ 'text': mdiProjectTweetTextListClean, 'sentiments': mdiProjectTweetSentimentsList }) return mdiProjectCleanedDataFrame
def naiveBayesClassifierTestModel(): mdiProjectSampleTweetList = mdiProjectReadSampleFile( mdiProjectReadFiles("mdiProjectFiles", "SampleTrainingData.csv")) mdiProjectTrainingSet, mdiProjectTestingSet = mdiProjectSplitTrainTest( mdiProjectSampleTweetList) mdiProjectStopWordsList = mdiProjectStopWordList( mdiProjectReadFiles("mdiProjectFiles", "StopWords.txt")) #mdiProjectTestingSet = mdiProjectTestingSet[1:10] mdiProjectActualSentiments = [] mdiProjectPredictedSentiments = [] #Open NaiveBayesClassifier picke file with open("NaiveBayesClassifierModel.pkl", "rb") as NaiveBayesClassifier: NaiveBayesClassifierModel = pickle.load(NaiveBayesClassifier) for mdiProjectTestingTweetRow in mdiProjectTestingSet: mdiProjectSentiment = mdiProjectTestingTweetRow[0] mdiProjectTweet = mdiProjectTestingTweetRow[1] #Process the tweet porcessedTweet = mdiProjectProcessTweet(mdiProjectTweet) #Get the features vector for a single tweet featureVector = mdiProjectGetFeatureVectorForSingleTweet( porcessedTweet, mdiProjectStopWordsList) #Get the feacture words featureWords = mdiProjectExtractFeatures(featureVector) #Get sentiments based on the feature words tweetSentiment = NaiveBayesClassifierModel.classify(featureWords) mdiProjectActualSentiments.append(mdiProjectSentiment) mdiProjectPredictedSentiments.append(tweetSentiment) return mdiProjectActualSentiments, mdiProjectPredictedSentiments
def sentimentsAnalysisSingleTweetUsingNaiveBayes(): """ Predict sentiments of a tweet --- produces: - "application/xml" - "application/json" parameters: - name: tweet in: query type: string required: true responses: content: application/json: schema: type: object """ #Open NaiveBayesClassifier picke file with open("NaiveBayesClassifierModel.pkl", "rb") as NaiveBayesClassifier: NaiveBayesClassifierModel = pickle.load(NaiveBayesClassifier) #Get the tweet from UI mdiProjectTweet = request.args.get("tweet") #Process the tweet porcessedTweet = mdiProjectProcessTweet(mdiProjectTweet) #Get all the stop words stopWordList = mdiProjectStopWordList( mdiProjectReadFiles("mdiProjectFiles", "StopWords.txt")) #Get the features vector for a single tweet featureVector = mdiProjectGetFeatureVectorForSingleTweet( porcessedTweet, stopWordList) #Get the feacture words featureWords = mdiProjectExtractFeatures(featureVector) #Get sentiments based on the feature words tweetSentiment = NaiveBayesClassifierModel.classify(featureWords) return tweetSentiment
def sentimentsAnalysisBasedOnTweetTopicUsingNaiveBayes(): """ Predict sentiments of a tweet --- parameters: - name: tweetTopic in: query type: string required: true responses: content: application/json: schema: type: object """ #Open NaiveBayesClassifier picke file with open("NaiveBayesClassifierModel.pkl", "rb") as NaiveBayesClassifier: NaiveBayesClassifierModel = pickle.load(NaiveBayesClassifier) #Get the tweet topic mdiProjectTweetTopic = request.args.get("tweetTopic") #Get the tweets based on the tweet topic and save in file mdiProjectSearchTweetBasedOnSearchTerm( mdiProjectTweeterAuthHandler( mdiProjectGetNormalizedTweeterConfig( "mdiProjectFiles/tweeterConfig.json")), mdiProjectTweetTopic) #Process saves tweets json file mdiProjectTweetTextList, mdiProjectTweetText = mdiProjectProcessTweetJsonFile( "mdiProjectTweets.json") #Get all the stop words stopWordList = mdiProjectStopWordList( mdiProjectReadFiles("mdiProjectFiles", "StopWords.txt")) sentiments = [] for mdiProjectTweet in mdiProjectTweetTextList: #Process the tweet porcessedTweet = mdiProjectProcessTweet(mdiProjectTweet) #Get the features vector for a single tweet featureVector = mdiProjectGetFeatureVectorForSingleTweet( porcessedTweet, stopWordList) #Get the feacture words featureWords = mdiProjectExtractFeatures(featureVector) #Get sentiments based on the feature words tweetSentiment = NaiveBayesClassifierModel.classify(featureWords) tweetSentiment = tweetSentiment.replace("\"", "") sentiments.append(tweetSentiment) mdiProjectTweetText["sentiments"] = sentiments #Makes an excel file with sentimets and make it downloadable mdiProjectOutput = BytesIO() mdiProjectOutputWriter = pd.ExcelWriter(mdiProjectOutput, engine="xlsxwriter") mdiProjectTweetText.to_excel(mdiProjectOutputWriter, sheet_name="twitter_sentiments", encoding="utf-8", index=False) mdiProjectOutputWriter.save() memory_file = BytesIO() with zipfile.ZipFile(memory_file, 'w') as zf: names = ['twitter_sentiments.xlsx'] files = [mdiProjectOutput] for i in range(len(files)): data = zipfile.ZipInfo(names[i]) data.date_time = time.localtime(time.time())[:6] data.compress_type = zipfile.ZIP_DEFLATED zf.writestr(data, files[i].getvalue()) memory_file.seek(0) response = make_response( send_file(memory_file, attachment_filename='twitter_sentiments.zip', as_attachment=True)) response.headers['Access-Control-Allow-Origin'] = '*' return response
""" Created on Mon Oct 14 08:47:51 2019 @author: Santosh Sah """ from nltk.stem.wordnet import WordNetLemmatizer import string import re import pandas as pd from MDIProjectSentimentAnalysis import (mdiProjectReadFiles, mdiProjectReadSampleFile, mdiProjectStopWordList) mdiProjectStopWordsList = mdiProjectStopWordList( mdiProjectReadFiles("mdiProjectFiles", "StopWords.txt")) mdiProjectPunctuationList = set(string.punctuation) mdiProjectLemmitization = WordNetLemmatizer() mdiProjectCleanedTweetList = [] #Clean tweet list def mdiProjectCleanTweet(mdiProjectTweetList): #Looping over list of tweets for i in range(len(mdiProjectTweetList)): mdiProjectTweet = re.sub('[^a-zA-Z]', ' ', mdiProjectTweetList[i]) #Lowering the case of tweet mdiProjectTweet = mdiProjectTweet.lower()