def runSentanal(train, test):
sentanal = SentimentAnalyzer()
all_words_neg = sentanal.all_words([mark_negation(doc) for doc in train])
unigramFeats = sentanal.unigram_word_feats(all_words_neg, min_freq=4)
sentanal.add_feat_extractor(extract_unigram_feats,
unigrams=unigramFeats,
handle_negation=True)
# bigramFeats = sentanal.
# sentanal.add_feat_extractor(extract_bigram_feats, bigrams=bigramFeats)
trainList = sentanal.apply_features(train)
testList = sentanal.apply_features(test)
trainer = NaiveBayesClassifier.train
classifier = sentanal.train(trainer, trainList)
classifier.show_most_informative_features()
# creates array for storing values
values = []
# display results
for key, value in sorted(sentanal.evaluate(testList).items()):
print('{0}: {1}'.format(key, value))
values.append(value)
# write results to csv
with open(OUTPUT_CSV, mode='a') as csvFile:
writer = csv.writer(csvFile, delimiter=',')
writer.writerow(values)
def createClassifier(ignoreTweets=False):
neg_ids = movie_reviews.fileids('neg')
pos_ids = movie_reviews.fileids('pos')
neg_sents = [(extractWords(movie_reviews.words(fileids=[f])), 'neg')
for f in neg_ids]
pos_sents = [(extractWords(movie_reviews.words(fileids=[f])), 'pos')
for f in pos_ids]
#if you dont want to process all tweets, just call :
(neg_pols, pos_pols) = getSentPolarities()
(neg_tweets, pos_tweets) = getTweets(ignoreTweets)
neg_sents = neg_sents + neg_tweets + neg_pols
pos_sents = pos_sents + pos_tweets + pos_pols
trainsizeneg = int(0.75 * len(neg_sents))
trainsizepos = int(0.75 * len(pos_sents))
all_train = neg_sents[:trainsizeneg] + pos_sents[:trainsizepos]
all_test = neg_sents[trainsizeneg:] + pos_sents[trainsizepos:]
# train size = 1500, test size = 500
s_analyzer = SentimentAnalyzer()
classifier = NaiveBayesClassifier.train(all_train)
print accuracy(classifier, all_test)
#classifier.show_most_informative_features()
return classifier
def train():
subj_docs = [
(sent, 'subj')
for sent in subjectivity.sents(categories='subj')[:n_instances]
]
obj_docs = [(sent, 'obj')
for sent in subjectivity.sents(categories='obj')[:n_instances]]
train_subj_docs = subj_docs[:80]
test_subj_docs = subj_docs[80:100]
train_obj_docs = obj_docs[:80]
test_obj_docs = obj_docs[80:100]
training_docs = train_subj_docs + train_obj_docs
testing_docs = test_subj_docs + test_obj_docs
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words(
[mark_negation(doc) for doc in training_docs])
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg,
min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats,
unigrams=unigram_feats)
training_set = sentim_analyzer.apply_features(training_docs)
test_set = sentim_analyzer.apply_features(testing_docs)
trainer = NaiveBayesClassifier.train
classifier = sentim_analyzer.train(trainer, training_set)
for key, value in sorted(sentim_analyzer.evaluate(test_set).items()):
print('{0}: {1}'.format(key, value))
def sentiment_classifier(df):
df = df.copy()
# prepping data
df = df[['txgot_binary', 'Convo_1']].dropna()
text_process_col = pre.process_corpus(np.asarray(df['Convo_1']), [])
txgot_col = np.asarray(df['txgot_binary'])
# turns into list of tuples (convo, label)
docs = list(zip(text_process_col, txgot_col))
shuffle(docs)
training_docs = docs[:int(len(docs) * 2 / 3)]
test_docs = docs[int(len(docs) * 2 / 3):]
# sentiment analyzer
sentim_analyzer = SentimentAnalyzer()
# simple unigram word features, handling negation
all_words_neg = sentim_analyzer.all_words(
[mark_negation(doc) for doc in training_docs])
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg,
min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats,
unigrams=unigram_feats)
# train classifier
training_set = sentim_analyzer.apply_features(training_docs)
test_set = sentim_analyzer.apply_features(test_docs)
trainer = NaiveBayesClassifier.train
classifier = sentim_analyzer.train(trainer, training_set)
# show results
for key, value in sentim_analyzer.evaluate(test_set).items():
print('{}: {}'.format(key, value))
def load_data(self, classifier=None):
# source: http://www.nltk.org/book/ch06.html, http://www.nltk.org/howto/sentiment.html
print "Loading training data...",
sys.stdout.flush()
training_docs, testing_docs = self.load_web_reviews()
# documents = [(word_tokenize(movie_reviews.raw(fileid)), category)
# for category in movie_reviews.categories()
# for fileid in movie_reviews.fileids(category)]
# random.shuffle(documents)
# cutoff = int(len(documents) * 0.1)
# training_docs, testing_docs = documents[cutoff:], documents[:cutoff]
print "Done!"
print "Extracting unigram features and applying to training data...",
sys.stdout.flush()
sentim_analyzer = SentimentAnalyzer(classifier=classifier)
all_words = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])
unigram_feats = sentim_analyzer.unigram_word_feats(all_words)#, top_n=5000)
# print len(unigrams)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats, handle_negation=True)
training_set = sentim_analyzer.apply_features(training_docs)
testing_set = sentim_analyzer.apply_features(testing_docs)
print "Done!"
return sentim_analyzer, training_set, testing_set
def train():
positive_tweets = read_tweets('/root/295/new/positive.txt', 'positive')
negative_tweets = read_tweets('/root/295/new/negative.txt', 'negative')
print len(positive_tweets)
print len(negative_tweets)
#pos_train = positive_tweets[:2000]
#neg_train = negative_tweets[:2000]
#pos_test = positive_tweets[2001:3000]
#neg_test = negative_tweets[2001:3000]
pos_train = positive_tweets[:len(positive_tweets)*80/100]
neg_train = negative_tweets[:len(negative_tweets)*80/100]
pos_test = positive_tweets[len(positive_tweets)*80/100+1:]
neg_test = negative_tweets[len(positive_tweets)*80/100+1:]
training_data = pos_train + neg_train
test_data = pos_test + neg_test
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_data])
#print all_words_neg
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg, min_freq=4)
#print unigram_feats
print len(unigram_feats)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
training_set = sentim_analyzer.apply_features(training_data)
test_set = sentim_analyzer.apply_features(test_data)
print test_set
trainer = NaiveBayesClassifier.train
classifier = sentim_analyzer.train(trainer, training_set)
for key,value in sorted(sentim_analyzer.evaluate(test_set).items()):
print('{0}: {1}'.format(key, value))
print sentim_analyzer.classify(tokenize_sentance('I hate driving car at night'))
return sentim_analyzer
def nb_cv(cleaned_df):
# Get Features
training_set = get_nb_features(cleaned_df)
# Get 10-Fold. Important: Shuffle=True
cv = KFold(n_splits=10, random_state=0, shuffle=True)
# Model
sentiment_analyzer = SentimentAnalyzer()
trainer = NaiveBayesClassifier.train
# Store Result
Accuracy = []
# For each fold, train model, evaluate
for train_index, test_index in cv.split(training_set):
classifier = sentiment_analyzer.train(
trainer,
np.array(training_set)[train_index].tolist())
truth_list = np.array(training_set)[test_index].tolist()
performance = sentiment_analyzer.evaluate(truth_list, classifier)
Accuracy.append(performance['Accuracy'])
'''## Can add all other measures here. Sample Result as below:
{'Accuracy': 0.525,
'Precision [negative]': 0.28337874659400547, 'Recall [negative]': 0.7272727272727273, 'F-measure [negative]': 0.407843137254902,
'Precision [neutral]': 0.5011933174224343, 'Recall [neutral]': 0.30837004405286345, 'F-measure [neutral]': 0.38181818181818183,
'Precision [positive]': 0.7461629279811098, 'Recall [positive]': 0.611810261374637, 'F-measure [positive]': 0.672340425531915}
'''
return np.mean(np.asarray(Accuracy))
def __init__(self):
#document represented by a tuple (sentence,labelt)
n_instances = 100
subj_docs = [(sent, 'subj') for sent in subjectivity.sents(categories='subj')[:n_instances]]
obj_docs = [(sent, 'obj') for sent in subjectivity.sents(categories='obj')[:n_instances]]
#split subj and objinstances to keep a balanced uniform class distribution in both train and test sets.
train_subj_docs = subj_docs[:80]
test_subj_docs = subj_docs[80:100]
train_obj_docs = obj_docs[:80]
test_obj_docs = obj_docs[80:100]
training_docs = train_subj_docs+train_obj_docs
testing_docs = test_subj_docs+test_obj_docs
#train classifier
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])
#use simple unigram word features, handling negation
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg, min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
#apply features to obtain a feature_value representations of our datasets
training_set = sentim_analyzer.apply_features(training_docs)
test_set = sentim_analyzer.apply_features(testing_docs)
self.trainer = NaiveBayesClassifier.train
self.classifier = sentim_analyzer.train(self.trainer, training_set)
for key,value in sorted(sentim_analyzer.evaluate(test_set).items()):
print('{0}: {1}'.format(key, value))
self.sid = SentimentIntensityAnalyzer()
def trainSubjectivity():
# Subjective vs. objective sentence classifier. Borrows from NLTK Documentation.
# Plan on using it in larger machine learning sentiment model as pre-processing
# Must differentiate between objective and subjective
subjDocs = [(sent, 'subj') for sent in subjectivity.sents(categories='subj')]
objDocs = [(sent, 'obj') for sent in subjectivity.sents(categories='obj')]
nSubj = len(subjDocs)
nObj = len(objDocs)
# 90% Training, 10% Test
subjTrain = int(.9 * nSubj)
objTrain = int(.9 * nObj)
trainSubj = subjDocs[:subjTrain]
testSubj = subjDocs[subjTrain:nSubj]
trainObj = objDocs[:objTrain]
testObj = objDocs[objTrain:nObj]
trainDocs = trainSubj + trainObj
testDocs = testSubj + testObj
# Create sentiment class, mark negation, create features (unigram)
sentiment = SentimentAnalyzer()
markNegation = sentiment.all_words([mark_negation(doc) for doc in trainDocs])
unigramFeats = sentiment.unigram_word_feats(markNegation, min_freq=4)
sentiment.add_feat_extractor(extract_unigram_feats, unigrams=unigramFeats)
training = sentiment.apply_features(trainDocs)
testing = sentiment.apply_features(testDocs)
# Train classifier
trainer = NaiveBayesClassifier.train
subjectivityClassifier = sentiment.train(trainer, training)
joblib.dump(subjectivityClassifier, 'subjectivity.pkl')
for key, value in sorted(sentiment.evaluate(testing).items()): print('{0}: {1}'.format(key, value))
'''
def demo_subjectivity(trainer, save_analyzer=False, n_instances=None, output=None):
"""
Train and test a classifier on instances of the Subjective Dataset by Pang and
Lee. The dataset is made of 5000 subjective and 5000 objective sentences.
All tokens (words and punctuation marks) are separated by a whitespace, so
we use the basic WhitespaceTokenizer to parse the data.
:param trainer: `train` method of a classifier.
:param save_analyzer: if `True`, store the SentimentAnalyzer in a pickle file.
:param n_instances: the number of total sentences that have to be used for
training and testing. Sentences will be equally split between positive
and negative.
:param output: the output file where results have to be reported.
"""
from nltk.sentiment import SentimentAnalyzer
from nltk.corpus import subjectivity
if n_instances is not None:
n_instances = int(n_instances/2)
subj_docs = [(sent, 'subj') for sent in subjectivity.sents(categories='subj')[:n_instances]]
obj_docs = [(sent, 'obj') for sent in subjectivity.sents(categories='obj')[:n_instances]]
# We separately split subjective and objective instances to keep a balanced
# uniform class distribution in both train and test sets.
train_subj_docs, test_subj_docs = split_train_test(subj_docs)
train_obj_docs, test_obj_docs = split_train_test(obj_docs)
training_docs = train_subj_docs+train_obj_docs
testing_docs = test_subj_docs+test_obj_docs
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])
# Add simple unigram word features handling negation
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg, min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
# Apply features to obtain a feature-value representation of our datasets
training_set = sentim_analyzer.apply_features(training_docs)
test_set = sentim_analyzer.apply_features(testing_docs)
classifier = sentim_analyzer.train(trainer, training_set)
try:
classifier.show_most_informative_features()
except AttributeError:
print('Your classifier does not provide a show_most_informative_features() method.')
results = sentim_analyzer.evaluate(test_set)
if save_analyzer == True:
save_file(sentim_analyzer, 'sa_subjectivity.pickle')
if output:
extr = [f.__name__ for f in sentim_analyzer.feat_extractors]
output_markdown(output, Dataset='subjectivity', Classifier=type(classifier).__name__,
Tokenizer='WhitespaceTokenizer', Feats=extr,
Instances=n_instances, Results=results)
return sentim_analyzer
def addfeatures(cleaned_tokens_list):
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words(
[mark_negation(token_list) for token_list in cleaned_tokens_list])
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg,
min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats,
unigrams=unigram_feats)
def GetSampleTrainDataForNLTK(self, trainSet):
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words(
[mark_negation(doc) for doc in trainSet])
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg,
min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats,
unigrams=unigram_feats)
sampleTrainData = sentim_analyzer.apply_features(trainSet)
return sampleTrainData
def main():
x, y = load_datasets(["../datasets/sentiment_uci/yelp_labelled.txt"])
stopwords = set()
with open('../stopwords.txt', 'r') as f:
for w in f:
stopwords.add(w.strip())
tok = TweetTokenizer()
x = [remove_stopwords(tok.tokenize(s.lower()), stopwords) for s in x]
x = np.array(x)
accumulate = dict()
folds = 10
for train_idx, test_idx in StratifiedKFold(y=y,
n_folds=folds,
shuffle=True):
train_x, train_y = x[train_idx], y[train_idx]
test_x, test_y = x[test_idx], y[test_idx]
# train_x = [remove_stopwords(tok.tokenize(s), stopwords) for s in train_x]
# test_x = [remove_stopwords(tok.tokenize(s), stopwords) for s in test_x]
train_docs = [(sent, label) for sent, label in zip(train_x, train_y)]
test_docs = [(sent, label) for sent, label in zip(test_x, test_y)]
cls = SentimentAnalyzer()
# train
words_with_neg = cls.all_words([mark_negation(a) for a in train_x])
unigram_feats = cls.unigram_word_feats(words_with_neg)
# bigram_feats = cls.bigram_collocation_feats(train_x)
cls.add_feat_extractor(extract_unigram_feats,
unigrams=unigram_feats,
handle_negation=True)
# cls.add_feat_extractor(extract_bigram_feats, bigrams=bigram_feats)
training_set = cls.apply_features(train_docs, labeled=True)
cls.train(MaxentClassifier.train, training_set, max_iter=10, trace=0)
# test & evaluate
test_set = cls.apply_features(test_docs)
for key, value in sorted(cls.evaluate(test_set).items()):
print('\t{0}: {1}'.format(key, value))
accumulate.setdefault(key, 0.0)
accumulate[key] += value if value is not None else 0.0
print("Averages")
for key, value in sorted(accumulate.items()):
print('\tAverage {0}: {1}'.format(key, value / folds))
def train_lr(training_set):
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_set])
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg, min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
training_set = sentim_analyzer.apply_features(training_set)
trainer = logreg.train
classifier = sentim_analyzer.train(trainer, training_set)
return [sentim_analyzer,classifier]
def demo_movie_reviews(trainer, n_instances=None, output=None):
"""
Train classifier on all instances of the Movie Reviews dataset.
The corpus has been preprocessed using the default sentence tokenizer and
WordPunctTokenizer.
Features are composed of:
- most frequent unigrams
:param trainer: `train` method of a classifier.
:param n_instances: the number of total reviews that have to be used for
training and testing. Reviews will be equally split between positive and
negative.
:param output: the output file where results have to be reported.
"""
from nltk.corpus import movie_reviews
from nltk.sentiment import SentimentAnalyzer
if n_instances is not None:
n_instances = int(n_instances/2)
pos_docs = [(list(movie_reviews.words(pos_id)), 'pos') for pos_id in movie_reviews.fileids('pos')[:n_instances]]
neg_docs = [(list(movie_reviews.words(neg_id)), 'neg') for neg_id in movie_reviews.fileids('neg')[:n_instances]]
# We separately split positive and negative instances to keep a balanced
# uniform class distribution in both train and test sets.
train_pos_docs, test_pos_docs = split_train_test(pos_docs)
train_neg_docs, test_neg_docs = split_train_test(neg_docs)
training_docs = train_pos_docs+train_neg_docs
testing_docs = test_pos_docs+test_neg_docs
sentim_analyzer = SentimentAnalyzer()
all_words = sentim_analyzer.all_words(training_docs)
# Add simple unigram word features
unigram_feats = sentim_analyzer.unigram_word_feats(all_words, min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
# Apply features to obtain a feature-value representation of our datasets
training_set = sentim_analyzer.apply_features(training_docs)
test_set = sentim_analyzer.apply_features(testing_docs)
classifier = sentim_analyzer.train(trainer, training_set)
try:
classifier.show_most_informative_features()
except AttributeError:
print('Your classifier does not provide a show_most_informative_features() method.')
results = sentim_analyzer.evaluate(test_set)
if output:
extr = [f.__name__ for f in sentim_analyzer.feat_extractors]
output_markdown(output, Dataset='Movie_reviews', Classifier=type(classifier).__name__,
Tokenizer='WordPunctTokenizer', Feats=extr, Results=results,
Instances=n_instances)
def train(self):
training_docs = list()
for index, row in self.training_data.iterrows():
row['text'] = self.clean_tweet(row['text'])
row['text'] = row['text'].translate(self.translate_table)
tokens = self.tokenizer.tokenize(row['text'])
training_docs.append((tokens, row['sentiment'].lower()))
sentim_analyzer = SentimentAnalyzer()
training_set = nltk.classify.apply_features(self.extract_features,
training_docs)
self.classifier = nltk.NaiveBayesClassifier.train(training_set)
def sentiment_analysis(data):
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import subjectivity
from nltk.sentiment import SentimentAnalyzer
from nltk.sentiment.util import *
n_instances = 100
subj_docs = [
(sent, 'subj')
for sent in subjectivity.sents(categories='subj')[:n_instances]
]
obj_docs = [(sent, 'obj')
for sent in subjectivity.sents(categories='obj')[:n_instances]]
train_subj_docs = subj_docs[:80]
test_subj_docs = subj_docs[80:100]
train_obj_docs = obj_docs[:80]
test_obj_docs = obj_docs[80:100]
training_docs = train_subj_docs + train_obj_docs
testing_docs = test_subj_docs + test_obj_docs
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words(
[mark_negation(doc) for doc in training_docs])
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg,
min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats,
unigrams=unigram_feats)
training_set = sentim_analyzer.apply_features(training_docs)
test_set = sentim_analyzer.apply_features(testing_docs)
trainer = NaiveBayesClassifier.train
classifier = sentim_analyzer.train(trainer, training_set)
for key, value in sorted(sentim_analyzer.evaluate(test_set).items()):
print('{0}: {1}'.format(key, value))
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk import tokenize
sid = SentimentIntensityAnalyzer()
for line in data:
ss = sid.polarity_scores(line['line_text'])
line['compound'] = ss['compound']
line['neg'] = ss['neg']
line['pos'] = ss['pos']
line['neu'] = ss['neu']
def train_sentiment():
instances = 8000
subj = [(sent, 'subj') for sent in subjectivity.sents(categories='subj')[:instances]]
obj = [(sent, 'obj') for sent in subjectivity.sents(categories='obj')[:instances]]
train_subj = subj
train_obj = obj
train_set = train_subj + train_obj
sentiment = SentimentAnalyzer()
all_neg = sentiment.all_words([mark_negation(doc) for doc in train_set])
uni_g = sentiment.unigram_word_feats(all_neg, min_freq=4)
sentiment.add_feat_extractor(extract_unigram_feats, unigrams=uni_g)
trained_set = sentiment.apply_features(train_set)
nb = NaiveBayesClassifier.train
classifier = sentiment.train(nb, trained_set)
return classifier
def runClassifier(classifierKey, emailsInSentenceForm, unigram_features):
#Create a sentiment analyzer to analyze the text documents. This analyzer
#provides an abstraction for managing a classifier, and feature extractor.
#It also provides convinence data metrics on classifier performance.
sentim_analyzer = SentimentAnalyzer()
#Create a feature extractor based on the unigram word features created.
#The unigram feature extractor is found in the sentiment utils package.
sentim_analyzer.add_feat_extractor(extract_unigram_feats,
unigrams=unigram_features)
print("Current classifier: " + classifierKey)
#Load the classifier.
classifier = loadModel(classifierKey)
#Set up sentiment counts for the emails.
posVote = 0
negVote = 0
classifierResultsDict = []
#int(emailsInSentenceForm.shape[0]/3.0)
for emailIndex in range(0, int(emailsInSentenceForm.shape[0])):
if (emailIndex % 100 == 0):
print(classifierKey + " On email: " + str(emailIndex))
#Write the results to a file.
f = open("./" + classifierKey + "Results.pkl", "w")
pickle.dump(classifierResultsDict, f)
f.close()
#Get the list of sentences for the email.
email = emailsInSentenceForm.iloc[emailIndex, :][0]
featurizedSentenceList = sentim_analyzer.apply_features(email)
for sent in featurizedSentenceList:
label = classifier.classify(sent[0])
if label == "pos":
posVote += 1
else:
negVote += 1
#Take the maximum vote for the class label. Use 1 and -1 to faciliitate the later correlation calculations.
if posVote >= negVote:
classifierResultsDict.append(1)
else:
classifierResultsDict.append(-1)
#Reset pos and neg votes to 0.
posVote = 0
negVote = 0
#Write the results to a file.
f = open("./" + classifierKey + "Results.pkl", "w")
pickle.dump(classifierResultsDict, f)
f.close()
def get_best_classifier_from_debates():
"""
The best classifier for debates turned out to be Freq Dist, Logitic
:return: the trained classifier using the entire corpus
"""
neg_docs, pos_docs = sentimentAnalysisDocumentBased.get_political_debates()
train_docs = neg_docs + pos_docs
# Set up the Sentiment Analyzer
analyzer = SentimentAnalyzer()
analyzer.add_feat_extractor(
sentimentAnalysisDocumentBased.extract_freq_dist)
train_feat = list(analyzer.apply_features(train_docs, labeled=True))
classifier = SklearnClassifier(LogisticRegression()).train(train_feat)
return analyzer, classifier
def run_sa_mov(train, test):
a = SentimentAnalyzer()
tr = NaiveBayesClassifier.train
all_words = mov_analyzer.all_words(train)
# Add simple unigram word features
unigram_feats = a.unigram_word_feats(all_words, min_freq=4)
a.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
# Apply features to obtain a feature-value representation of our datasets
tr_set = a.apply_features(train)
test_set = a.apply_features(test)
#Training
clf = a.train(tr, tr_set)
res = a.evaluate(test_set)
print(res)
def get_best_classifer_from_movies():
"""
The best classifier for movies turned out to be Bigram, Linear SVM
:return: the trained classifier using the entire corpus
"""
neg_docs, pos_docs = sentimentAnalysisDocumentBased.get_movie_corpus()
train_docs = neg_docs + pos_docs
# Set up the Sentiment Analyzer
analyzer = SentimentAnalyzer()
analyzer.add_feat_extractor(
sentimentAnalysisDocumentBased.extract_sig_bigram_feats)
train_feat = list(analyzer.apply_features(train_docs, labeled=True))
classifier = SklearnClassifier(LinearSVC()).train(train_feat)
return analyzer, classifier
def NB(df_train, df_dev):
# Feature extraction
# n=1200
df_train['clean_text'] = df_train['clean_text'].apply(
lambda x: stem_stop(x))
df_dev['clean_text'] = df_dev['clean_text'].apply(lambda x: stem_stop(x))
df_pos_train = df_train[df_train['tweet_sentiment'] == 'positive']
# df_pos_train= df_pos_train.sample(n=n, random_state=1)
pos_tweets = df_pos_train['clean_text'].tolist()
df_neg_train = df_train[df_train['tweet_sentiment'] == 'negative']
# df_neg_train= df_neg_train.sample(n=n, random_state=1)
neg_tweets = df_neg_train['clean_text'].tolist()
df_neutral_train = df_train[df_train['tweet_sentiment'] == 'neutral']
# df_neutral_train= df_neutral_train.sample(n=n, random_state=1)
neutral_tweets = df_neutral_train['clean_text'].tolist()
positive_featuresets = [(features(tweet), 'positive')
for tweet in pos_tweets]
negative_featuresets = [(features(tweet), 'negative')
for tweet in neg_tweets]
neutral_featuresets = [(features(tweet), 'neutral')
for tweet in neutral_tweets]
training_features = positive_featuresets + negative_featuresets + neutral_featuresets
ngram_vectorizer = CountVectorizer(analyzer='word',
binary=True,
lowercase=False,
ngram_range=(1, 2))
# train the model
sentiment_analyzer = SentimentAnalyzer()
trainer = NaiveBayesClassifier.train
classifier = sentiment_analyzer.train(trainer, training_features)
truth_list = list(df_dev[['clean_text',
'tweet_sentiment']].itertuples(index=False,
name=None))
# test the model
for i, (text, expected) in enumerate(truth_list):
text_feats = features(text)
truth_list[i] = (text_feats, expected)
re = sentiment_analyzer.evaluate(truth_list, classifier)
print(re)
return classifier
def sentimentAnalysis(self, number):
ids = self.getHandleIds(number)
rows = self.query(date=True,
text=True,
is_from_me=True,
condition='handle_id in (' + ','.join(ids) + ')')
sent_analyzer = SentimentAnalyzer()
res = {"Me": {}, number: {}}
plt.title("Sentiment Analysis")
plt.ylabel("Sentiment")
plt.xlabel("Date")
plt.legend()
plt.savefig("results/sentiment")
plt.show()
def subjectivity_classifier():
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import subjectivity
from nltk.sentiment import SentimentAnalyzer
from nltk.sentiment.util import *
"""
Initializes and trains categorical subjectivity analyzer
"""
N_INSTANCES = 100
subj_docs = [
(sent, 'subj')
for sent in subjectivity.sents(categories='subj')[:N_INSTANCES]
]
obj_docs = [(sent, 'obj')
for sent in subjectivity.sents(categories='obj')[:N_INSTANCES]]
train_subj_docs = subj_docs[:80]
test_subj_docs = subj_docs[80:]
train_obj_docs = obj_docs[:80]
test_obj_docs = obj_docs[80:]
training_docs = train_subj_docs + train_obj_docs
testing_docs = test_subj_docs + test_obj_docs
sent_analyzer = SentimentAnalyzer()
all_words_neg = sent_analyzer.all_words(
[mark_negation(doc) for doc in training_docs])
unigram_feats = sent_analyzer.unigram_word_feats(all_words_neg, min_freq=4)
print(f"unigram feats: {len(unigram_feats)}")
sent_analyzer.add_feat_extractor(extract_unigram_feats,
unigrams=unigram_feats)
training_set = sent_analyzer.apply_features(training_docs)
test_set = sent_analyzer.apply_features(testing_docs)
trainer = NaiveBayesClassifier.train
classifier = sent_analyzer.train(trainer, training_set)
for k, v in sorted(sent_analyzer.evaluate(test_set).items()):
print(f"{k}: {v}")
return sent_analyzer
def __init__(self, n_instances=500):
self.n_instances = n_instances
self.subj_classifier = None
self.sentim_analyzer = None
try:
BASE_DIR = os.path.dirname(
os.path.dirname(os.path.abspath(__file__)))
with open(os.path.join(BASE_DIR, 'main\my_classifier.pickle'),
'rb') as f:
sentim_analyzer = pickle.load(f)
self.sentim_analyzer = sentim_analyzer
except IOError:
with open('plot.tok.gt9.5000') as obj_sents:
obj_sents = obj_sents.read()
with open('quote.tok.gt9.5000') as subj_sents:
subj_sents = subj_sents.read()
self.obj_sents = obj_sents
self.sentim_analyzer = SentimentAnalyzer()
self.train_diploma(subj_sents, obj_sents)
def getNonEmptyEmailBodysTokenized():
db = sql.connect("./data/database.sqlite")
cursor = db.cursor()
#Used to mark all words that come between negations.
sentiment_analyzer = SentimentAnalyzer()
#Load the pre-trained sentence tokenizer.
sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
#Get the keys for the pandas data frame.
cursor.execute('''PRAGMA table_info('Emails');''')
columnNames = []
for row in cursor:
columnNames.append(row[1])
#Retrieve all email data.
cursor.execute(
'''SELECT ExtractedBodyText FROM Emails WHERE (ExtractedBodyText != '');'''
)
dataDict = {'ExtractedBodyText': []}
for row in cursor:
#Extract each column value and add it to the dictionary.
tokenizedToSentences = sent_detector.tokenize(row[0].strip())
tokenizedToWordsList = []
for sentence in tokenizedToSentences:
#Split sentences into words.
tokenizedWords = word_tokenize(sentence)
#Add negation tag to each word that has been negated, from the sentiment utils package.
tokenizedWordsWithNeg = mark_negation(tokenizedWords)
#Re-encode each word
tokenizedWordsWithNeg = [
string.encode('ascii', 'ignore').decode('ascii')
for string in tokenizedWordsWithNeg
]
#Add newly tokenized word to list.
tokenizedToWordsList.append(tokenizedWordsWithNeg)
#Add list of tokenized sentences to dataDict.
dataDict['ExtractedBodyText'].append(tokenizedToWordsList)
#Return pandas data frame with all data.
data = pd.DataFrame(dataDict)
db.close()
return data
def get_nltk_NB(NEG_DATA, POS_DATA, num_train):
train_neg, test_neg = get_nltk_train_test(NEG_DATA, 'neg', num_train)
train_pos, test_pos = get_nltk_train_test(POS_DATA, 'pos', num_train)
training_docs = train_neg + train_pos
testing_docs = test_neg + test_pos
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg)
sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
training_set = sentim_analyzer.apply_features(training_docs)
test_set = sentim_analyzer.apply_features(testing_docs)
trainer = NaiveBayesClassifier.train
classifier = sentim_analyzer.train(trainer, training_set)
#results = []
for key,value in sorted(sentim_analyzer.evaluate(test_set).items()):
print('{0}: {1}'.format(key,value))
def analyze_sentiment(paragraph):
n_instances = 100
subj_docs = [
(sent, 'subj')
for sent in subjectivity.sents(categories='subj')[:n_instances]
]
obj_docs = [(sent, 'obj')
for sent in subjectivity.sents(categories='obj')[:n_instances]]
train_subj_docs = subj_docs[:80]
test_subj_docs = subj_docs[80:100]
train_obj_docs = obj_docs[:80]
test_obj_docs = obj_docs[80:100]
training_docs = train_subj_docs + train_obj_docs
testing_docs = test_subj_docs + test_obj_docs
sentim_analyzer = SentimentAnalyzer()
all_words_neg = sentim_analyzer.all_words(
[mark_negation(doc) for doc in training_docs])
unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg,
min_freq=4)
sentim_analyzer.add_feat_extractor(extract_unigram_feats,
unigrams=unigram_feats)
training_set = sentim_analyzer.apply_features(training_docs)
test_set = sentim_analyzer.apply_features(testing_docs)
trainer = NaiveBayesClassifier.train
classifier = sentim_analyzer.train(trainer, training_set)
sid = SentimentIntensityAnalyzer()
total_sum = 0
count = 0.0
sentences = sent_tokenize(paragraph)
for sentence in sentences:
total_sum += sid.polarity_scores(sentence)["compound"]
count += 1
return total_sum * 10 / count
def run_sa_twitt(train, test):
a = SentimentAnalyzer()
tr = NaiveBayesClassifier.train
all_words = [word for word in a.all_words(train)]
# Add simple unigram word features
unigram_feats = a.unigram_word_feats(all_words, top_n=1000)
a.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
# Add bigram collocation features
bigram_collocs_feats = a.bigram_collocation_feats(
[tweet[0] for tweet in train_twitt], top_n=100, min_freq=12)
a.add_feat_extractor(extract_bigram_feats, bigrams=bigram_collocs_feats)
tr_set = a.apply_features(train)
test_set = a.apply_features(test)
#Training
clf = a.train(tr, tr_set)
res = a.evaluate(test_set)
print(res)
