def sentence_sentiment(data_set):
speech_sentences = sent_tokenize(data_set)
decisions = []
for sentence in speech_sentences:
sentiment = s.sentiment(sentence)
decisions.append(sentiment)
return decisions
def on_data(self, data):
try:
#print (data) --only for full data
# tweet = data.split (',"text":"')[1].split('","source')[0] #trims down to text
#saveThis = str(time.time())+ '::'+tweet #avoid common punctuation
# saveThis=open('tweetDB2.csv','a')
# saveThis.write(tweet)
# saveThis.write('\n')
# saveThis.close()
#
'''
saveFile=open('tweetDB.csv','a')#create a file iot save the data
saveFile.write(data)
saveFile.write('\n')
saveFile.close()
'''
all_data = json.loads(data)
tweet = all_data["text"]
sentiment_value, confidence = s.sentiment(tweet)
print(tweet, sentiment_value, confidence)
if confidence * 100 >= 80:
output = open("twitter-out.txt", "a")
output.write(sentiment_value)
output.write('\n')
output.close()
return True
except BaseException as e: #rate limitation
print('failed ondata,', str(e))
time.sleep(8) #avoids constant reconnection ico of rate limitation
def on_data(self, data):
all_data = json.loads(data)
tweet = all_data["text"]
sentiment_value, confidence = s.sentiment(tweet)
print(tweet, sentiment_value, confidence)
if confidence * 100 >= 80:
output = open("twitter-out.txt", "a")
output.write(sentiment_value)
output.write('\n')
output.close()
return True
def analyze_sentiment(election):
df = pd.read_csv('../data/{}.csv'.format(election))
positions = []
for comment in df.comment:
# print(comment)
pos, conf = s_mod.sentiment(comment)
# print(pos, conf)
positions += [coefficient[pos] * conf]
df['position'] = pd.Series(positions)
return election, df
def on_data(self, data):
all_data = json.loads(data)
tweet = all_data["text"]
sentiment_val, confidence = s_mod.sentiment(tweet)
print('Tweet: {} \n Sentiment: {} \n Confidence: {}'.format(tweet, sentiment_val, confidence))
if confidence*100 >= 80:
out = open(dir + 'twitter_output.txt', 'a')
out.write(sentiment_val)
out.write('\n')
out.close()
return True
def manual_rules(input):
allowed_word_types = ["J", "V"]
tokens = word_tokenize(input)
pos = nltk.pos_tag(tokens)
has_allowed_words = False
all_words = []
for w in pos:
if w[1][0] in stop_words:
continue
elif w[1][0] not in allowed_word_types:
continue
else:
all_words.append(w[0].lower())
has_allowed_words = True
filtered_input = ' '.join(str(x) for x in all_words)
print("ALL WORDS", all_words)
print("FILTERED INPUT", filtered_input)
if has_allowed_words:
# Classify with machine learning classifier
result = ml_classifiers.sentiment(filtered_input)
# Negation Handling
# Check for sentiment inversion
if result[0] == 'pos' and should_invert(input):
result = ('neg', result[1])
elif result[0] == 'neg' and should_invert(input):
result = ('pos', result[1])
# Sentiment score calculation
# Might need filtering before calculating the score
score = sentiment_score(input)
# Check for polarity flips
if should_invert(input):
score *= -1
# Assign result values
if result[0] == 'pos':
return (result, score)
elif result[0] == 'neg':
return (result, score)
else:
return (('neutral', 100), 0)
def on_data(self, data):
try:
all_data = json.loads(data)
tweet = all_data["text"]
(sentiment_val, confidence) = s.sentiment(tweet)
print(tweet, sentiment_val, confidence)
if confidence >= 60:
output = open("twitter-output.txt", "a")
output.write(sentiment_val)
output.write('\n')
output.close()
print(tweet)
return True
except:
return True
def cohesive_classify(text):
ml_result = ml_classifiers.sentiment(text)[0]
tb_polarity = tb(text).sentiment.polarity
if tb_polarity > 0:
tb_result = "pos"
elif tb_polarity < 0:
tb_result = "neg"
else:
tb_result = "ambiguous"
if tb_result and ml_result == 'pos':
result = "pos"
elif tb_result and ml_result == 'neg':
result = "neg"
else:
result = "ambiguous"
return result
def on_data(self, data):
#while self.tweet_count < self.max_tweet_count:
jsonData = json.loads(data)
#if not jsonData['retweeted'] and 'RT @' not in jsonData['text']:
today = date.today()
try:
if 'text' in jsonData:
tweet = jsonData["text"]
sentiment_value, confidence = sent_mod.sentiment(tweet)
myCur.execute("INSERT INTO tweets (my_date, tweet, sentiment, confidence) Values (%s, %s, %s, %s)", #VALUES ({0}, '{1}', '{2}', {3})".format(time.time(), tweet, sentiment_value, confidence))
(today, tweet, sentiment_value, confidence))
print("IT IS WORKING!!!")
conn.commit()
return True
else:
return True
except Exception:
pass
def on_data(self, data):
#while self.tweet_count < self.max_tweet_count:
jsonData = json.loads(data)
#if not jsonData['retweeted'] and 'RT @' not in jsonData['text']:
today = date.today()
try:
if 'text' in jsonData:
tweet = jsonData["text"]
sentiment_value, confidence = sent_mod.sentiment(tweet)
myCur.execute(
"INSERT INTO tweets (my_date, tweet, sentiment, confidence) Values (%s, %s, %s, %s)", #VALUES ({0}, '{1}', '{2}', {3})".format(time.time(), tweet, sentiment_value, confidence))
(today, tweet, sentiment_value, confidence))
print("IT IS WORKING!!!")
conn.commit()
return True
else:
return True
except Exception:
pass
import sentiment_module as s
print(s.sentiment("this is good news"))
print(s.sentiment("this is bad news"))
print(s.sentiment("End of the world is cumming"))
print(s.sentiment("I am ill and i do not have any money I kill myself"))
print(s.sentiment("python and nltk are great "))
posNews = """ Sea Cow Population Thriving in Australia Thanks to Baby Boom
In 2011, the dugong population of the Great Barrier Reef was estimated to be 600 without any sign of calves
or newborns – the lowest it had been since the 1980s. The condition of the species, categorized as
“vulnerable to extinction”, caused great alarm amongst conservationists.
Half a decade later, however, sea cows have rebounded to about 5,000 in number – 10% of which are calves,
according to aerial surveys conducted off the east coast of Queensland.
This exciting comeback can be credited to the return of sea grass to the region; the main component of a
sea cow’s diet. Most of the sea grass near the Great Barrier Reef was wiped out due to the effects of Cyclone Yasi."""
print(s.sentiment(posNews))
import sentiment_module as s
print(
s.sentiment(
"the movie was awesome really nice mix of action , comedy and romance")
)
print(
s.sentiment(
"This party was illogical non sense manisfesto. its leader is criminal and this party has resorted to bad practises"
))
import sentiment_module as s_mod
pos_ex = 'The movie was good. ' \
'The movie had a happy ending.' \
'Sound of the movie was perfect.' \
'Lovely movie, every should go watch it once.' \
'The acting was great, plot was wonderful, and the casting was splendid.'
neg_ex = 'The movie was horrible. ' \
'Nothing is good about the moive. ' \
'This movie is not worth the time to watch. ' \
'The movie is a waste of time.'
print(s_mod.sentiment(pos_ex))
print(s_mod.sentiment(neg_ex))
def manualSentiment():
if request.method == 'POST':
# Get Form Fields
input = request.form['keyword']
# Basic validation
if len(input) > 0:
# Extract verbs and adjectives
try:
allowed_word_types = ["J", "V"]
tokens = word_tokenize(input)
pos = nltk.pos_tag(tokens)
has_allowed_words = False
all_words = []
# Filter out stop-words (e.g. the, on, at)
for w in pos:
if w[1][0] in stop_words:
continue
elif w[1][0] in allowed_word_types:
all_words.append(w[0].lower())
has_allowed_words = True
# Combine the filtered sentence
filtered_input = ' '.join(str(x) for x in all_words)
print(all_words)
print(filtered_input)
# Check if sentence contains sentiment words
if has_allowed_words:
# Classify with machine learning classifier
result = ml_classifiers.sentiment(filtered_input)
# Negation Handling & Check for sentiment inversion
if result[0] == 'pos' and should_invert(input):
result = ('neg', result[1])
elif result[0] == 'neg' and should_invert(input):
result = ('pos', result[1])
# Sentiment score calculation
score = sentiment_score(input)
# Check for polarity flips
if should_invert(input):
score *= -1
# Form a list of top notable features
j = 1
showable_feature_list = []
for word in all_words:
if word in showable_feature_list:
j += 1
continue
else:
number = str(j) + ". " + word
showable_feature_list.append(number)
j += 1
# Generate list of features
important_features = ', '.join(
str(x) for x in showable_feature_list)
# Display result
if result[0] == 'pos':
flash(
'The sentiment of the text is Positive with the confidence of '
+ str(round(result[1] * 100, 2)) + "%" +
". Manual calculated score = " +
str(round(score, 2)), 'success')
flash("Top features:\t" + important_features,
'primary')
elif result[0] == 'neg':
flash(
'The sentiment of the text is Negative with the confidence of '
+ str(round(result[1] * 100, 2)) + "%" +
". Manual calculated score = " +
str(round(score, 2)), 'danger')
flash("Top features:\t" + important_features,
'primary')
else:
flash(
'The sentiment of the text is Neutral with the confidence of 100%. Manual calculated score = 0',
'warning')
flash(
'Make sure to use words which express sentiment, e.g. adjectives and verbs to make the best use of this system',
'warning')
except Exception as e:
flash(
'A server side error has occured ' + "Exception: \"" +
str(e) + "\"", 'warning')
return redirect(
url_for('manualSentiment', title="Manual Sentiment"))
else:
flash('Please enter some text to be analyzed', 'warning')
return render_template('manualsentiment.html', title="Manual Sentiment")
csecret="XXXXXXXXXXXXXXXX"
atoken="XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
asecret="XXXXXXXXXXXXXXXXXXXXXXXX"
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
api = tweepy.API(auth)
cursor = tweepy.Cursor(api.search, q='#Trump', result_type="recent", lang='en')
tweets = [tweet.text for tweet in cursor.items(1000)]
sentiments = []
for t in tweets:
sentiment_value, confidence = s.sentiment(t)
if confidence > 0.80:
sentiments.append(sentiment_value)
print t, sentiment_value, confidence
x = 0.00
for sent in sentiments:
if sent == "pos":
x += 1.00
print len(sentiments)
try:
print "%"+str(x / len(sentiments) * 100)
except ZeroDivisionError:
pass
import sentiment_module as s
print(
s.sentiment(
"This movie is awesome, what ever the review may be but awesome, from my point of view"
))
print(
s.sentiment(
"This movie is ugly and the last part I will give 2/10 marks to it "))
print(
s.sentiment(
"This movie is not bad, but the last part I will give 2/10 marks to it "
))
print(
s.sentiment(
"This movie is not good, but the last part I will give 2/10 marks to it "
))
neg_ex = 'The movie was horrible. ' \
'Nothing is good about the moive. ' \
'This movie is not worth the time to watch. ' \
'The movie is a waste of time.'
print(s_mod.sentiment(pos_ex))
print(s_mod.sentiment(neg_ex))
'''
elections = ['presidential', 'house', 'senate', 'gov']
coefficient = {'pos': 1, 'neg': -1}
election = elections[2]
df = pd.read_csv('../data/{}.csv'.format(election))
positions = []
for comment in df.comment:
# print(comment)
pos, conf = s_mod.sentiment(str(comment))
# print(pos, conf)
positions += [coefficient[pos] * conf]
df['position'] = pd.Series(positions)
print(df.head())
df.to_csv('../data/{}_labeled.csv'.format(election), index=False)
df.to_json('../data/{}_labeled.json'.format(election), orient='records')
# -*- coding: utf-8 -*-
"""
Created on Wed May 8 22:00:13 2019
@author: Juliette
"""
import sentiment_module as s
print(s.sentiment("it was not great"))
print(
s.sentiment(
"Paid critics gotta temper their criticism because they need access. I get it. But f**k the critics who are coming after and frankly mocking fans for their critical comments on GoT. We are not the hypocrites in this situation."
))
import sentiment_module as s
print(s.sentiment("This movie was inspiring good. timepass awesome"))
print(
s.sentiment(
"This movie was f*****g awesome! The acting was great, plot was wonderful, and there were pythons...so yea!"
))
print(s.sentiment("This movie was inspiring good.so awesome well directed"))
from sentiment_module import sentiment
import io
pos_count = 0
pos_correct = 0
pos_lines = io.open("main/res/data/sample/positive.txt",
encoding="latin-1").read().split('\n')
for line in pos_lines:
if sentiment(line, isAnime=False) == 'pos':
pos_correct += 1
pos_count += 1
neg_count = 0
neg_correct = 0
neg_lines = io.open("main/res/data/sample/negative.txt",
encoding="latin-1").read().split('\n')
for line in neg_lines:
if sentiment(line, isAnime=False) == 'neg':
neg_correct += 1
neg_count += 1
print("Positive Accuracy: {}%, Sample Size: {}, Correct Identified: {}".format(
(pos_correct / pos_count) * 100.0, pos_count, pos_correct))
print("Negative Accuracy: {}%, Sample Size: {}, Correct Identified: {}".format(
(neg_correct / neg_count) * 100.0, neg_count, neg_correct))