# apply textblob into sentiment analysis
from textblob import TextBlob
from textblob.sentiments import NaiveBayesAnalyzer
import pandas as pd
import numpy as np
import nltk
# nltk.download('punkt')
# nltk.download('averaged_perceptron_tagger')
#nltk.download('movie_reviews')
dataset = pd.read_csv("airbnb-reviews-part.csv", sep=';')
comments = dataset.iloc[0:500, 5].values
#print(comments)
comment = TextBlob(comments[120], analyzer=NaiveBayesAnalyzer())
print(comment)
print(comment.tags)
print(comment.sentiment)
virtually any safeguard, capable of--as a doomed doctor chillingly
describes it--"assimilating flesh on contact.
Snide comparisons to gelatin be damned, it's a concept with the most
devastating of potential consequences, not unlike the grey goo scenario
proposed by technological theorists. fearful of
artificial intelligence run rampant.
'''
blob = TextBlob(text)
print(blob.tags)
print("\n", blob.noun_phrases)
for sentence in blob.sentences:
print(sentence.sentiment.polarity)
b = TextBlob("I havv goood speling!")
print("Corrected spelling :", b.correct())
#blob3 = TextBlob("What is your name")
#print(blob3.detect_language())
#print(blob3.translate(to='de'))
blob1 = TextBlob("I hate hate hate this library badly",
analyzer=NaiveBayesAnalyzer())
print(blob1.sentiment)
blob2 = TextBlob("I love this library very much",
analyzer=NaiveBayesAnalyzer())
print(blob2.sentiment)
print(blob.sentences)
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
print('Choose an option (1 or 2): ')
print('1. Choose a topic to search tweets for. ')
print('2. Choose a Twitter Username to search tweets for. ')
input_data = input()
if input_data == '1':
print('Enter a topic: ')
topic_name = input()
new_tweets = api.search(q=topic_name)
for tweet in new_tweets:
analysis = TextBlob(tweet.text,
analyzer=NaiveBayesAnalyzer(),
np_extractor=ConllExtractor())
polarity = 'Positive'
if (analysis.sentiment.p_pos < 0.50):
polarity = 'Negative'
print("Sentiment Analysis and Topic of Interest")
print("Tweet : ", tweet.text)
print("Sentiment:", polarity)
print("Confidence : Positive score: ", analysis.sentiment.p_pos * 100,
" Negative score: ", analysis.sentiment.p_neg * 100)
print("Areas of interest: ", analysis.noun_phrases)
print(
"---------------------------------------------------------------------------"
)
else:
def get_sentiment(comment):
blob = TextBlob(comment, analyzer=NaiveBayesAnalyzer())
# blob = TextBlob(comment)
return blob.sentiment.p_pos
def stylebank(img_name, message, out, use_text_api=True):
img = Image.open(img_name)
h, w = (384, 512)
img = img.resize((h, w))
# plt.imshow(img)
seg_model_path = "deeplabv3_mnv2_pascal_train_aug_2018_01_29.tar.gz"
seg_model = DeepLabModel(seg_model_path)
resized_img, seg_map = seg_model.run(img)
resized_img = np.array(resized_img)[:w, :h]
seg_map = seg_map[:w, :h]
mask = extract_largest_blob(seg_map)
mask = fill_holes(mask)
mask_extended = np.zeros((512, 512), dtype=np.bool)
mask_extended[-512:, :384] = mask
seg_image = resized_img * np.repeat(mask[:, :, np.newaxis], 3, axis=2)
seg_image_extended = np.zeros((512, 512, 3), dtype=np.uint8)
seg_image_extended[-512:, :384, :] = seg_image
# sys.path.append('../stylebank/')
content_img_transform = transforms.Compose([
util.Resize(513),
transforms.CenterCrop([513, 513]),
transforms.ToTensor()
])
trans = transforms.ToPILImage()
if use_text_api:
sentiment = indicoio.sentiment(message)
else:
blob = TextBlob(message, analyzer=NaiveBayesAnalyzer())
sentiment = blob.sentiment.p_pos
positive = ['cyberpunk3', 'hotlinemiami2', 'popart1']
neutral = ['anime1', 'popart2', 'walkingdead2']
negative = ['manga1', 'sincity1', 'mickeymouse1']
styles = pd.read_fwf('stylebank/styles_2.txt')
styles = styles.values[:, 0]
if sentiment > 2 / 3:
style = random.choice(positive)
elif sentiment > 1 / 3:
style = random.choice(neutral)
else:
style = random.choice(negative)
style_idx = (styles == style).argmax()
model = StyleBankNet(1).to("cuda")
model.encoder_net.load_state_dict(
torch.load("stylebank/weights_test_2/encoder_2.pth"))
model.decoder_net.load_state_dict(
torch.load("stylebank/weights_test_2/decoder_2.pth"))
model.style_bank[0].load_state_dict(
torch.load(
"stylebank/weights_test_2/bank_2/{}_2.pth".format(style_idx)))
x = content_img_transform(trans(seg_image_extended)).cuda()
styled_image = model(x.expand((1, 3, 513, 513)),
util.get_sid_batch(list(range(1)), 1))
styled_image = styled_image[0].cpu().detach()
styled_image = styled_image.clamp(min=0, max=1)
styled_image = styled_image.cpu().numpy().transpose(1, 2, 0)
styled_image = styled_image[:w, :w, :]
styled_image = 255 * styled_image # Now scale by 255
styled_image = styled_image.astype(np.uint8)
#styled_image = seg_image_extended
rgb = np.zeros(3, dtype=np.uint8)
if sentiment > 0.8:
rgb = [0, 255, 0]
elif sentiment > 0.6:
rgb = [0, 255, 255]
elif sentiment > 0.4:
rgb = [0, 0, 255]
elif sentiment > 0.2:
rgb = [255, 255, 0]
else:
rgb = [255, 0, 0]
gradient = get_gradient(512, color=rgb)
kek = styled_image * np.repeat(mask_extended[:, :, np.newaxis], 3, axis=2)
kek = cv2.cvtColor(kek, cv2.COLOR_RGB2RGBA)
kek[~mask_extended] = 0
asd = transparent_overlay_2(gradient, kek)
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
gray = cv2.cvtColor(resized_img, cv2.COLOR_BGR2GRAY)
# detect faces in the grayscale image
rects = detector(gray, 1)
shape = predictor(gray, rects[0])
shape = face_utils.shape_to_np(shape)
mouth_left_xy = shape[48]
mouth_right_xy = shape[54]
mouth_x, mouth_y = mouth_right_xy
chin_xy = shape[8]
cloud = cv2.imread('data/images/clouds/cloud4.png', -1)
cloud_w = asd.shape[0] - mouth_x
cloud_ratio = cloud.shape[0] / cloud.shape[1]
cloud_h = int(cloud_ratio * cloud_w)
cloud_resized = cv2.resize(cloud, (cloud_w, cloud_h))
cloud_pos_x = mouth_x
cloud_pos_y = mouth_y - int(cloud_h * (5 / 4))
cloud_center_x = cloud_pos_x + int(cloud_w / 5)
lines = check_text(message, max_line_length=cloud_w // 20)
cloud_center_y = cloud_pos_y + int(cloud_h / 2)
transparent_overlay(asd,
cloud_resized,
text=lines,
pos=(cloud_pos_x, cloud_pos_y),
pos_txt=(cloud_center_x, cloud_center_y),
mouth_left_xy=mouth_left_xy,
mouth_right_xy=mouth_right_xy,
chin_xy=chin_xy,
save_path=out)
def test_can_use_different_sentanalyzer(self):
blob = tb.TextBlob("I love this car", analyzer=NaiveBayesAnalyzer())
assert_true(isinstance(blob.analyzer, NaiveBayesAnalyzer))
def test_can_get_subjectivity_and_polarity_with_different_analyzer(self):
blob = tb.TextBlob("I love this car.", analyzer=NaiveBayesAnalyzer())
pattern = PatternAnalyzer()
assert_equal(blob.polarity, pattern.analyze(str(blob))[0])
assert_equal(blob.subjectivity, pattern.analyze(str(blob))[1])
# introduction to the textblob package
from textblob import TextBlob
from nltk.tokenize import TabTokenizer
from textblob import Word
from textblob.sentiments import NaiveBayesAnalyzer
text_blob_object = TextBlob("Tom is great!")
print(text_blob_object.tags)
tokenizer = TabTokenizer()
text_blob_object = TextBlob("Tom is great! Liuna is even better.", tokenizer = tokenizer)
print(text_blob_object.tokens)
print(text_blob_object.words)
print(text_blob_object.sentences)
word = Word("equilibria")
print(word.lemmatize())
text_blob_object = TextBlob("Thiss is a coorrect sentence.")
print(text_blob_object.correct())
text_blob_object = TextBlob("Baby Shark Do do do do do", analyzer=NaiveBayesAnalyzer())
print(text_blob_object.sentiment)
from textblob import TextBlob
from textblob.sentiments import NaiveBayesAnalyzer
from textblob.sentiments import PatternAnalyzer
analysis = TextBlob("He is a greatest liar but he is not that bad", analyzer = NaiveBayesAnalyzer())
#analysis1 = TextBlob("He is a greatest liar but he is not that bad", analyzer = PatternAnalyzer())
print(analysis.polarity)
print(analysis1.polarity)
access_token, access_secret = map(lambda x: x.strip(), secrets.readlines())
# Open pre-downloaded data if present
try:
f = open("tweetdump.pkl", "rb")
searched_tweets = pickle.load(f)
f.close()
# else get from twitter api and save
except FileNotFoundError:
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
api = tweepy.API(auth)
query = 'novartis'
max_tweets = 100
searched_tweets = [
status
for status in tweepy.Cursor(api.search, q=query).items(max_tweets)
]
# save for next time
f = open("tweetdump.pkl", "wb")
pickle.dump(searched_tweets, f)
f.close()
# concatenate tweets
alltext = ''.join([tweet.text for tweet in searched_tweets])
# naive bayes on the text blob
theblob = TextBlob(alltext, analyzer=NaiveBayesAnalyzer())
# sentiment analyser on the blob
print(theblob.sentiment)
from textblob import TextBlob
from textblob.sentiments import NaiveBayesAnalyzer
print(TextBlob("Hummus is amazingly awesome. Hummus is terrible.").sentiment)
print(
TextBlob("Hummus is amazingly awesome. Hummus is terrible.",
analyzer=NaiveBayesAnalyzer()).sentiment.classification)
print(
TextBlob("Hummus is amazingly awesome. Hummus is terrible.",
analyzer=NaiveBayesAnalyzer()).sentiment.p_pos)
#print("sentence = " + sentence)
#print(sentence.sentiment.polarity) # it shows only polarity
#print(sentence.sentiment) # it shows polarity and subjectivity
res1 = sentence.sentiment
res_polarity = res1[0]
res_subjectivity = res1[1]
print("Polarity = '%s' " % res_polarity)
print("Subjectivity = '%s' " % res_subjectivity)
#blob.translate(to="es") # 'La amenaza titular de The Blob...'
#print(TextBlob(text).sentiment)
print("###################################################################")
print("############ MODEL 2 ###############")
model2 = TextBlob(
text1, analyzer=NaiveBayesAnalyzer()) # USE THIS FOR NaiveBayesAnalyzer
# We disagree with Andutta research findings, and wish to highlight here.
# Polarity = 'neg'
# Subjectivity = '0.44760860969048066'
#model2.tags # [('The', 'DT'), ('titular', 'JJ'),
# ('threat', 'NN'), ('of', 'IN'), ...]
#model2.noun_phrases # WordList(['titular threat', 'blob',
# 'ultimate movie monster',
# 'amoeba-like mass', ...])
xx = 0
for sentence in model2.sentences:
xx = xx + 1
print("###################################")
def find_feature(filename, dataFrame, drugList, tokens, concepts):
with open(filename, 'r', encoding='ascii', errors='ignore') as content:
rawData = content.readlines()
corpus = []
for line in rawData:
account, category, tweet = line.strip().split('\t')
tweet = re.sub(
r'(@\_*\w+\_*)|(http://.+\s?)|(pic.twitter.com/.+\s?)|(\#)', '',
tweet)
corpus.append(tweet.lower().strip())
contents = []
for line in corpus:
token = re.findall(r'[A-Za-z]+', line)
contents.append(token)
lexicon = []
for content in contents:
count = 0
for token in content:
if token in tokens:
count += 1
lexicon.append(count)
drugFeature = []
for content in contents:
count = 0
for token in content:
if token in drugList:
count += 1
drugFeature.append(count)
drug_syndrome = list(map(lambda x: x[0] + x[1], zip(lexicon, drugFeature)))
drug = []
for i in drugFeature:
if i > 0:
drug_mention = 'Y'
else:
drug_mention = 'N'
drug.append(drug_mention)
head = '(^|\s+|\'|!|\"|\$|&|\(|\)|\*|\+|,|\-|\.|/|:|;|=|>|\?|\[|\]|\_|~)'
tail = '($|\s+|\'|!|\"|\$|&|\(|\)|\*|\+|,|\-|\.|/|:|;|=|>|\?|\[|\]|\_|~)'
matchObj = []
sentiment = []
for tweet in corpus:
count = 0
blob = tb(tweet, analyzer=NaiveBayesAnalyzer())
if blob.sentiment[0] == 'neg':
mood = 'Y'
else:
mood = 'N'
for event in concepts:
pattern = head + event.strip() + tail
match = re.search(pattern, tweet.strip())
if match:
count += 1
matchObj.append(count)
sentiment.append(mood)
adr = []
for item in matchObj:
if item > 0:
mention = 'Y'
else:
mention = 'N'
adr.append(mention)
col_name = list(dataFrame.columns)
dataFrame.insert(col_name.index('class'), 'drug_syndrome', drug_syndrome)
dataFrame.insert(col_name.index('class'), 'drug_mention', drug)
dataFrame.insert(col_name.index('class'), 'mood', sentiment)
dataFrame.insert(col_name.index('class'), 'ADR_mention', adr)
"""Create, read, update and delete"""
from model import *
from textblob import TextBlob, Blobber
from textblob.sentiments import NaiveBayesAnalyzer
from sqlalchemy import func
#Train NaiveBayesAnalyzer from Movie review lib to only have to train it once
Naive_Analysis = Blobber(analyzer=NaiveBayesAnalyzer())
#### USER CREDENTIAL FUNCTIONS ####
def create_user(user, pwd):
"""Create and return a new user.
>>> create_user("flowers", "flowers")
<Username:flowers>
"""
user = User(username=user)
#Using class fucntion to set password using hash
user.set_password(pwd)
db.session.add(user)
db.session.commit()
return user
def login_check(username, password):
"""Check if email matches password
def is_negative(tweet):
t = TextBlob(tweet, analyzer=NaiveBayesAnalyzer())
if t.sentiment.polarity.classification == "neg":
return True
from PyLyrics import *
from textblob import TextBlob
from textblob.sentiments import NaiveBayesAnalyzer
lyricsOfSong = (PyLyrics.getLyrics('Ed Sheeran ',
'Perfect')) #Print the lyrics directly
lyricsList = lyricsOfSong.split()
print(lyricsOfSong)
for word in lyricsList:
opinion = TextBlob(str(word), analyzer=NaiveBayesAnalyzer())
print(word + "-------")
print(opinion.sentiment)
def test_override_analyzer(self):
b = tb.Blobber(analyzer=NaiveBayesAnalyzer())
blob = b("How now?")
blob2 = b("Brown cow")
assert_true(isinstance(blob.analyzer, NaiveBayesAnalyzer))
assert_true(blob.analyzer is blob2.analyzer)
import sys
import csv
import itertools
import re
import glob
import pandas as pd
from textblob import TextBlob
from textblob.sentiments import NaiveBayesAnalyzer
def clean_tweet(tweet):
return ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ", tweet).split())
for fname in glob.glob('*.csv'):
table = pd.read_csv(fname)
count = 0
table[len(table.columns)] = ""
table[len(table.columns)] = ""
table[len(table.columns)] = ""
for index in table.iterrows():
string = table.ix[count,1]
if(string != None)
{
analysis = TextBlob(clean_tweet(string), analyzer=NaiveBayesAnalyzer())
table.ix[count,len(table.columns)-3] = analysis.sentiment.classification
table.ix[count,len(table.columns)-2] = analysis.sentiment.p_pos
table.ix[count,len(table.columns)-1] = analysis.sentiment.p_neg
}
count = count + 1
table.to_csv(fname, index=False, header=False)
def test_discrete_sentiment(self):
blob = tb.TextBlob("I feel great today.", analyzer=NaiveBayesAnalyzer())
assert_equal(blob.sentiment[0], 'pos')
def compute_blob_class_polarity(msg_body):
pol_class = TextBlob(msg_body, analyzer=NaiveBayesAnalyzer()).sentiment
return {'classification': -1 if pol_class.classification == 'neg' else 1, 'p_pos': pol_class.p_pos, 'p_neg': pol_class.p_neg}
#######################
polar = 0
subj = 0
count = 0
pos = 0
neg = 0
countForSentiment = 0
print("CSV_FILE_NAME:" + csv_file)
for t in csv.DictReader(open(csv_file), delimiter=','):
count = count + 1
into = str(t['title'])
into = into.decode('utf-8')
blob = TextBlob(into, analyzer=NaiveBayesAnalyzer())
print "positivity:", blob.sentiment.p_pos
print "negativity:", blob.sentiment.p_neg
pos = pos + blob.sentiment.p_pos
neg = neg + blob.sentiment.p_neg
test = TextBlob(into)
if (test.sentiment.polarity != 0 or test.sentiment.subjectivity != 0):
polar = polar + test.sentiment.polarity
subj = subj + test.sentiment.subjectivity
print "subjectivity:", test.sentiment.subjectivity
print "polarity:", test.sentiment.polarity
countForSentiment = countForSentiment + 1
print(
str(t['id']) + "\t" + str(test.sentiment.subjectivity) + "\t" +
str(test.sentiment.polarity))
def textblobSentiment_NB_neg(x):
t = TextBlob(x,analyzer=NaiveBayesAnalyzer())
return t.sentiment.p_neg
import csv
import pickle
import re
from textblob.sentiments import NaiveBayesAnalyzer
from textblob.sentiments import PatternAnalyzer
from pprint import pprint
# import keras
pattern_analyzer = PatternAnalyzer()
bayes_analyzer = NaiveBayesAnalyzer()
link_pattern = r"((http|ftp|https):\/\/)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)"
tag_pattern = r"@[\w]*"
emoji_pattern = r"(\:\w+\:|\<[\/\\]?3|[\(\)\\\D|\*\$][\-\^]?[\:\;\=]|[\:\;\=B8][\-\^]?[3DOPp\@\$\*\\\)\(\/\|])(?=\s|[\!\.\?]|$)"
non_char_pattern = r"^[\W]+"
white_spaces_pattern = r"^[\s]|$[\s]"
def csv_to_pickle(input_path, output_path):
try:
with open(input_path, 'r') as f:
reader = csv.reader(f)
lista = list(reader)
tags = [element[0] for element in lista]
tweets = [element[1:] for element in lista]
with open(output_path, 'wb') as fb:
pickle.dump((tweets, tags), fb)
return True
finally:
return False
def get_score(string):
blob = TextBlob(string, analyzer=NaiveBayesAnalyzer())
result = []
result.append(blob.sentiment[1])
result.append(blob.sentiment[2])
return result
import topComments
import averageComments
import finalClassifier
import pandas as pd
app = Flask(__name__)
@app.route('/')
def homepage():
return render_template("main.html")
@app.route('/home')
def homeP():
return render_template("home.html")
blobBaiyes =Blobber(analyzer=NaiveBayesAnalyzer())
blobPattern = Blobber()
analyzer = SentimentIntensityAnalyzer()
@app.route('/createClassifier',methods=['post','get'])
def createClassifer():
if (request.method=='POST'):
print("post method createClassifier")
f =request.files['fileUpload']
negFile = request.files['fileUploadNegative']
if not f and not negFile:
print("not file entered")
else:
file_contentsPos = f.stream.read().decode("latin-1")
file_contentsNeg= negFile.stream.read().decode("latin-1")
print("before writing to the file")
newFile= open("trainingSetPos.txt","w+",encoding='utf-8')
def get_sentiment(argued_text):
formatted_argued_text = TextBlob(argued_text,
analyzer=NaiveBayesAnalyzer())
return formatted_argued_text.sentiment
wordFeatures = wordFeatures(bagOfWords(tweets))
data_trainer = return_trainer()
training_set = nltk.classify.apply_features(getFeatures, tweets)
classifier = nltk.NaiveBayesClassifier.train(training_set)
print(classifier.show_most_informative_features(32))
ConsumerKey = "6TO19L8LlJouqnztJ6hZkCgsA"
ConsumerSecret = "gK5dcQFlgS1lLTMPdhsqh046v48VfGYIpWqENwEhwLaFEpX0Pv"
AccessToken = "257611954-EBfaOkObL04YTCB2NEC39C5GzhyTIGMGcF1TNXul"
AccessTokenSecret = "YrvPzGj97TsLR9XRUp5ESbP3KxlMfezHVnmex5RNZb3y6"
twitter = Twython(ConsumerKey, ConsumerSecret, AccessToken, AccessTokenSecret)
queryText = "#VemPraRuaBrasil"
result = twitter.search(q=queryText)
for status in result["statuses"]:
print("Tweet: {0} \n Sentiment: {1} \n".format(
status["text"],
classifier.classify(getFeatures(status["text"].split()))))
for status in result["statuses"]:
blob = TextBlob(status['text'], analyzer=NaiveBayesAnalyzer())
print("Tweet: {0} \n Sentiment: {1} \n".format(status['text'],
blob.sentiment))
from textblob import TextBlob
from textblob.sentiments import NaiveBayesAnalyzer
blob = TextBlob("It is very bad, I am so sad", analyzer=NaiveBayesAnalyzer())
print(blob.sentiment.p_pos)
# credential go to James Tollefson: https://www.kaggle.com/jamestollefson/enron-network-analysis
########################################################################
import numpy as np
import pandas as pd
import re
from datetime import datetime
# text processing
from textblob import TextBlob, Blobber
from nltk.tokenize import TabTokenizer
from textblob.sentiments import NaiveBayesAnalyzer
tb = Blobber(analyzer=NaiveBayesAnalyzer())
# defining helper functions
def get_text(Series, row_num_slicer):
"""returns a Series with text sliced from a list split from each message. Row_num_slicer
tells function where to slice split text to find only the body of the message."""
result = pd.Series(index=Series.index)
for row, message in enumerate(Series):
message_words = message.split('\n')
del message_words[:row_num_slicer]
message_words = ' '.join(message_words)
result.iloc[row] = message_words
return result
def get_row(Series, row_num):
"""returns a single row split out from each message. Row_num is the index of the specific
#! python2
import mysql.connector as mariadb
from textblob import TextBlob
from textblob.sentiments import NaiveBayesAnalyzer
import unicodedata
mariadb_connection = mariadb.connect(user='******', password='******', database='tesina')
cursor = mariadb_connection.cursor()
cursorInsert = mariadb_connection.cursor()
cursor.execute("SELECT a.id,b.texto_procesado from Stocks a inner join Noticias b on a.id=b.id where a.id<=1000")
id_motor="3"
filas = cursor.fetchall()
for texto in filas:
tt= texto[1]
print tt
blob = TextBlob(tt, analyzer=NaiveBayesAnalyzer())
print blob.sentiment
sql = "INSERT INTO Sentimientos(id_motor, puntaje_positivo, puntaje_negativo,sentimiento, noticia) values (" + id_motor +", " + str(blob.sentiment.p_pos) + ","+ str(blob.sentiment.p_neg) + ",'"+ blob.sentiment.classification +"',"+ str(texto[0])+")"
cursorInsert.execute(sql)
mariadb_connection.commit()
mariadb_connection.close()
