def testConstruct(self):
request = self.request
self.assertEqual(self.URL, request.getUrl())
self.assertIsNotNone(request.getParams())
params = request.getParams()
self.assertEqual('key' in params.keys(), True)
self.assertEqual(params['key'], self.KEY)
self.assertIsNotNone(request.getTimeout())
self.assertEqual(self.TIMEOUT_DEFAULT, request.getTimeout())
extraHeaders = ["Accept: application/json"]
request2 = meaningcloud.SentimentRequest(self.KEY,
lang=self.lang,
txt=self.text,
txtf=self.txtf,
extraheaders=extraHeaders)
self.assertIsNotNone(request2.sendReq())
otherparams = {'key2': 'my_key2'}
request3 = meaningcloud.SentimentRequest(self.KEY,
lang=self.lang,
txt=self.text,
txtf=self.txtf,
extraheaders=extraHeaders,
otherparams=otherparams)
self.assertIsNotNone('key2' in request3.getParams().keys(), True)
self.assertEqual(request3.getParams()['key2'], 'my_key2')
url = 'https://en.wikipedia.org/wiki/Star_Trek'
request4 = meaningcloud.SentimentRequest(self.KEY,
lang=self.lang,
url=url,
txtf=self.txtf,
extraheaders=extraHeaders,
otherparams=otherparams)
self.assertIsNotNone('url' in request4.getParams().keys(), True)
self.assertEqual(request4.getParams()['url'], url)
file = self.RESOURCES_DIR + 'file.txt'
request5 = meaningcloud.SentimentRequest(self.KEY,
lang=self.lang,
doc=file,
txtf=self.txtf,
extraheaders=extraHeaders,
otherparams=otherparams)
self.assertIsNotNone('doc' in request5.getParams().keys(), False)
doc = request5._file['doc'].read().decode('utf-8')
request5._file['doc'].close()
aux_doc = open(self.RESOURCES_DIR + 'file.txt', 'rb')
aux_conten = aux_doc.read().decode('utf-8')
aux_doc.close()
self.assertEqual(aux_conten, doc)
return request
def runMeaningCloud(text, license_key):
try:
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(license_key,
lang='en',
txt=text,
txtf='plain').sendReq())
# If there are no errors in the request, we print the output
if (sentiment_response.isSuccessful()):
#print("\nThe request to 'Sentiment Extraction' finished successfully!\n")
sentiments = sentiment_response.getResults()
if (sentiments):
return sentiments
else:
print("\nOh no! There was the following error: " +
topics_response.getStatusMsg() + "\n")
else:
if (sentiment_response.getResponse() is None):
print("\nOh no! The request sent did not return a Json\n")
else:
print("\nOh no! There was the following error: " +
topics_response.getStatusMsg() + "\n")
except ValueError:
e = sys.exc_info()[0]
print("\nException: " + str(e))
def getSentimentAnalysis(text):
polarity = ''
# We are going to make a request to the Sentiment Analysis API
print("\tGetting sentiment analysis...")
sentiment_response = meaningcloud.SentimentResponse(meaningcloud.SentimentRequest(license_key, lang='en', txt=text, txtf='markup').sendReq())
if sentiment_response.isSuccessful():
polarity = sentiment_response.getGlobalScoreTag()
else:
print("\tOops! Request to sentiment was not succesful: (" + sentiment_response.getStatusCode() + ') ' + sentiment_response.getStatusMsg())
return polarity
def request_tweet_sans_compte(nombreTweet):
auth = tweepy.OAuthHandler(
'74kR1Uv1LGw3qllwqyIrIW2WH',
'GyVxV3O4Wks9akUFxRQAmXh1nVvtTnvvLi1nsMdkLt4EjqxIrO')
auth.set_access_token('1257599001433706496-HKChl1uTs0M7aHo2B80dk4Yy9mwyh4',
'VE2qcM0lGIY4ux1HGOaz4y2jfuIVZ0qMCMrSbPJkdadRM')
api = tweepy.API(auth)
page_list = []
tweets = []
n = 0
if (nombreTweet > 200):
for page in tweepy.Cursor(api.home_timeline,
count=200,
tweet_mode="extended").pages(
int(nombreTweet / 200)):
page_list.append(page)
n = n + 1
print(n)
for page in page_list:
for tweet in page:
tweets.append(tweet)
else:
tweets = api.home_timeline(count=nombreTweet, tweet_mode="extended")
liste_tweets = []
liste_ids = []
for tweet in tweets:
liste_tweets.append(tweet.full_text)
liste_ids.append(tweet.id)
model = 'IAB_en'
license_key = 'af76f805c5f86590fee318303f3d3ac3'
result = []
i = -1
for tweet in liste_tweets:
i = i + 1
if len(tweet.split(' ')) >= 10:
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(license_key,
lang='en',
txt=tweet,
txtf='plain').sendReq())
score = sentiment_response.getResponse().get('score_tag')
if (str(score) == "P"):
result.append([tweets[i], tweet, 1])
if (str(score) == "P+"):
result.append([tweets[i], tweet, 2])
return result
def request_tweet():
auth = tweepy.OAuthHandler(
'74kR1Uv1LGw3qllwqyIrIW2WH',
'GyVxV3O4Wks9akUFxRQAmXh1nVvtTnvvLi1nsMdkLt4EjqxIrO')
auth.set_access_token('999402738290233344-GB6h7ahayXolXu6MYTMVlc4VKPAbOPN',
'c0wB3SwFjiTpTWnR6fyudBX9Sel7SxUwxKeJ4FLJpaHcV')
api = tweepy.API(auth)
tweets = api.home_timeline(tweet_mode="extended",
include_rts=False,
count=50)
liste_tweets = []
liste_id = []
for tweet in tweets:
liste_tweets.append(tweet.full_text.split('http')[0])
liste_id.append(tweet.id)
model = 'IAB_en'
license_key = 'af76f805c5f86590fee318303f3d3ac3'
i = 0
liste_sentiment = []
for tweet in liste_tweets:
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(license_key,
lang='en',
txt=tweet,
txtf='plain').sendReq())
liste_sentiment.append([
str(tweet), liste_id[i],
sentiment_response.getResponse().get('score_tag')
])
i = i + 1
df = pandas.DataFrame(liste_sentiment, columns=["tweet", "id", "polarity"])
for i in range(len(df)):
print(df.loc[[i], ["tweet"]].values)
print(df.loc[[i], ["id"]].values)
print(df.loc[[i], ["polarity"]])
id_list = []
for i in range(len(df)):
if (str(df.iloc[i]["polarity"]) == "P"
or str(df.iloc[i]["polarity"]) == "P+"):
id_list.append(id_list(df.iloc[i]["id"]))
return id_list
def analyzeText(text, lang):
global index_count
print("Analyzing sentiment for text #%s" % str(index_count))
# this is where we are going to store our results
polarity = ""
subjectivity = ""
irony = ""
agreement = ""
confidence = ""
try:
# We are going to make a request to the Sentiment Analysis API
request = meaningcloud.SentimentRequest(license_key,
lang=lang,
txt=text,
server=server)
setRequestSource(request)
response = meaningcloud.SentimentResponse(request.sendReq())
if response.isSuccessful():
polarity = response.scoreTagToString(response.getGlobalScoreTag())
subjectivity = response.getSubjectivity()
irony = response.getIrony()
agreement = response.getGlobalAgreement()
confidence = response.getGlobalConfidence()
else:
if isBlockingErrorType(response.getStatusCode()):
raise ValueError(
"Something went wrong in the MeaningCloud request!: (" +
response.getStatusCode() + ") " + response.getStatusMsg())
else:
print("Oops! The request to Sentiment Analysis for text #" +
str(index_count) + " was not succesful: (" +
response.getStatusCode() + ") " +
response.getStatusMsg())
polarity = ("ERROR (" + response.getStatusCode() + "): " +
response.getStatusMsg())
except ValueError as e:
raise ValueError(str(e))
index_count += 1
return pd.Series([polarity, subjectivity, irony, agreement, confidence])
def meaning_sentiment(tupleFiles):
rawtweets = tupleFiles[0]
license_key = {YOUR_MEANINGCLOUD_LICENSE_KEY}
filenameCSV = 'tweets.csv'
filepathCSV = '/'+filenameCSV
with open(filenameCSV, 'w') as csv_file:
fieldnames = ['sentiment', 'tweet']
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writeheader()
with open(rawtweets, 'r') as file_tweets:
for line in file_tweets.readlines():
sentiment_response = meaningcloud.SentimentResponse(meaningcloud.SentimentRequest(license_key, lang='es', txt=line, txtf='plain').sendReq())
s = sentiment_response.getGlobalScoreTag()
writer.writerow({'sentiment': s , 'tweet': line})
tupleFilesCSV = (filenameCSV,filepathCSV)
return tupleFilesCSV
def get_sentiment_analysis(self):
logger.debug('TextAnalysis.get_sentiment_analysis()')
if self.sentiment_analysis is None:
logger.debug('sentiment_analysis not set')
logger.debug('setting sentiment_analysis')
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(
self.license_key,
lang=self.language,
txt=self.text,
txtf='plain',
).sendReq())
# Save the raw and serialized sentiment analysis
self.sentiment_analysis_raw = sentiment_response.getResults()
print(self.sentiment_analysis_raw)
self.sentiment_analysis = MeaningCloudResponse(
self.sentiment_analysis_raw).absa()
logger.debug(self.sentiment_analysis)
return self.sentiment_analysis
def analisisMeaning(full_text):
analisis = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(license_key_MC,
lang='es',
txt=full_text,
txtf='plain').sendReq())
if analisis.getGlobalScoreTag() == 'N+':
polaridad = "Negativo"
polarity = -1
elif analisis.getGlobalScoreTag() == 'N':
polaridad = "Negativo"
polarity = -0.5
elif analisis.getGlobalScoreTag() == 'P':
polaridad = "Positivo"
polarity = 0.5
elif analisis.getGlobalScoreTag() == 'P+':
polaridad = "Positivo"
polarity = 1
else:
polaridad = "Neutro"
polarity = 0
return (polaridad, polarity)
def extractFeatures(self):
# The language of the tweets
tweetLang = 'en'
# Tweet counter
counter = 0
# List of the IDs of tweets whose requests failed
wrongRequestTweetIds = []
# Open the data set file
with open(self.dataSetFilePath, 'rb') as f:
# Flag to know whether the first line of the file (column names)
# has been read
headerPassed = False
for line in f:
line = line.decode(errors='ignore')
row = line.split(',')
# i.e. if this is the first line of the file
if headerPassed == False:
headerPassed = True
continue
# Instantiate a tweet. Assume that the
# current CSV file which is being read
# already contains the following features
# per tweet:
#
# - text,
# - isBot
# - joy
# - surprise
# - fear
# - sadness
# - anger
#
t = Tweet(text=row[0],
isBot=int(row[1]),
joy=float(row[2]),
surprise=float(row[3]),
fear=float(row[4]),
sadness=float(row[5]),
anger=float(row[6]))
text = t.text
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(self.apiKey,
lang=tweetLang,
txt=text,
txtf='plain').sendReq())
if (sentiment_response.isSuccessful()):
# Populate the remaining fields of this Tweet instance
t.score_tag = sentiment_response.getGlobalScoreTag()
t.agreement = sentiment_response.getGlobalAgreement()
t.subjectivity = sentiment_response.getSubjectivity()
t.confidence = sentiment_response.getGlobalConfidence()
t.irony = sentiment_response.getIrony()
self.tweetList.append(t)
else:
# Let's see why the request failed
requestStatusCode = sentiment_response.getStatusCode()
requestStatusMsg = sentiment_response.getStatusMsg()
print("Tweet id = " + str(counter) +
"; request status code = " + str(requestStatusCode) +
"; request status msg = " + str(requestStatusMsg))
# Store this tweet ID for later. Another request will have to be made.
wrongRequestTweetIds.append(counter)
counter += 1
print("Tweets analyzed so far: " + str(counter))
# Wait for a few seconds before making a new request
time.sleep(self.requestIntervalSeconds)
print("\nTotal analyzed tweets = " + str(counter))
# If all requests succeeded
if len(wrongRequestTweetIds) == 0:
print("All " + str(counter) + " requests were successful.")
else:
print("The requests for the following tweets failed:\n")
print(wrongRequestTweetIds)
liste_tweets = []
liste_id = []
for tweet in tweets:
liste_tweets.append(tweet.full_text.split('http')[0])
liste_id.append(tweet.id)
model = 'IAB_en'
license_key = 'af76f805c5f86590fee318303f3d3ac3'
i = 0
liste_sentiment = []
for tweet in liste_tweets:
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(license_key,
lang='en',
txt=tweet,
txtf='plain').sendReq())
liste_sentiment.append([
str(tweet), liste_id[i],
sentiment_response.getResponse().get('score_tag')
])
i = i + 1
df = pandas.DataFrame(liste_sentiment, columns=["tweet", "id", "polarity"])
for i in range(len(df)):
print(df.loc[[i], ["tweet"]].values)
print(df.loc[[i], ["id"]].values)
print(df.loc[[i], ["polarity"]])
for i in range(len(df)):
class SentimentRequestTest(unittest.TestCase):
URL = 'https://api.meaningcloud.com/sentiment-2.1'
KEY = 'MY_KEY'
TIMEOUT_DEFAULT = 60
RESOURCES_DIR = './resources/'
text = 'London is big'
lang = 'en'
txtf = 'plain'
request = meaningcloud.SentimentRequest(KEY,
lang=lang,
txt=text,
txtf=txtf)
def testConstruct(self):
request = self.request
self.assertEqual(self.URL, request.getUrl())
self.assertIsNotNone(request.getParams())
params = request.getParams()
self.assertEqual('key' in params.keys(), True)
self.assertEqual(params['key'], self.KEY)
self.assertIsNotNone(request.getTimeout())
self.assertEqual(self.TIMEOUT_DEFAULT, request.getTimeout())
extraHeaders = ["Accept: application/json"]
request2 = meaningcloud.SentimentRequest(self.KEY,
lang=self.lang,
txt=self.text,
txtf=self.txtf,
extraheaders=extraHeaders)
self.assertIsNotNone(request2.sendReq())
otherparams = {'key2': 'my_key2'}
request3 = meaningcloud.SentimentRequest(self.KEY,
lang=self.lang,
txt=self.text,
txtf=self.txtf,
extraheaders=extraHeaders,
otherparams=otherparams)
self.assertIsNotNone('key2' in request3.getParams().keys(), True)
self.assertEqual(request3.getParams()['key2'], 'my_key2')
url = 'https://en.wikipedia.org/wiki/Star_Trek'
request4 = meaningcloud.SentimentRequest(self.KEY,
lang=self.lang,
url=url,
txtf=self.txtf,
extraheaders=extraHeaders,
otherparams=otherparams)
self.assertIsNotNone('url' in request4.getParams().keys(), True)
self.assertEqual(request4.getParams()['url'], url)
file = self.RESOURCES_DIR + 'file.txt'
request5 = meaningcloud.SentimentRequest(self.KEY,
lang=self.lang,
doc=file,
txtf=self.txtf,
extraheaders=extraHeaders,
otherparams=otherparams)
self.assertIsNotNone('doc' in request5.getParams().keys(), False)
doc = request5._file['doc'].read().decode('utf-8')
request5._file['doc'].close()
aux_doc = open(self.RESOURCES_DIR + 'file.txt', 'rb')
aux_conten = aux_doc.read().decode('utf-8')
aux_doc.close()
self.assertEqual(aux_conten, doc)
return request
def testSendReq(self):
request = self.request
requestRq = request.sendReq()
self.assertIsNotNone(requestRq)
def analyzeText(text):
global index_count
print("Analyzing text " + str(index_count))
# this is where we are going to store our results
polarity = ''
entities = ''
concepts = ''
iab2 = ''
try:
# We are going to make a request to the Sentiment Analysis API
print("\tGetting sentiment analysis...")
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(license_key,
lang='en',
txt=text,
txtf='markup').sendReq())
if sentiment_response.isSuccessful():
polarity = sentiment_response.getGlobalScoreTag()
else:
print('Request to sentiment was not succesful: ' +
sentiment_response.getStatusMsg())
# We are going to make a request to the Topics Extraction API
print("\tGetting entities and concepts...")
topics_req = meaningcloud.TopicsRequest(license_key,
txt=text,
lang='en',
topicType='ec',
otherparams={'txtf': 'markup'})
topics_response = meaningcloud.TopicsResponse(topics_req.sendReq())
# If there are no errors in the request, we extract the entities and concepts
if topics_response.isSuccessful():
entities_list = topics_response.getEntities()
formatted_entities = []
if entities_list:
for entity in entities_list:
if int(
topics_response.getTopicRelevance(entity)
) >= 100: #we limit the entities to those with relevance higher than 100
formatted_entities.append(
topics_response.getTopicForm(entity) + ' (' +
topics_response.getTypeLastNode(
topics_response.getOntoType(entity)) + ')')
entities = ', '.join(formatted_entities)
concepts_list = topics_response.getConcepts()
formatted_concepts = []
if concepts_list:
for concept in concepts_list:
if int(
topics_response.getTopicRelevance(concept)
) >= 100: #we limit the entities to those with relevance higher than 100
formatted_concepts.append(
topics_response.getTopicForm(concept))
concepts = ', '.join(list(dict.fromkeys(formatted_concepts)))
else:
print('Request to topics was not succesful: ' +
topics_response.getStatusMsg())
# We are going to make a request to the Deep Categorization API
print("\tGetting IAB 2.0 classification...")
deepcat_response = meaningcloud.DeepCategorizationResponse(
meaningcloud.DeepCategorizationRequest(license_key,
model='IAB_2.0_en',
txt=text,
otherparams={
'txtf': 'markup'
}).sendReq())
if deepcat_response.isSuccessful():
categories = deepcat_response.getCategories()
iab2 = (', '.join(
deepcat_response.getCategoryCode(cat)
for cat in categories[:1])) if categories else ''
else:
print('Request to Deep Categorization was not succesful: ' +
deepcat_response.getStatusMsg())
except ValueError:
e = sys.exc_info()[0]
print("\nException: " + str(e))
index_count += 1
return pd.Series([polarity, entities, concepts, iab2])
def sentiment(self, inp):
return meaningcloud.SentimentResponse(meaningcloud.SentimentRequest(cred['sent-api2'], lang='en', txt=inp, txtf='plain').sendReq())