def sentiment_calc(Review):
try:
return mc.SentimentResponse(
mc.ClassRequest(license_key, lang='en', txt=Review,
txtf='Plain').sendReq)
except:
return None
def runMeaningCloud(text, license_key):
try:
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(license_key,
lang='en',
txt=text,
txtf='plain').sendReq())
# If there are no errors in the request, we print the output
if (sentiment_response.isSuccessful()):
#print("\nThe request to 'Sentiment Extraction' finished successfully!\n")
sentiments = sentiment_response.getResults()
if (sentiments):
return sentiments
else:
print("\nOh no! There was the following error: " +
topics_response.getStatusMsg() + "\n")
else:
if (sentiment_response.getResponse() is None):
print("\nOh no! The request sent did not return a Json\n")
else:
print("\nOh no! There was the following error: " +
topics_response.getStatusMsg() + "\n")
except ValueError:
e = sys.exc_info()[0]
print("\nException: " + str(e))
def getSentimentAnalysis(text):
polarity = ''
# We are going to make a request to the Sentiment Analysis API
print("\tGetting sentiment analysis...")
sentiment_response = meaningcloud.SentimentResponse(meaningcloud.SentimentRequest(license_key, lang='en', txt=text, txtf='markup').sendReq())
if sentiment_response.isSuccessful():
polarity = sentiment_response.getGlobalScoreTag()
else:
print("\tOops! Request to sentiment was not succesful: (" + sentiment_response.getStatusCode() + ') ' + sentiment_response.getStatusMsg())
return polarity
def request_tweet_sans_compte(nombreTweet):
auth = tweepy.OAuthHandler(
'74kR1Uv1LGw3qllwqyIrIW2WH',
'GyVxV3O4Wks9akUFxRQAmXh1nVvtTnvvLi1nsMdkLt4EjqxIrO')
auth.set_access_token('1257599001433706496-HKChl1uTs0M7aHo2B80dk4Yy9mwyh4',
'VE2qcM0lGIY4ux1HGOaz4y2jfuIVZ0qMCMrSbPJkdadRM')
api = tweepy.API(auth)
page_list = []
tweets = []
n = 0
if (nombreTweet > 200):
for page in tweepy.Cursor(api.home_timeline,
count=200,
tweet_mode="extended").pages(
int(nombreTweet / 200)):
page_list.append(page)
n = n + 1
print(n)
for page in page_list:
for tweet in page:
tweets.append(tweet)
else:
tweets = api.home_timeline(count=nombreTweet, tweet_mode="extended")
liste_tweets = []
liste_ids = []
for tweet in tweets:
liste_tweets.append(tweet.full_text)
liste_ids.append(tweet.id)
model = 'IAB_en'
license_key = 'af76f805c5f86590fee318303f3d3ac3'
result = []
i = -1
for tweet in liste_tweets:
i = i + 1
if len(tweet.split(' ')) >= 10:
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(license_key,
lang='en',
txt=tweet,
txtf='plain').sendReq())
score = sentiment_response.getResponse().get('score_tag')
if (str(score) == "P"):
result.append([tweets[i], tweet, 1])
if (str(score) == "P+"):
result.append([tweets[i], tweet, 2])
return result
def request_tweet():
auth = tweepy.OAuthHandler(
'74kR1Uv1LGw3qllwqyIrIW2WH',
'GyVxV3O4Wks9akUFxRQAmXh1nVvtTnvvLi1nsMdkLt4EjqxIrO')
auth.set_access_token('999402738290233344-GB6h7ahayXolXu6MYTMVlc4VKPAbOPN',
'c0wB3SwFjiTpTWnR6fyudBX9Sel7SxUwxKeJ4FLJpaHcV')
api = tweepy.API(auth)
tweets = api.home_timeline(tweet_mode="extended",
include_rts=False,
count=50)
liste_tweets = []
liste_id = []
for tweet in tweets:
liste_tweets.append(tweet.full_text.split('http')[0])
liste_id.append(tweet.id)
model = 'IAB_en'
license_key = 'af76f805c5f86590fee318303f3d3ac3'
i = 0
liste_sentiment = []
for tweet in liste_tweets:
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(license_key,
lang='en',
txt=tweet,
txtf='plain').sendReq())
liste_sentiment.append([
str(tweet), liste_id[i],
sentiment_response.getResponse().get('score_tag')
])
i = i + 1
df = pandas.DataFrame(liste_sentiment, columns=["tweet", "id", "polarity"])
for i in range(len(df)):
print(df.loc[[i], ["tweet"]].values)
print(df.loc[[i], ["id"]].values)
print(df.loc[[i], ["polarity"]])
id_list = []
for i in range(len(df)):
if (str(df.iloc[i]["polarity"]) == "P"
or str(df.iloc[i]["polarity"]) == "P+"):
id_list.append(id_list(df.iloc[i]["id"]))
return id_list
def analyzeText(text, lang):
global index_count
print("Analyzing sentiment for text #%s" % str(index_count))
# this is where we are going to store our results
polarity = ""
subjectivity = ""
irony = ""
agreement = ""
confidence = ""
try:
# We are going to make a request to the Sentiment Analysis API
request = meaningcloud.SentimentRequest(license_key,
lang=lang,
txt=text,
server=server)
setRequestSource(request)
response = meaningcloud.SentimentResponse(request.sendReq())
if response.isSuccessful():
polarity = response.scoreTagToString(response.getGlobalScoreTag())
subjectivity = response.getSubjectivity()
irony = response.getIrony()
agreement = response.getGlobalAgreement()
confidence = response.getGlobalConfidence()
else:
if isBlockingErrorType(response.getStatusCode()):
raise ValueError(
"Something went wrong in the MeaningCloud request!: (" +
response.getStatusCode() + ") " + response.getStatusMsg())
else:
print("Oops! The request to Sentiment Analysis for text #" +
str(index_count) + " was not succesful: (" +
response.getStatusCode() + ") " +
response.getStatusMsg())
polarity = ("ERROR (" + response.getStatusCode() + "): " +
response.getStatusMsg())
except ValueError as e:
raise ValueError(str(e))
index_count += 1
return pd.Series([polarity, subjectivity, irony, agreement, confidence])
def meaning_sentiment(tupleFiles):
rawtweets = tupleFiles[0]
license_key = {YOUR_MEANINGCLOUD_LICENSE_KEY}
filenameCSV = 'tweets.csv'
filepathCSV = '/'+filenameCSV
with open(filenameCSV, 'w') as csv_file:
fieldnames = ['sentiment', 'tweet']
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writeheader()
with open(rawtweets, 'r') as file_tweets:
for line in file_tweets.readlines():
sentiment_response = meaningcloud.SentimentResponse(meaningcloud.SentimentRequest(license_key, lang='es', txt=line, txtf='plain').sendReq())
s = sentiment_response.getGlobalScoreTag()
writer.writerow({'sentiment': s , 'tweet': line})
tupleFilesCSV = (filenameCSV,filepathCSV)
return tupleFilesCSV
def get_sentiment_analysis(self):
logger.debug('TextAnalysis.get_sentiment_analysis()')
if self.sentiment_analysis is None:
logger.debug('sentiment_analysis not set')
logger.debug('setting sentiment_analysis')
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(
self.license_key,
lang=self.language,
txt=self.text,
txtf='plain',
).sendReq())
# Save the raw and serialized sentiment analysis
self.sentiment_analysis_raw = sentiment_response.getResults()
print(self.sentiment_analysis_raw)
self.sentiment_analysis = MeaningCloudResponse(
self.sentiment_analysis_raw).absa()
logger.debug(self.sentiment_analysis)
return self.sentiment_analysis
def analisisMeaning(full_text):
analisis = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(license_key_MC,
lang='es',
txt=full_text,
txtf='plain').sendReq())
if analisis.getGlobalScoreTag() == 'N+':
polaridad = "Negativo"
polarity = -1
elif analisis.getGlobalScoreTag() == 'N':
polaridad = "Negativo"
polarity = -0.5
elif analisis.getGlobalScoreTag() == 'P':
polaridad = "Positivo"
polarity = 0.5
elif analisis.getGlobalScoreTag() == 'P+':
polaridad = "Positivo"
polarity = 1
else:
polaridad = "Neutro"
polarity = 0
return (polaridad, polarity)
def extractFeatures(self):
# The language of the tweets
tweetLang = 'en'
# Tweet counter
counter = 0
# List of the IDs of tweets whose requests failed
wrongRequestTweetIds = []
# Open the data set file
with open(self.dataSetFilePath, 'rb') as f:
# Flag to know whether the first line of the file (column names)
# has been read
headerPassed = False
for line in f:
line = line.decode(errors='ignore')
row = line.split(',')
# i.e. if this is the first line of the file
if headerPassed == False:
headerPassed = True
continue
# Instantiate a tweet. Assume that the
# current CSV file which is being read
# already contains the following features
# per tweet:
#
# - text,
# - isBot
# - joy
# - surprise
# - fear
# - sadness
# - anger
#
t = Tweet(text=row[0],
isBot=int(row[1]),
joy=float(row[2]),
surprise=float(row[3]),
fear=float(row[4]),
sadness=float(row[5]),
anger=float(row[6]))
text = t.text
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(self.apiKey,
lang=tweetLang,
txt=text,
txtf='plain').sendReq())
if (sentiment_response.isSuccessful()):
# Populate the remaining fields of this Tweet instance
t.score_tag = sentiment_response.getGlobalScoreTag()
t.agreement = sentiment_response.getGlobalAgreement()
t.subjectivity = sentiment_response.getSubjectivity()
t.confidence = sentiment_response.getGlobalConfidence()
t.irony = sentiment_response.getIrony()
self.tweetList.append(t)
else:
# Let's see why the request failed
requestStatusCode = sentiment_response.getStatusCode()
requestStatusMsg = sentiment_response.getStatusMsg()
print("Tweet id = " + str(counter) +
"; request status code = " + str(requestStatusCode) +
"; request status msg = " + str(requestStatusMsg))
# Store this tweet ID for later. Another request will have to be made.
wrongRequestTweetIds.append(counter)
counter += 1
print("Tweets analyzed so far: " + str(counter))
# Wait for a few seconds before making a new request
time.sleep(self.requestIntervalSeconds)
print("\nTotal analyzed tweets = " + str(counter))
# If all requests succeeded
if len(wrongRequestTweetIds) == 0:
print("All " + str(counter) + " requests were successful.")
else:
print("The requests for the following tweets failed:\n")
print(wrongRequestTweetIds)
liste_tweets = []
liste_id = []
for tweet in tweets:
liste_tweets.append(tweet.full_text.split('http')[0])
liste_id.append(tweet.id)
model = 'IAB_en'
license_key = 'af76f805c5f86590fee318303f3d3ac3'
i = 0
liste_sentiment = []
for tweet in liste_tweets:
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(license_key,
lang='en',
txt=tweet,
txtf='plain').sendReq())
liste_sentiment.append([
str(tweet), liste_id[i],
sentiment_response.getResponse().get('score_tag')
])
i = i + 1
df = pandas.DataFrame(liste_sentiment, columns=["tweet", "id", "polarity"])
for i in range(len(df)):
print(df.loc[[i], ["tweet"]].values)
print(df.loc[[i], ["id"]].values)
print(df.loc[[i], ["polarity"]])
class SentimentResponseTest(unittest.TestCase):
outputOK = '{"status":{"code":"0","msg":"OK","credits":"1","remaining_credits":"5000"},"model":"general_en","score_tag":"P+","agreement":"AGREEMENT","subjectivity":"SUBJECTIVE","confidence":"98","irony":"NONIRONIC","sentence_list":[{"text":"Main dishes were quite good, but desserts were too sweet for me.","inip":"0","endp":"65","bop":"y","confidence":"98","score_tag":"P+","agreement":"AGREEMENT","segment_list":[{"text":"Main dishes were quite good","segment_type":"main","inip":"0","endp":"27","confidence":"98","score_tag":"P+","agreement":"AGREEMENT","polarity_term_list":[{"text":"(quite) good","inip":"24","endp":"27","confidence":"98","score_tag":"P+"}]},{"text":"desserts were too sweet for me","segment_type":"main","inip":"34","endp":"63","confidence":"100","score_tag":"P","agreement":"AGREEMENT","polarity_term_list":[{"text":"sweet@A","inip":"52","endp":"56","confidence":"100","score_tag":"P","sentimented_concept_list":[{"form":"dessert","id":"0e15bbd941","variant":"desserts","inip":"34","endp":"41","type":"Top>Product>Food","score_tag":"P"}]}]},{"text":"","segment_type":"secondary","inip":"65","endp":"65","confidence":"100","score_tag":"NONE","agreement":"AGREEMENT","polarity_term_list":[],"sentimented_concept_list":[{"form":"dessert","id":"0e15bbd941","variant":"desserts","inip":"34","endp":"41","type":"Top>Product>Food","score_tag":"NONE"}]}],"sentimented_entity_list":[],"sentimented_concept_list":[{"form":"dessert","id":"0e15bbd941","type":"Top>Product>Food","score_tag":"P"}]}],"sentimented_entity_list":[],"sentimented_concept_list":[{"form":"dessert","id":"0e15bbd941","type":"Top>Product>Food","score_tag":"P"}]}'
response = meaningcloud.SentimentResponse(outputOK)
outputEmpty = '{"status": {"code": "0","msg": "OK","credits": "1","remaining_credits":"5000"}}'
empty_response = meaningcloud.SentimentResponse(outputEmpty)
def testConstruct(self):
self.assertIsNotNone(self.response.getResponse())
def testConstructWithWrongJson(self):
outputWrong = 'malformed json'
with self.assertRaises(json.JSONDecodeError):
meaningcloud.SentimentResponse(outputWrong)
def testConstructWithEmptyParam(self):
with self.assertRaises(Exception):
meaningcloud.SentimentResponse('')
def testConstructEmptyResult(self):
self.assertIsNotNone(self.empty_response.getResponse())
def testGetGlobalModel(self):
self.assertIsNotNone(self.response.getModel())
def testGetGlobalModelWithEmptyInput(self):
self.assertEqual(self.empty_response.getModel(), '')
def testGetGlobalScoreTag(self):
self.assertIsNotNone(self.response.getGlobalScoreTag())
def testGetGlobalScoreTagWithEmptyInput(self):
self.assertEqual(self.empty_response.getGlobalScoreTag(), '')
def testGetGlobalAgreement(self):
self.assertIsNotNone(self.response.getGlobalAgreement())
def testGetGlobalAgreementWithEmptyInput(self):
self.assertEqual(self.empty_response.getGlobalAgreement(), '')
def testGetAgreement(self):
sentence_list = self.response.getResponse()['sentence_list']
self.assertIsNotNone(self.response.getAgreement(sentence_list[0]))
def testGetAgreementWithEmptyInput(self):
with self.assertRaises(AttributeError):
self.empty_response.getAgreement([])
def testGetGlobalSubjectivity(self):
self.assertIsNotNone(self.response.getSubjectivity())
def testGetGlobalSubjectivityWithEmptyInput(self):
self.assertIsNotNone(self.empty_response.getSubjectivity())
def testGetGlobalConfidence(self):
self.assertIsNotNone(self.response.getGlobalConfidence())
def testGetGlobalConfidenceWithEmptyInput(self):
self.assertIsNotNone(self.empty_response.getGlobalConfidence())
def testGetConfidence(self):
sentence_list = self.response.getResponse()['sentence_list']
self.assertIsNotNone(self.response.getConfidence(sentence_list[0]))
def testGetConfidenceWithEmptyInput(self):
with self.assertRaises(AttributeError):
self.empty_response.getAgreement([])
def testGetGlobalIrony(self):
self.assertIsNotNone(self.response.getIrony())
def testGetGlobalIronyWithEmptyInput(self):
self.assertIsNotNone(self.empty_response.getIrony())
def testGetGlobalSentimentedEntities(self):
self.assertEqual(
isinstance(self.response.getGlobalSentimentedEntities(), list),
True)
def testGetGlobalSentimentedEntitiesWithEmptyInput(self):
self.assertEqual(self.empty_response.getGlobalSentimentedEntities(),
'')
def testGetSentimentedEntities(self):
sentence_list = self.response.getResponse()['sentence_list']
self.assertEqual(
isinstance(self.response.getSentimentedEntities(sentence_list[0]),
list), True)
def testGetSentimentedEntitiesWithEmptyInput(self):
self.assertEqual(self.empty_response.getSentimentedEntities([]), '')
def testGetGlobalSentimentedConcepts(self):
self.assertEqual(
isinstance(self.response.getGlobalSentimentedConcepts(), list),
True)
def testGetGlobalSentimentedConceptsWithEmptyInput(self):
self.assertEqual(self.empty_response.getGlobalSentimentedConcepts(),
'')
def testGetSentimentedConcepts(self):
sentence_list = self.response.getResponse()['sentence_list']
self.assertEqual(
isinstance(self.response.getSentimentedConcepts(sentence_list[0]),
list), True)
def testGetSentimentedConceptsWithEmptyInput(self):
self.assertEqual(self.empty_response.getSentimentedConcepts([]), '')
def testScoreTagToString(self):
d = {
'P+': 'strong positive',
'P': 'positive',
'NEU': 'neutral',
'N': 'negative',
'N+': 'strong negative',
'NONE': 'no sentiment'
}
for x in d.items():
self.assertEqual(x[1], self.response.scoreTagToString(x[0]))
def testConstructWithEmptyParam(self):
with self.assertRaises(Exception):
meaningcloud.SentimentResponse('')
def testConstructWithWrongJson(self):
outputWrong = 'malformed json'
with self.assertRaises(json.JSONDecodeError):
meaningcloud.SentimentResponse(outputWrong)
def analyzeText(text):
global index_count
print("Analyzing text " + str(index_count))
# this is where we are going to store our results
polarity = ''
entities = ''
concepts = ''
iab2 = ''
try:
# We are going to make a request to the Sentiment Analysis API
print("\tGetting sentiment analysis...")
sentiment_response = meaningcloud.SentimentResponse(
meaningcloud.SentimentRequest(license_key,
lang='en',
txt=text,
txtf='markup').sendReq())
if sentiment_response.isSuccessful():
polarity = sentiment_response.getGlobalScoreTag()
else:
print('Request to sentiment was not succesful: ' +
sentiment_response.getStatusMsg())
# We are going to make a request to the Topics Extraction API
print("\tGetting entities and concepts...")
topics_req = meaningcloud.TopicsRequest(license_key,
txt=text,
lang='en',
topicType='ec',
otherparams={'txtf': 'markup'})
topics_response = meaningcloud.TopicsResponse(topics_req.sendReq())
# If there are no errors in the request, we extract the entities and concepts
if topics_response.isSuccessful():
entities_list = topics_response.getEntities()
formatted_entities = []
if entities_list:
for entity in entities_list:
if int(
topics_response.getTopicRelevance(entity)
) >= 100: #we limit the entities to those with relevance higher than 100
formatted_entities.append(
topics_response.getTopicForm(entity) + ' (' +
topics_response.getTypeLastNode(
topics_response.getOntoType(entity)) + ')')
entities = ', '.join(formatted_entities)
concepts_list = topics_response.getConcepts()
formatted_concepts = []
if concepts_list:
for concept in concepts_list:
if int(
topics_response.getTopicRelevance(concept)
) >= 100: #we limit the entities to those with relevance higher than 100
formatted_concepts.append(
topics_response.getTopicForm(concept))
concepts = ', '.join(list(dict.fromkeys(formatted_concepts)))
else:
print('Request to topics was not succesful: ' +
topics_response.getStatusMsg())
# We are going to make a request to the Deep Categorization API
print("\tGetting IAB 2.0 classification...")
deepcat_response = meaningcloud.DeepCategorizationResponse(
meaningcloud.DeepCategorizationRequest(license_key,
model='IAB_2.0_en',
txt=text,
otherparams={
'txtf': 'markup'
}).sendReq())
if deepcat_response.isSuccessful():
categories = deepcat_response.getCategories()
iab2 = (', '.join(
deepcat_response.getCategoryCode(cat)
for cat in categories[:1])) if categories else ''
else:
print('Request to Deep Categorization was not succesful: ' +
deepcat_response.getStatusMsg())
except ValueError:
e = sys.exc_info()[0]
print("\nException: " + str(e))
index_count += 1
return pd.Series([polarity, entities, concepts, iab2])
def sentiment(self, inp):
return meaningcloud.SentimentResponse(meaningcloud.SentimentRequest(cred['sent-api2'], lang='en', txt=inp, txtf='plain').sendReq())
