class NLPEngine:
def __init__(self):
self.alchemyapi = AlchemyAPI()
self.alchemyapi.apikey = 'a6e464bee67aebc9f2197ecffd88d48187bf469e'
def sentiment(self, text):
response = self.alchemyapi.sentiment("text", text)
if response['status'] == 'OK':
if response['docSentiment']['type'] == 'neutral':
return 0
else:
return response['docSentiment']['score']
else:
print response
return 0
def place(self,text):
response = self.alchemyapi.entities('text',text)
result = None
if response['status'] == 'OK':
for entitie in response['entities']:
if entitie['type'] == 'City':
result = entitie['text']
break
else:
print response
return result
def getData(filename):
with open(
'C:\\Users\\gatesyp\\Documents\\GitHub\\twiddit\\tweetsRepo\\' +
filename + '.txt', 'r') as myfile:
data = myfile.read().replace('\n', ' ')
alchemyapi = AlchemyAPI()
response = alchemyapi.entities('text', data, {'sentiment': 1})
if response['status'] == 'OK':
#print(json.dumps(response, indent=4))
for entity in response['entities']:
try:
result = testGoog.searchGoog(entity['text'])
if float(entity['relevance']) > 0.4:
#print(result[0]['url'])
#print(result[1]['url'])
redditHandler.subscribe(result[0]['url'], mybiszti,
lolipop123)
redditHandler.subscribe(result[1]['url'], mybiszti,
lolipop123)
except:
print("none or code 403")
class NLPEngine:
def __init__(self):
self.alchemyapi = AlchemyAPI()
self.alchemyapi.apikey = 'a6e464bee67aebc9f2197ecffd88d48187bf469e'
def sentiment(self, text):
response = self.alchemyapi.sentiment("text", text)
if response['status'] == 'OK':
if response['docSentiment']['type'] == 'neutral':
return 0
else:
return response['docSentiment']['score']
else:
print response
return 0
def place(self, text):
response = self.alchemyapi.entities('text', text)
result = None
if response['status'] == 'OK':
for entitie in response['entities']:
if entitie['type'] == 'City':
result = entitie['text']
break
else:
print response
return result
def test_alchemy():
text_1 = "The decision by the independent MP Andrew Wilkie to withdraw his support for the minority Labor government sounded dramatic but it should not further threaten its stability. When, after the 2010 election, Wilkie, Rob Oakeshott, Tony Windsor and the Greens agreed to support Labor, they gave just two guarantees: confidence and supply"
text_2 = "inflation plan, initially hailed at home and abroad as the saviour of the economy, is limping towards its first anniversary amid soaring prices, widespread shortages and a foreign payments crisis. Announced last February 28 the plan prices, fixed the value of the new Cruzado currency and ended widespread indexation of the economy in a bid to halt the country's 250 pct inflation rate. But within a year the plan has all but collapsed. \"The situation now is worse than it was. Although there was inflation, at least the economy worked,\" a leading bank economist said. The crumbling of the plan has been accompanied by a dramatic reversal in the foreign trade account. In 1984 and 1985 Brazil's annual trade surpluses had been sufficient to cover the 12 billion dlrs needed to service its 109 billion dlr foreign debt. For the first nine months of 1986 all seemed to be on target for a repeat, with monthly surpluses averaging one billion dlrs. But as exports were diverted and imports increased to avoid further domestic shortages the trade surplus plunged to 211 mln dlrs in October and since then has averaged under 150 mln. Reuter "
alchemyapi = AlchemyAPI()
response = alchemyapi.entities('text', text_2, {'sentiment': 1})
pp.pprint(response)
print len(response['entities'])
def test_alchemy():
text_1 = "The decision by the independent MP Andrew Wilkie to withdraw his support for the minority Labor government sounded dramatic but it should not further threaten its stability. When, after the 2010 election, Wilkie, Rob Oakeshott, Tony Windsor and the Greens agreed to support Labor, they gave just two guarantees: confidence and supply"
text_2 = "inflation plan, initially hailed at home and abroad as the saviour of the economy, is limping towards its first anniversary amid soaring prices, widespread shortages and a foreign payments crisis. Announced last February 28 the plan prices, fixed the value of the new Cruzado currency and ended widespread indexation of the economy in a bid to halt the country's 250 pct inflation rate. But within a year the plan has all but collapsed. \"The situation now is worse than it was. Although there was inflation, at least the economy worked,\" a leading bank economist said. The crumbling of the plan has been accompanied by a dramatic reversal in the foreign trade account. In 1984 and 1985 Brazil's annual trade surpluses had been sufficient to cover the 12 billion dlrs needed to service its 109 billion dlr foreign debt. For the first nine months of 1986 all seemed to be on target for a repeat, with monthly surpluses averaging one billion dlrs. But as exports were diverted and imports increased to avoid further domestic shortages the trade surplus plunged to 211 mln dlrs in October and since then has averaged under 150 mln. Reuter "
alchemyapi = AlchemyAPI()
response = alchemyapi.entities('text', text_2, {'sentiment': 1})
pp.pprint(response)
print len(response['entities'])
def getAnnotation(text):
annotations = spotlight.annotate(
'http://spotlight.dbpedia.org/rest/annotate',
text,
confidence=0.25,
support=40)
annotationsSorted = sorted(annotations, key=lambda k: k['similarityScore'])
setSpotlight = set(map(lambda x: x['URI'], annotationsSorted))
"""
{ u'URI': u'http://dbpedia.org/resource/People',
u'offset': 321,
u'percentageOfSecondRank': -1.0,
u'similarityScore': 0.08647863566875458,
u'support': 426, #
u'surfaceForm': u'people',
u'types': u'DBpedia:TopicalConcept'}
"""
alchemyapi = AlchemyAPI()
response = alchemyapi.entities('text', text, {'sentiment': 1})
resFilt = filter(lambda x: 'disambiguated' in x, response['entities'])
key = ['dbpedia', 'geonames', 'yago', 'opencyc']
resFilt
entitySet = set()
for r in resFilt:
for k in key:
if k in r['disambiguated']:
entitySet.add(r['disambiguated'][k])
"""
{u'count': u'1',
u'disambiguated': {u'dbpedia': u'http://dbpedia.org/resource/Kathmandu',
u'freebase': u'http://rdf.freebase.com/ns/m.04cx5',
u'geo': u'27.716666666666665 85.36666666666666',
u'geonames': u'http://sws.geonames.org/1283240/',
u'name': u'Kathmandu',
u'subType': [u'TouristAttraction'],
u'website': u'http://www.kathmandu.gov.np/',
u'yago': u'http://yago-knowledge.org/resource/Kathmandu'},
u'relevance': u'0.33',
u'sentiment': {u'type': u'neutral'},
u'text': u'Kathmandu',
u'type': u'City'},
"""
entitySet.update(setSpotlight)
return entitySet, annotationsSorted, response
def getAnnotation(text):
annotations = spotlight.annotate('http://spotlight.dbpedia.org/rest/annotate',text,confidence=0.25, support=40)
annotationsSorted = sorted(annotations, key=lambda k: k['similarityScore'])
setSpotlight=set(map(lambda x:x['URI'],annotationsSorted))
"""
{ u'URI': u'http://dbpedia.org/resource/People',
u'offset': 321,
u'percentageOfSecondRank': -1.0,
u'similarityScore': 0.08647863566875458,
u'support': 426,
u'surfaceForm': u'people',
u'types': u'DBpedia:TopicalConcept'}
"""
alchemyapi = AlchemyAPI()
response = alchemyapi.entities('text', text, {'sentiment': 1})
resFilt=filter(lambda x: 'disambiguated' in x, response['entities'])
key=['dbpedia','geonames','yago','opencyc']
resFilt
entitySet=set()
for r in resFilt:
for k in key:
if k in r['disambiguated']:
entitySet.add(r['disambiguated'][k])
"""
{u'count': u'1',
u'disambiguated': {u'dbpedia': u'http://dbpedia.org/resource/Kathmandu',
u'freebase': u'http://rdf.freebase.com/ns/m.04cx5',
u'geo': u'27.716666666666665 85.36666666666666',
u'geonames': u'http://sws.geonames.org/1283240/',
u'name': u'Kathmandu',
u'subType': [u'TouristAttraction'],
u'website': u'http://www.kathmandu.gov.np/',
u'yago': u'http://yago-knowledge.org/resource/Kathmandu'},
u'relevance': u'0.33',
u'sentiment': {u'type': u'neutral'},
u'text': u'Kathmandu',
u'type': u'City'},
"""
entitySet.update(setSpotlight)
return entitySet,annotationsSorted,response
class ProductReviewAnalyzer:
'''
wrapper class which implements sentiment analysis using custom analyzer or by using APIs.
presently alchemyapi is being used.
'''
def __init__(self):
self.alchemyapi = AlchemyAPI()
def AnalyzeOverallSentiment(self , product_review):
if (product_review.userReview and len(product_review.userReview) > 0 ):
response = self.alchemyapi.sentiment("text" , product_review.userReview)
if response['status'] == 'OK':
return response
else:
print('Error in sentiment analysis call: ', response['statusInfo'])
return None
else :
print "error : product review is not set"
return None
def KeywordExtractor(self,product_review):
if (product_review.userReview and len(product_review.userReview) > 0) :
response = self.alchemyapi.keywords("text" , product_review.userReview , {'sentiment' : 1})
if response['status'] == 'OK':
return response['keywords']
else :
print 'Error in keyword extractor call: ' , response['statusInfo']
else :
print "error product review is not set"
return None
def EntityExtractor(self,product_review):
if (product_review.userReview and len(product_review.userReview) > 0) :
response = self.alchemyapi.entities("text" , product_review.userReview , {'sentiment' : 1})
if response['status'] == 'OK':
return response['entities']
else :
print 'Error in Entity extractor call: ' , response['statusInfo']
else :
print "error product review is not set"
return None
import time
import sys
import requests
import unirest
import os
import json
import simplejson
import subprocess
import shlex
import numpy as np
#import alchemyapi_python.__future__ import print_function
from alchemyapi_python.alchemyapi import AlchemyAPI
alchemyapi = AlchemyAPI()
demo_text = 'Yesterday dumb Bob destroyed my fancy iPhone in beautiful Denver, Colorado. I guess I will have to head over to the Apple Store and buy a new one.'
response = alchemyapi.entities('text', demo_text, {'sentiment': 1})
key = 'd7d457mawe4jc68nja63sjcn'
search_url1 = 'http://api.rottentomatoes.com/api/public/v1.0/movies.json?apikey='+key+'&q='
search_url2 = '&page_limit=1&page='
rev_url1='http://api.rottentomatoes.com/api/public/v1.0/movies/'
rev_url2='/reviews.json?apikey='+key+'&page_limit=50'
mpaa_conversion = {"G":0, "PG":1, "PG-13":2,"R":3,"NC-17":4}
ratings_conversion = {"A":1.0,"B":0.7,"C":0.4,"D":0.2,"E":0}
def rt_features(search_str):
search_url = search_url1+search_str+search_url2
response = urllib.urlopen(search_url+str(1))
def alc_sent_system(search_results, text_to_info, topic):
# Lists of example texts
pos_examples = []
neg_examples = []
alc = AlchemyAPI()
alc_neg_count = 0
alc_pos_count = 0
alc_sent_score = 0.0
# Grab the entity of the topic
# There should only be one with the unigram/bigram provided
try:
ent_topic = alc.entities("text", topic, {'sentiment': 0, 'language': 'english'})["entities"][0]["text"].lower()
print "valid topic entity", ent_topic
except:
# topic has no entity so just grab basic doc text sentiment
ent_topic = None
# Loop through search_results to classify each tweet in results
for tweet in search_results:
#Look into entity recognition and actually extracting sentiment towards the entity searched for and not just the overall tweet
the_sent = "neutral"
ent_response = alc.entities("text", tweet, {'sentiment': 1, 'language': 'english'})
if (ent_topic and ("entities" in ent_response)):
# Entity recognition based sentiment analysis
for ent in ent_response["entities"]:
#print ent["text"].lower()
#check if the entity in the tweet is the same/is within the topic entity or vice versa
if ((ent["text"].lower() in ent_topic) or (ent_topic in ent["text"].lower())):
#print "entity in tweet was part of topic entity", ent_topic
# Get Sentiment
if ent["sentiment"]["type"] == "negative":
alc_neg_count += 1
the_sent = "negative"
elif ent["sentiment"]["type"] == "positive":
alc_pos_count += 1
the_sent = "postive"
# Get Sentiment score
if "score" in ent["sentiment"]:
alc_sent_score += float(ent["sentiment"]["score"])
else: #no entities found in the tweet just clasify the entire tweet
#use alchemyAPI to determine if positive or negative
sent_response = alc.sentiment("text", tweet, {'language': 'english'})
#calculate if there are more positive or more negative examples
if sent_response["docSentiment"]["type"] == "negative":
alc_neg_count += 1
the_sent = "negative"
elif sent_response["docSentiment"]["type"] == "positive":
alc_pos_count += 1
the_sent = "positive"
#add sent score value to system to determine overall score
#when neutral no score is present
if "score" in sent_response["docSentiment"]:
alc_sent_score += float(sent_response["docSentiment"]["score"])
#Gather tweet examples
if the_sent == "positive":
if len(pos_examples) < 10:
print "adding positive example"
pos_examples.append(tweet)
elif the_sent == "negative":
if len(neg_examples) < 10:
neg_examples.append(tweet)
# Creates a list for of json objects of tweets of negative and positive examples
for i in range(len(pos_examples)):
pos_examples[i] = text_to_info[pos_examples[i]]
for i in range(len(neg_examples)):
neg_examples[i] = text_to_info[neg_examples[i]]
# print pos
#print pos_examples
# print neg
print neg_examples
if (alc_pos_count + alc_neg_count > 0):
pos_percent = (alc_pos_count/float(alc_pos_count+alc_neg_count)) * 100
neg_percent = (alc_neg_count/float(alc_pos_count+alc_neg_count)) * 100
else:
print "Something went wrong/no tweets were found"
pos_percent = 0
neg_percent = 0
print pos_percent
print neg_percent
if alc_neg_count > alc_pos_count:
alc_sent_from_examples = "negative"
elif alc_neg_count < alc_pos_count:
alc_sent_from_examples = "positive"
else:
alc_sent_from_examples = "neutral"
print "Sentiment from examples: ", alc_sent_from_examples
if alc_sent_score < 0:
alc_sent_from_score = "negative"
elif alc_sent_score > 0:
alc_sent_from_score = "positive"
else:
alc_sent_from_score = "neutral"
print "Sentiment from score: ", alc_sent_from_score
if ent_topic:
return (alc_pos_count, alc_neg_count, pos_examples, neg_examples, pos_percent, neg_percent, ent_topic)
else:
return (alc_pos_count, alc_neg_count, pos_examples, neg_examples, pos_percent, neg_percent, topic)
