def SelectState_rel_only(policy_mode,
str_rule,
relavance,
user_input,
pre_history,
TreeState,
force_strategy=None):
branch_idx = TreeState.keys()[0]
branch = TreeState[branch_idx]['node']
if not force_strategy == None:
bool_idx, int_idx = force_strategy
return TreeState[branch_idx][bool_idx][int_idx]
if relavance >= branch['threshold_relavance']:
#print 'we are in the h'
return TreeState[branch_idx][True][
0] # only use the continue, don't expand
else:
if policy_mode == 0 or pre_history == None:
return random.choice(
TreeState[branch_idx][False]
[0:-1]) # don't choose the last leave, go back
else:
#choose this based on the previous utterances' sentiment.
curr_1 = sentiment.get_sentiment(user_input)
curr_2 = sentiment.get_sentiment(pre_history[-1])
curr_3 = sentiment.get_sentiment(pre_history[-2])
print 'this is the previous history'
print curr_1
print curr_2
print curr_3
strategy = str_rule[(curr_1, curr_2, curr_3)]
return {'name': strategy}
def sentimentize(self, compare=True):
from sentiment import get_sentiment
if compare:
self.sentiment = dict()
for mode in ['nltk', 'vader', 'textblob', 'api']:
self.sentiment[mode] = get_sentiment(self.text, mode)
else:
self.sentiment = {
'textblob': get_sentiment(self.text, mode='textblob')
}
def get_social_score():
posts = graph.get_all_connections(
id='me',
connection_name='posts',
fields='reactions,likes,comments,created_time,message')
num_likes = 0
num_posts = 0
num_comments = 0
results = []
for post in posts:
num_comments = 0
num_likes = 0
content = ''
if 'comments' in post:
num_comments = len(post['comments']['data'])
if 'likes' in post:
num_likes += len(post['likes']['data'])
sentiment = 0.5
if 'message' in post:
content = post['message']
sentiment = get_sentiment(content)
time = post['created_time']
result = (time, sentiment, num_comments + num_likes)
results.append(result)
print(format_result(result))
return results
async def on_ready(self):
print("Generating report for guild %d..." % self.guild_id)
guild = self.get_guild_guaranteed(self.guild_id)
today = datetime.utcnow().date()
day_start = datetime(today.year, today.month, today.day)
message_infos: "List[MessageInfo]" = []
for _, channel in enumerate(guild.text_channels):
messages = channel.history(limit=None, after=day_start)
async for unk_message in messages:
message = cast(discord.Message, unk_message)
author = cast(discord.User, message.author)
if message.content == "" or author.bot:
continue
has_prefix = False
for prefix in self.command_prefixes:
if message.content.startswith(prefix):
has_prefix = True
if has_prefix:
continue
sentiment = get_sentiment(message.content)
message_infos.append(
MessageInfo(message.id, message.created_at,
message.content, sentiment))
df = pd.DataFrame(
[message_info.as_dict() for message_info in message_infos])
df.to_csv(self.output_path, index=False, index_label="message_id")
await self.logout()
await self.close()
def on_data(self, data):
print("data")
tweet = json.loads(data)
tweetid = tweet.get('id', None)
place = tweet.get('place', None)
user = tweet['user']['name'] if tweet.get('user',
None) is not None else None
text = tweet.get('text', None)
lang = None
try:
lang = detect(text)
except LangDetectException:
pass
if lang in supported_langs.keys():
s = get_sentiment(text, supported_langs[lang])
tweet = Tweet(tweetid, place, user, text, s)
print(tweet)
for connection in WSHandler.connections:
data = json.dumps(tweet.__dict__).encode('utf-8')
connection.write_message(data)
else:
print("detected lang '{}' is not supported".format(lang))
return True
def sentiment_analysis():
data = request.get_json()
input_text = data['inputText']
input_text2 = data['inputText2']
input_text3 = data['inputText3']
response = sentiment.get_sentiment(input_text, input_text2, input_text3)
return jsonify(response)
def sentiment_analysis():
data = request.get_json()
if 'language' not in data:
input_lang = 'en-US'
else:
input_lang = data['language']
if 'text' not in data:
print("No text provided to sentiment")
abort(400)
input_text = data['text']
response = sentiment.get_sentiment(input_text, input_lang,
SUBSCRIPTION_KEY, API_URL)
evalnum = response['documents'][0]['score']
print(str(evalnum))
if 0.2 >= evalnum >= 0:
answer = "VERY NEGATIVE"
elif 0.4 >= evalnum > 0.2:
answer = "NEGATIVE"
elif 0.8 > evalnum >= 0.6:
answer = "POSITIVE"
elif 1 >= evalnum >= 0.8:
answer = "VERY POSITIVE"
else:
answer = "NEUTRAL"
return answer
def text_badcases(self):
t = [
"We hate the boring new project with @V_and_A find out more about #ReelToReal http://bit.ly/1xR3bKc",
]
for txt in t:
print txt
self.assertTrue(get_sentiment(txt)<0.3)
def sentiment_analysis():
data = request.get_json()
input_text = data['inputText']
input_lang = data['inputLanguage']
output_text = data['outputText']
output_lang = data['outputLanguage']
response = sentiment.get_sentiment(input_text, input_lang, output_text, output_lang)
return jsonify(response)
def text_ambivalentcases(self):
t = [
'more info on exciting exhibition about visual representation of protest that we are involved in @V_and_A here: http://www.vam.ac.uk/content/exhibitions/disobedient-objects/',
'Really looking forward to Disobedient Objects - show of activist folk art - opening next week at @V_and_A pic.twitter.com/l2EjaqBVYt',
'@V_and_A @sciencemuseum Congrats for making it into our 101 things to do in London for teenagers list #KidsLondon http://ow.ly/z82lQ',
]
for txt in t:
print txt
self.assertTrue(-0.3< get_sentiment(txt) < 0.3)
def test_positivecases(self):
t = [
'Designers, @barberosgerby have collaborated with @BMWGroup to create a memorable experience in the @V_and_A #LDF14 http://bit.ly/W2IDRu',
"We've launched an exciting new learning project with @V_and_A find out more about #ReelToReal http://bit.ly/1xR3bKc",
"@V_and_A Spent half day at The V&A with my daughter Eva. She loved it, surrounded by beauty and history. Thanks! pic.twitter.com/sCAU9TRJNq"
]
for txt in t:
print txt
self.assertTrue(get_sentiment(txt)>0.3)
def main():
client = WebClient(os.getenv("SLACK_TOKEN"))
channel_name = os.getenv(os.getenv("CHANNEL_NAME"))
channel_id = fetch_conversation(client, channel_name)
conversation_history = fetch_messages(client, channel_id)
sentiment_analysis = 0
for message in conversation_history:
sentiment_analysis += sentiment.get_sentiment(message["text"])
sentiment_analysis = sentiment_analysis / len(conversation_history)
send_message(client, channel_id, sentiment_analysis)
def sentiment():
city = get_arg('city', 'melbourne').lower()
years = list(
map(int,
get_arg('years', '2014,2015,2016,2017,2018').split(',')))
months = list(
map(int,
get_arg('months', '1,2,3,4,5,6,7,8,9,10,11,12').split(',')))
weekdays = list(map(int, get_arg('weekdays', '0,1,2,3,4,5,6').split(',')))
return as_json(get_sentiment(city, years, months, weekdays))
def sentiment():
a = dict()
records = Watchlist.query.filter_by(panid=session["panid"]).all()
news_list = []
for x in records:
news_list.append(companynews(x.stockname))
flat = [j for sub in news_list for j in sub]
sentiment_list = get_sentiment(flat)
for i, x in enumerate(records):
a[f"{x.stockname}"] = sentiment_list[i]
return render_template("sentiment.html", records=records, a=a)
def post(self, id):
args = self.reqparse.parse_args()
text = args["text"]
# getting sentiment analysis from google nlp api
annotations = get_sentiment(text)
sentiment = annotations.document_sentiment.score
# getting emotion from deepaffects text api
emotion = list(json.loads(get_emotion(text).text)["response"].keys())[0]
ketchup = CheckIn(id, text, sentiment, emotion)
self.add_checkin_to_db(ketchup)
most_common, average, slope, r2 = self.get_data(id)
return jsonify({"emotion": emotion, "sentiment": sentiment, "most_freq_emotion": most_common, "average_sentiment": average, "slope": slope, "r2": r2})
def post(self):
tw=self.get_argument('twitter')
frm=self.get_argument('from')
to=self.argument('to')
msg=self.argument('message')
getVars(msg,to,frm)
if msg=="":
pol1=get_sentiment(tw_nm)
if twitter=="":
MSG=filtering(to)
pol2=get_sentiment(MSG)
if pol1==0:
pol=pol2
elif pol2==0:
pol==pol1
else:
pol=(pol1+pol2)/2
if pol>0:
emo="happy"
elif pol<0:
emo="sad"
elif pol==0:
emo="neutral"
get_info(nm,emo)
def SelectState_rel_only(policy_mode,str_rule, relavance, user_input, pre_history, TreeState, force_strategy=None):
branch_idx = TreeState.keys()[0]
branch = TreeState[branch_idx]['node']
if not force_strategy == None:
bool_idx, int_idx = force_strategy
return TreeState[branch_idx][bool_idx][int_idx]
if relavance >= branch['threshold_relavance']:
#print 'we are in the h'
return TreeState[branch_idx][True][0] # only use the continue, don't expand
else:
if policy_mode ==0 or pre_history==None:
return random.choice(TreeState[branch_idx][False][0:-1])# don't choose the last leave, go back
else:
#choose this based on the previous utterances' sentiment.
curr_1 = sentiment.get_sentiment(user_input)
curr_2 = sentiment.get_sentiment(pre_history[-1])
curr_3 = sentiment.get_sentiment(pre_history[-2])
print 'this is the previous history'
print curr_1
print curr_2
print curr_3
strategy = str_rule[(curr_1,curr_2,curr_3)]
return {'name':strategy}
def store_post_data(classifier, status_data, topic, game_related_data=False):
post_list = []
for i in status_data:
if unidecode(i['lang']):
post_text = unidecode(i['text'])
if post_text not in post_list:
post_sentiment = get_sentiment(classifier, post_text)
post_item = {"_id": int(i['id']),
"gamer": game_related_data,
"topic": topic,
"post_text": post_text,
"post_sentiment": post_sentiment}
add_to_db(post_item, "posts")
post_list.append(post_text)
def trade(ticker):
max_shares = 0
avg_price = 0
if request.method == 'GET':
tick = ticker
sentiment_news = str(get_sentiment(ticker))
name, price, market, isOpen = get_company_data(ticker)
image = plot_historical(ticker)
CurrCash = Money.query.filter_by(id=1).first().CurrCash
if(Portfolio.query.filter_by(ticker=ticker).count() != 0):
max_shares = Portfolio.query.filter_by(ticker=ticker)[0].shares
avg_price = Portfolio.query.filter_by(ticker=ticker)[0].avg_value
return render_template('trade.html', sentiment=sentiment_news, name=name, price=price,
market=market, isOpen = isOpen, ticker=tick, avg_price=avg_price, image=image, CurrCash=CurrCash, maxshares=int(max_shares))
else:
if request.form['tick']:
tick = request.form['tick']
return redirect('/trade/'+tick)
def twitter_login():
# If the user is not authorized, redirect to the twitter login page
if not twitter.authorized:
return redirect(url_for('twitter.login'))
# If user is authorized retrieve his/her account details
account_info = twitter.get('account/settings.json')
# If user is authorized retrieve his/her tweets
user_tweets = twitter.get("statuses/user_timeline.json")
# If account information is successfully retrieved, proceed to analyse and display it
if account_info.ok:
# Convert retrieved info to json format
user_tweets_json = user_tweets.json()
account_info_json = account_info.json()
# Get tweet text from the objects returned
all_tweets = []
print(account_info_json)
for tweet in user_tweets_json:
all_tweets.append(tweet['text'])
# Text Cleaning for tweets
all_tweets_cleaned = text_cleaning.clean_tweets(all_tweets)
# BTM model for topic modeling results
classified_tweets, topics = btm_model.categorize(all_tweets_cleaned)
# Sentiment analysis
tweet_sentiment = sentiment.get_sentiment(all_tweets_cleaned)
# Prepare data to be sent and rendered on the template for user dashboard
data = {
"all_tweets": all_tweets,
"account_info_json": account_info_json,
"classified_tweets": classified_tweets,
"topics": topics,
"sentiment": tweet_sentiment
}
# Render template with user data
return render_template('user_dash.html', data=data)
# If account info is not retrieved successfully return an error message.
return '<h2>Error</h2>'
def infer_sentiment(self, s, pos=False):
'''
Get sentiment scores as a dict only for select POS, and 0 score for rest
'''
#pos_included = ['ADJ', 'NOUN', 'PROPN', 'INTJ']
pos_included = ['ADJ', 'NOUN', 'PROPN', 'VERB', 'ADV', 'INTJ']
s = self.nlp(s)
sentiments = get_sentiment(s)
if pos:
for t in s:
if t.text.lower() in sentiments and t.pos_ not in pos_included:
sentiments[ t.text ] = None
if t.lemma_.lower() in sentiments and t.pos_ not in pos_included:
sentiments[ t.lemma_ ] = None
sentiments = {k: v for k,v in sentiments.items() if v}
res = []
for t in s:
res.append( sentiments.get( t.text.lower(), 0.0 ) )
return self.convert_sentiment( res )
def get_business_score(reviews, id):
sentences = reviews_to_sentences(reviews)
final_cat = {"food":0, "service":0,"ambiance":0, "money":0}
cat_count = {"food":1, "service":1, "ambiance":1, "money":1}
items_food = {}
items_service = {}
items_amb = {}
items_money = {}
for sent in sentences:
sentr = senti.get_sentiment(sent)
if sentr != 0.5:
cat = sim.get_similarity(sent)
final_cat[cat] = sentr + final_cat[cat]
cat_count[cat] = 1 + cat_count[cat]
if cat == 'food':
items_food[sent] = sentr
elif cat == 'service':
items_service[sent] = sentr
elif cat == 'ambiance':
items_amb[sent] = sentr
elif cat == 'money':
items_money[sent] = sentr
for key in final_cat.keys():
final_cat[key] = final_cat[key] / (1.0 * cat_count[key])
total_sen = 5
new_f = dict(sorted(items_food.iteritems(), key=operator.itemgetter(1), reverse=True)[:total_sen])
new_s = dict(sorted(items_service.iteritems(), key=operator.itemgetter(1), reverse=True)[:total_sen])
new_a = dict(sorted(items_amb.iteritems(), key=operator.itemgetter(1), reverse=True)[:total_sen])
new_m = dict(sorted(items_money.iteritems(), key=operator.itemgetter(1), reverse=True)[:total_sen])
f = " ".join(new_f.keys())
s = " ".join(new_s.keys())
a = " ".join(new_a.keys())
m = " ".join(new_m.keys())
write_image(f, s, a, m, id)
return final_cat
from sentiment import get_sentiment
tweets = pd.read_csv('mytweets.csv')
states = pd.read_pickle('base_us_table.pd').code.values
locs = [i for i in tweets['loc'].values]
sents = [get_sentiment(i) for i in tweets.text.values]
result = []
for l, sent in zip(locs, sents):
l = str(l)
for s in states:
s = str(s)
if s in l:
result.append((s, float(sent)))
continue
data_ = pd.DataFrame(result, columns=['state', 'score'])
data_means = data_.groupby('state').score.mean()
help='Analyse review tiped by user',
action='store_true')
parser.add_argument('-f',
'--file',
help='File with reviews',
default='amazon_reviews.txt')
args = parser.parse_args()
user_input = args.userinput
fileName = args.file
reviews = []
r = Reviews_manager(fileName)
if user_input == True:
user_review = input('Write your reviews or let me select some for you: ')
reviews.append(user_review)
else:
reviews = r.get_reviews()
scoring = get_sentiment(reviews)
if user_input == True:
print('Your score is {} Do you agree with that?'.format(
scoring[0]['score']))
else:
compare = r.compare_reviews(scoring)
for e in compare:
print('{} \nThe score for this review is {}, the user {} with that\n'.
format(e['review'], e['score'], e['compare']))
if __name__ == "__main__":
filepath = 'video_ids.txt'
output_file = 'output_analysis.txt'
DATA_PATH = 'InputComments'
FILE = "Output/Output.%s.json"
comment_files = [join(DATA_PATH, f) for f in listdir(DATA_PATH) if (isfile(join(DATA_PATH, f)) and "json" in f)]
#print(comment_files)
for f in comment_files:
#print(f)
with open(f, 'r') as outfile:
data = json.load(outfile)
data = pd.DataFrame(data)
dictionary_comments = sentiment.get_sentiment(data['text'],100)
print("-----dictionary----", dictionary_comments)
video_id = f[26:37]
with open(FILE % video_id, "w") as f:
f.write(json.dumps(dictionary_comments))
#data['author_replied'] = data['author_replied'].astype(int)
# with open(filepath,'r') as fp:
# youtube_url = fp.readline()
# cnt = 1
# while youtube_url != '':
# print(cnt,"------url--------", youtube_url,"\n")
# video_id = extract_video_id(youtube_url)
def get_portfolio_info(api, stock_ticker):
"""
This function gets various information about stocks held in your portfolio.
Parameters
----------
api: Alpaca api object
stock_ticker: Stock ticker string (ex: NRZ)
Returns
-------
None
"""
# Checking moving averages
ma_200 = support.get_ma(api, stock_ticker=stock_ticker, days="200")
print(f"200-day moving average of {stock_ticker} is", ma_200)
ma_50 = support.get_ma(api, stock_ticker=stock_ticker, days="50")
print(f"50-day moving average of {stock_ticker} is", ma_50)
# Checking last trading price
price = support.get_price(api, stock_ticker=stock_ticker).price
account = api.get_account()
# Checking Sentiment
stock_sentiment = sentiment.get_sentiment(stock_ticker=stock_ticker)
if stock_sentiment <= -0.5:
print(
f"News sentiment of {stock_ticker} is negative ({stock_sentiment})."
)
elif stock_sentiment > -0.5 and stock_sentiment < 0.5:
print(
f"News sentiment of {stock_ticker} is neutral ({stock_sentiment})."
)
elif stock_sentiment >= 0.5:
print(
f"News sentiment of {stock_ticker} is positive ({stock_sentiment})."
)
# Checking stock position
currently_own_this_stock = support.currently_own_this_stock(
api, stock_ticker=stock_ticker)
if currently_own_this_stock == 0:
print(f"{stock_ticker} stock is currently not held.")
else:
print(f"{stock_ticker} stock is currently held.")
# Checking pending orders
pending_buy = support.check_for_pending(api,
stock_ticker=stock_ticker,
trade_type="buy",
day_range=5,
zone='UTC',
result_limit=200)
pending_sell = support.check_for_pending(api,
stock_ticker=stock_ticker,
trade_type="sell",
day_range=5,
zone='UTC',
result_limit=200)
def analyze_sentiment(textdata):
print('<<<<<<<In analyze_sentiment function>>>>>>')
response = get_sentiment(textdata)
return response
import galbackend_online
import sentiment
galbackend_online.InitLogging()
galbackend_online.InitResource('v4')
while True:
user_id = 'a'
history = {user_id : ['aa','bb']}
theme = {user_id: 'movies'}
theme, strategy,utt,previous_history, word2vec = galbackend_online.get_response('joke',1,'joke_joke',user_id,history,theme)
print utt
sent = sentiment.get_sentiment(utt)
print sent
if sent not in ['pos','neg','neutral']:
break
import sentiment
import pickle
import readall
rating_logs = readall.readall("/home/ubuntu/zhou/Backend/rating_log/")
user_input = readall.get_log(rating_logs)
'''
with open('user_input_v2.pkl') as ff:
user_input = pickle.load(ff)
'''
f = open ('sentiment_log.txt','w')
sentiment_label = []
for turn in user_input:
question = turn['question']
label = sentiment.get_sentiment(question)
sentiment_label.append(label)
f.write(question+'\n')
f.write('sentiment: '+ label + '\n')
pos_number = sentiment_label.count('pos')
print pos_number
neg_number = sentiment_label.count('neg')
print neg_number
neutral_number = sentiment_label.count('neutral')
print neutral_number
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
try:
user_timeline = twitter.get_user_timeline(screen_name=username, count=200)
except TwythonError as e:
print e
date_str = str((date.today() - timedelta(days=0)).strftime('%a %b %d'))
date_key = str((date.today() - timedelta(days=0)).strftime('%y%m%d'))
tweet_count = 0
total_sentiment_score = 0
tweets = ""
for i in range(0, len(user_timeline)):
if date_str in user_timeline[i]['created_at']:
tweet_count += 1
sentiment_score = sentiment.get_sentiment(user_timeline[i]['text'])
total_sentiment_score += sentiment_score
tweets += user_timeline[i][
'text'] + ' - ' + str(sentiment_score) + '\n'
else:
break
if tweet_count > 0:
avg_sentiment_score = total_sentiment_score / tweet_count
print "Today's tweet count for " + username + ": " + str(tweet_count)
print('\nAvg Sentiment Score : %.2f' % avg_sentiment_score)
else:
print 'No tweets for ' + username + ' @ ' + date_str
from requests import get
from bs4 import BeautifulSoup
from bs4 import SoupStrainer
import lxml
import html2text
from sentiment import get_sentiment
def find_song(song_name, artist):
token = "Bearer token_here"
genius_search = "http://api.genius.com/search?q=" + "%20".join(song_name.split(" ")) + "%20" + "%20".join(
artist.split(" "))
Header = {
"Authorization": token
}
res = get(genius_search, headers=Header).json()
return (res['response']['hits'][0]['result']['url'])
def get_lyrics(url):
url = find_song(title, artist)
only_div = SoupStrainer('div', {"class": "lyrics"})
scrape = get(url).text
soup = BeautifulSoup(scrape, 'lxml', parse_only=only_div)
return (html2text.html2text(soup.text))
lyrics = get_lyrics(find_song(title, artist))
print(get_sentiment(lyrics))
for tt_utt in user_input_all:
conv = []
conv.append(tt_utt)
# clean the cash for alice
commands.getstatusoutput("rm c.txt")
f = open('simulate_conv/' + str(conv_index) + '.txt', 'w')
f.write('Turn: 0' + '\n')
f.write('You: Hello' + '\n')
f.write('TickTock: ' + tt_utt + '\n')
f.write('Appropriateness: ' + '\n')
f.write('Strategy: new' + '\n')
f.write('')
f.write('\n')
sent_3 = sentiment.get_sentiment(tt_utt)
sent_2 = 'null'
sent_1 = 'null'
theme[str(conv_index)] = random.choice(TopicLib)
previous_history[str(conv_index)] = ['Hello', tt_utt]
for turn_id in range(1, 10):
print turn_id
al_utt = alice.alice(tt_utt)
conv.append(al_utt)
f.write('Turn: ' + str(turn_id) + '\n')
f.write('You: ' + al_utt + '\n')
next_sent_1 = sent_3
next_sent_2 = sentiment.get_sentiment(al_utt)
state = (sent_1, sent_2, sent_3, turn_id)
# here we see if we go into get_response, it happen to be in any of the five strategy, then we select one to excecute. otherwise we stick to the original strategy.
theme_new, strategy, response, previous_history_new, word2vec = galbackend_online.get_response(
def find_stats(text):
sent_tokenizer = pickle.load(open("static/english.pickle", "rb"))
statistics = {}
sents = sent_tokenizer.tokenize(text)
words = []
for x in sents:
words.extend(nltk.word_tokenize(x))
total_words = len(words)
statistics["total_words"] = total_words
letters = "".join(words)
total_letters = len(letters)
statistics["total_letters"] = total_letters
total_sents = len(sents)
statistics["total_sents"] = total_sents
if total_words > 0:
statistics["avg_word_len"] = total_letters/total_words
else:
statistics["avg_word_len"] = 0
if total_sents > 0:
statistics["avg_sent_len"] = total_words/total_sents
else:
statistics["avg_sent_len"] = 0
try:
ari = (4.71*(total_letters/total_words) + 0.5*(total_words/total_sents) - 21.43)
except ZeroDivisionError:
ari = 1
ari = ceil(ari)
ari = max(ari, 1)
ari = min(ari, 14)
statistics["ari"] = ari
ari_table = ["Kindergarten", "First grade", "Second grade",
"Third grade", "Fourth grade", "Fifth grade",
"Sixth grade", "Seventh grade", "Eighth grade",
"Ninth grade", "Tenth grade", "Eleventh grade",
"Twelfth grade", "College"]
ari_grade_level = ari_table[ari-1]
statistics["ari_grade_level"] = ari_grade_level
ari_age_lower = ari + 4
if ari_age_lower == 18:
ari_age_upper = 22
else:
ari_age_upper = ari_age_lower + 1
statistics["ari_age_lower"] = ari_age_lower
statistics["ari_age_upper"] = ari_age_upper
ari_age_lower = str(ari_age_lower)
ari_age_upper = str(ari_age_upper)
ari_age_range = "%s -- %s" % (ari_age_lower, ari_age_upper)
statistics["ari_age_range"] = ari_age_range
# sentiment
pos, neg = get_sentiment(words)
sentiment = (pos-neg)/len(words)
statistics["sentiment"] = sentiment
return statistics
def analyze_text_block(text,
sentiment_library="textblob",
entity_library="spacy",
get_sentiment_per_topic=True):
text = re.sub('\s+', ' ', text)
text = text.replace("\n", ' ')
entities_res = entities.get_entities(text, library=entity_library)
keywords_res = keywords.get_keywords(text)
sentiment_res = sentiment.get_sentiment(text, library=sentiment_library)
lemmas_dict = {}
# Calculate sentiment per lemmas, keywords and entities, by averaging
# the sentiment for all the sentences that they appear in:
if get_sentiment_per_topic:
blob = TextBlob(text)
for sentence in blob.sentences:
sentence_score = sentiment.get_sentiment(
str(sentence), library=sentiment_library)['sentiment.score']
sentence_lemmas = lemmas.get_lemmas(sentence)
sentence = str(sentence).lower()
for lemma in sentence_lemmas:
lemmatxt = lemma['text']
if lemmatxt in lemmas_dict.keys():
lemmas_dict[lemmatxt]['sentiment.score'].append(
sentence_score)
else:
lemmas_dict[lemmatxt] = {
'sentiment.score': [sentence_score]
}
for keyword in keywords_res:
word = keyword['text']
if word.lower() in sentence:
if 'sentiment.score' not in keyword.keys():
keyword['sentiment.score'] = []
keyword['sentiment.score'].append(sentence_score)
for entity in entities_res:
word = entity['text']
if word.lower() in sentence:
if 'sentiment.score' not in entity.keys():
entity['sentiment.score'] = []
entity['sentiment.score'].append(sentence_score)
for keyword in keywords_res:
# WARNING: This is a hack. Happens when we have different libraries not agreeing on sentence boundaries!
if 'sentiment.score' not in keyword.keys():
keyword['sentiment.score'] = [sentiment_res['sentiment.score']]
keyword['num.sentences'] = len(keyword['sentiment.score'])
keyword['sentiment.score'] = np.mean(keyword['sentiment.score'])
for entity in entities_res:
# WARNING: This is a hack. Happens when we have different libraries not agreeing on sentence boundaries!
if 'sentiment.score' not in entity.keys():
entity['sentiment.score'] = [sentiment_res['sentiment.score']]
entity['num.sentences'] = len(entity['sentiment.score'])
entity['sentiment.score'] = np.mean(entity['sentiment.score'])
lemmas_res = []
for lemma in lemmas_dict.keys():
scores = lemmas_dict[lemma]['sentiment.score']
lemmas_res.append({
'text': lemma,
'num.sentences': len(scores),
'sentiment.score': np.mean(scores)
})
else:
lemmas_res = lemmas.get_lemmas(text)
results = {
'entities': entities_res,
'sentiment': sentiment_res,
'keywords': keywords_res,
'lemmas': lemmas_res
}
return (results)
def main():
"""
Run the app by initiating the conversation set to True
First there is a brief greeting opportunity, then you can ask questions
"""
storyNum = -1 # arbitrary value to mark first story being told
conversation = True
# speech = Pygsr()
# speech.record(5) # duration in seconds (3)
# phrase, complete_response = speech.speech_to_text('en_US') # select the language
# print phrase
# Start the greeting portion of the interaction
print_slow("\n>>Hello! I am Richard P Feynman. How are you?\n")
greeting = raw_input()
if not (s.get_sentiment(greeting) == "neg"):
# Use sentiment analysis to only accept positive or neutral sentences
# Used to avoid troll inputs
feyn_response = [line.rstrip("\n") for line in open("text/responses.txt")]
numLines = file_length("text/responses.txt")
rand = random.randint(0, numLines)
print_slow(">>" + feyn_response[rand])
else:
print ">>You're a bit of odd person arent you."
conversation = False
# Start the main conversation
while conversation:
print_slow("\n>>Do you have a question?\n")
response = str(raw_input())
if "beauty" in response or "flower" in response:
display("image/flower.jpg")
play_audio("audio/beauty.mp3")
elif "suggestion" in response:
print_slow(">>Well, I could talk to you about ...\n")
suggestions()
elif "principle" in response:
print_slow(
">>The first principle is that you must not fool yourself - and you are the easiest person to fool.\n"
)
elif "poetry" in response:
display("image/glass.jpg")
play_audio("audio/glass.mp3")
print_slow(
">>Poets say that science takes away from the beauty of the stars--mere globs of gas atoms. Nothing is mere. "
"I too can see the stars on a desert night and feel them. But do I see less or more? The vastness of the heavens stretches "
"my imagination - stuck on this carosusel my little eye can catch one-million-year-old light. A vast pattern - of which I am a "
"part - perhaps my stuff was belched from some forgotten star, as one is belching there. Or see them with the greater eye of Palomar, "
"rushing all apart from some common starting point when they were perhaps all together. What is the pattern, or the meaning, or the why? "
"It does not do harm to the mystery to know a little about it. For far more marvelous is the truth than any artists of the past imagined. "
"Why do the poets of the present not speak of it? What men are poets who can speak of Jupiter if he were like a man, but if he is an "
"immense spinning sphere of methane and ammonia must be silent?\n"
)
elif "philosophy" in response:
play_audio("audio/philosophy.mp3")
elif "radio" in response:
print_slow(">>Oh, here's a good story!\n")
play_audio("audio/Radios.mp3")
elif "father" in response or "inertia" in response:
play_audio("audio/inertia.mp3")
elif "teaching" in response:
play_audio("audio/teaching.mp3")
elif "doubt" in response or "uncertainty" in response:
display("image/knowing.jpg")
play_audio("audio/doubt.mp3")
elif "train" in response:
print_slow(">>Ah! Here's an interesting problem!\n")
play_audio("audio/train.mp3")
elif "social science" in response:
print_slow("Well here's my opinion: \n")
play_audio("audio/social.mp3")
elif "science" in response and "learn" in response:
play_audio("audio/miracle-people.mp3")
elif "nature" in response:
play_audio("audio/simplicity.mp3")
elif "aliens" in response or "flying saucers" in response:
play_audio("audio/aliens.mp3")
elif "why" in response:
play_audio("audio/why.mp3")
elif "tell me a story" in response:
storyNum = newStory(storyNum)
elif "feeling lost" in response:
with open("text/letter.txt", "r") as myfile:
letter = myfile.read()
print_slow(
">>Perhaps I should read you what I once wrote to a friend of mine who was feeling as you are now. I wrote: \n\n"
)
print_slow(letter)
converstaion = False
# end the conversation
elif "done" in response or "finished" in response:
print_slow(">>Well I'm glad to have talked a little with you! I hope we talk again soon!\n")
conversation = False
else:
print_slow(">>I have nothing to say about that!\n")
conversation = False
def push_tweet(data, timelineable, parse_terms):
global max_id
global min_id
global id_policy_bits
id = data["id"]
# merge tweet by id
tweet = graph.merge_one("Tweet", "id", id)
# timelining stuff
if timelineable:
if id > max_id:
max_id = id
if id < min_id:
min_id = id
id_policy_bits = id_policy_bits | USE_MAX_ID
# authorship
if "user" in data:
user = push_user(data["user"])
graph.create_unique(Relationship(user, "POSTS", tweet))
# quotes
if "quoted_status" in data:
tweet2 = push_tweet(data["quoted_status"], False, False)
graph.create_unique(Relationship(tweet, "QUOTES", tweet2))
# is a retweet
if "retweeted_status" in data:
tweet2 = push_tweet(data["retweeted_status"], False, False)
graph.create_unique(Relationship(tweet, "RETWEETS", tweet2))
# reply
reply = data.get("in_reply_to_status_id")
if reply:
reply_tweet = graph.merge_one("Tweet", "id",
data["in_reply_to_status_id"])
graph.create_unique(Relationship(tweet, "REPLY_TO", reply_tweet))
# geolocation exact/estimated
if data["coordinates"] is not None:
tweet.properties["lon"] = data["coordinates"]["coordinates"][0]
tweet.properties["lat"] = data["coordinates"]["coordinates"][1]
elif data["place"] is not None:
coordinates = data["place"]["bounding_box"]["coordinates"][0]
lon = (coordinates[0][0] + coordinates[1][0] + coordinates[2][0] +
coordinates[3][0]) / 4
lat = (coordinates[0][1] + coordinates[1][1] + coordinates[2][1] +
coordinates[3][1]) / 4
tweet.properties["lon"] = lon
tweet.properties["lat"] = lat
# fav count
tweet.properties["favorite_count"] = data["favorite_count"]
# rt count
tweet.properties["retweet_count"] = data["retweet_count"]
# text
tweet.properties["text"] = data["text"]
if "user" in data and parse_terms:
for tok in process_text(data["text"]):
word = push_word(tok)
if "terms" in user.properties:
# terms = user.properties["terms"]
# q = "{0}:".format(tok)
# idx = terms.find(q)
# if not idx == -1:
# sub = terms[(idx + len(q)):]
# sub = sub[:sub.find(" ")]
# q += sub
# terms = terms.replace(q, "{0}:{1}".format(tok, int(sub) + 1))
# else:
# terms += "{0}:1 ".format(tok)
# user.properties["terms"] = terms
user.properties[
"term_count"] = user.properties["term_count"] + 1
else:
user.properties["term_count"] = 1
# user.properties["terms"] = "{0}:1 ".format(tok)
user.push()
rel = graph.match_one(user, "DISCUSSES", word)
if rel:
rel.properties["count"] = rel.properties["count"] + 1
rel.push()
else:
rel = Relationship(user, "DISCUSSES", word)
rel.properties["count"] = 1
graph.create_unique(rel)
if "text" in data:
sent = sentiment.get_sentiment(data["text"])
tweet["polarity"] = sent[0]
tweet["subjectivity"] = sent[1]
for tok in process_text(data["text"]):
word = push_word(tok)
rel = graph.match_one(tweet, "CONTAINS", word)
if rel:
rel.properties["count"] = rel.properties["count"] + 1
rel.push()
else:
rel = Relationship(tweet, "CONTAINS", word)
rel.properties["count"] = 1
graph.create_unique(rel)
# hashtags
for h in data["entities"].get("hashtags", []):
hashtag = push_hashtag(h)
graph.create_unique(Relationship(hashtag, "TAGS", tweet))
# mentions
for m in data["entities"].get("user_mentions", []):
mention = push_user(m)
graph.create_unique(Relationship(tweet, "MENTIONS", mention))
tweet.push()
return tweet
import pickle
def most_common(lst):
return max(set(lst), key=lst.count)
conn = sqlite3.connect('rs_ratings.db')
c = conn.cursor()
c.execute('''SELECT * FROM responses''')
table_num = {}
table_sum = {}
tmplist =[]
for item in c.fetchall():
print item
#print item[0]
#break
turn_1_user_sent = sentiment.get_sentiment(item[0])
turn_1_tt_sent = sentiment.get_sentiment(item[1])
turn_0_user_sent = sentiment.get_sentiment(item[7])
turn_0_tt_sent = sentiment.get_sentiment(item[8])
turn_str = item[10]
if item[11] == None or item[12] == None or item[13] ==None:
break
score_list = [ item[11], item[12], item[13] ]
score = most_common(score_list)
tmplist.append([turn_0_user_sent,turn_0_tt_sent,turn_1_user_sent,turn_str,score])
key = (turn_0_user_sent, turn_0_tt_sent,turn_1_user_sent,turn_str)
if table_sum.has_key(key):
table_sum[key]= table_sum[key]+score
table_num[key] = table_num[key]+1
else:
table_sum[key] = score
import sentiment
import pickle
import readall
rating_logs = readall.readall("/home/ubuntu/zhou/Backend/rating_log/")
user_input = readall.get_log(rating_logs)
'''
with open('user_input_v2.pkl') as ff:
user_input = pickle.load(ff)
'''
f = open('sentiment_log.txt', 'w')
sentiment_label = []
for turn in user_input:
question = turn['question']
label = sentiment.get_sentiment(question)
sentiment_label.append(label)
f.write(question + '\n')
f.write('sentiment: ' + label + '\n')
pos_number = sentiment_label.count('pos')
print pos_number
neg_number = sentiment_label.count('neg')
print neg_number
neutral_number = sentiment_label.count('neutral')
print neutral_number
return value
for tt_utt in user_input_all:
conv =[]
conv.append(tt_utt)
# clean the cash for alice
commands.getstatusoutput("rm c.txt")
f = open('simulate_conv/'+str(conv_index)+'.txt','w')
f.write('Turn: 0'+'\n')
f.write('You: Hello'+'\n' )
f.write('TickTock: ' + tt_utt +'\n')
f.write('Appropriateness: ' + '\n')
f.write('Strategy: new' + '\n')
f.write('')
f.write('\n')
sent_3 = sentiment.get_sentiment(tt_utt)
sent_2 = 'null'
sent_1 = 'null'
theme[str(conv_index)] = random.choice(TopicLib)
previous_history[str(conv_index)] = ['Hello',tt_utt]
for turn_id in range(1,10):
print turn_id
al_utt = alice.alice(tt_utt)
conv.append(al_utt)
f.write('Turn: ' + str(turn_id) +'\n')
f.write('You: ' + al_utt+'\n')
next_sent_1 = sent_3
next_sent_2 = sentiment.get_sentiment(al_utt)
state = (sent_1,sent_2,sent_3,turn_id)
# here we see if we go into get_response, it happen to be in any of the five strategy, then we select one to excecute. otherwise we stick to the original strategy.
theme_new, strategy, response,previous_history_new,word2vec = galbackend_online.get_response( None,policy_mode,al_utt, str(conv_index) ,previous_history,theme, oov_state,name_entity_state,short_answer_state,anaphra_state,word2vec_ranking_state,tfidf_state)
max_statuses_count = tweet["user"]["statuses_count"]
# Can be called when analysing data
# data_analysis(statuses)
for tweet in statuses:
data = {}
if tweet.has_key('user'):
if tweet['user'].has_key('name'):
data["screenname"] = tweet["user"]["name"]
elif tweet['user'].has_key('screenname'):
data["screenname"] = tweet["user"]["screenname"]
data["date"] = tweet["created_at"]
data["value"] = 1
data["text"] = tweet["text"]
if tweet["lang"] == "en" and len(tweet["text"]) > 2:
data["sentiment"] = get_sentiment(tweet["text"])
elif len(tweet["text"])% 7 < 3:
data["sentiment"] = 1
elif len(tweet["text"])% 7 > 5:
data["sentiment"] = 2
else:
data["sentiment"] = 0
date = tweet["created_at"].split()
data["time"] = int(date[3].split(":")[0])
data["minutes"] = int(date[3].split(":")[1])
data["lang"] = tweet["lang"]
try:
data["gender"] = user_gender.get(data["screenname"])
except:
data["gender"] = "female"
resolved_texts = [entity.resolve_anaphores(text) for text in texts]
domain_excluded_entities = domainexcludedentities.\
get_domain_excluded_entities(texts)
contexts_lists = [context.extract(text) for text in texts]
contexts = []
for contexts_list in contexts_lists:
contexts += contexts_list
entities = []
entities_sentiments = {}
concept_graph = conceptgraph.ConceptGraph()
for candidate_context in contexts:
entity_sentiment = sentiment.get_sentiment(candidate_context)
context_entities_set = set(entity.extract(candidate_context))
for extracted_entity in context_entities_set:
if extracted_entity in domain_excluded_entities:
continue
entities.append(extracted_entity)
if extracted_entity not in entities_sentiments:
entities_sentiments[extracted_entity] = []
entities_sentiments[extracted_entity].append(entity_sentiment)
for (entity1, entity2) in itertools.combinations(context_entities_set, 2):
if (entity1 in domain_excluded_entities) or\
(entity2 in domain_excluded_entities):
continue
concept_graph.add_edge(entity1, entity2)
from bs4 import BeautifulSoup
import requests
import sentiment
url = "https://itunes.apple.com/us/rss/customerreviews/id=954338382/sortBy=mostRecent/xml"
r = requests.get(url)
data = r.text
soup = BeautifulSoup(data, "html.parser")
for block in soup.find_all('title'):
text = block.find_next_sibling()
if text.name != 'content':
continue
text = text.text
print(text)
sentiment.get_sentiment(text)
import pdb
pdb.set_trace()