def get_text_analysis(self):
cur = self.conn.cursor()
query = ("SELECT tweet_id, text FROM tweets "
"WHERE text_analysis::text='{}'::text;")
cur.execute(query)
rows = cur.fetchall()
for row in rows:
text = row[1]
text = self.clean_text(text)
id = row[0]
try:
analysis = indicoio.analyze_text(
text,
apis=['sentiment', 'political', 'keywords', 'text_tags'])
json_analysis = json.dumps(analysis)
# print("analysing")
query = "update tweets set text_analysis=%s where tweet_id=%s;"
cur.execute(query, (json_analysis, id))
self.conn.commit()
except Exception as ex:
template = "An exception of type {0} occurred. Arguments:\n{1!r}"
message = template.format(type(ex).__name__, ex.args)
print message
print "Error with tweet. Id: " + id + " text: " + text
if ex.args[0] == 'Input contains one or more empty strings.':
query = "delete from tweets where tweet_id='%s';" % id
cur.execute(query)
self.conn.commit()
print('tweet deleted, text= ' + text)
print "done indico analysis"
def analyze_tweets(tweet_dict):
indico_dict = {}
for key in tweet_dict:
indico_dict[key] = []
for tweet in tweet_dict[key]:
indico_dict[key].append(indicoio.analyze_text(tweet, apis=['sentiment', 'political','twitter_engagement','personality','personas','emotion']))
def analyse_text():
input_text = request.form["text"]
user_key = current_talking_user
action = request.form.get("action")
if action == "political":
indico_response = indicoio.analyze_text(input_text, apis=['political'])
political_party = indico_response["political"]
top_political_party = sorted(political_party.keys(), key=lambda x: political_party[x], reverse=True)[0]
return top_political_party
else:
indico_response = indicoio.analyze_text(input_text, apis=['keywords'])
keywords = indico_response["keywords"]
keywords_ = (sorted(keywords.keys(), key=lambda x: keywords[x], reverse=True)[:5])
return keywords_
def trendtweets(request, query):
print(query)
tweets = []
text = []
for tweet in tweepy.Cursor(api.search, q=query + ' -RT',
lang="en").items(100):
jtweet = {}
jtweet['created_at'] = tweet._json['created_at']
jtweet['text'] = tweet._json['text']
text.append(jtweet['text'])
#params = {'text': jtweet['text']}
#try:
#analysis = client.get_request(params, HODApps.ANALYZE_SENTIMENT, async=False)
#sentiment = analysis['aggregate']['score']
#except:
#continue
jtweet['location'] = tweet._json['user']['location']
tweets.append(jtweet)
print(len(tweets))
analyze = indicoio.analyze_text(text,
apis=['people', 'places', 'organizations'],
threshold=0.7)
places = analyze['places']
people = analyze['people']
org = analyze['organizations']
print(org)
pl = []
ppl = []
og = []
for place in places:
if len(place) != 0:
pl.append(place[0]['text'])
for p in people:
if len(p) != 0:
ppl.append(p[0]['text'])
for o in org:
if len(o) != 0:
og.append(o[0]['text'])
pl = list(set(pl))
ppl = list(set(ppl))
og = list(set(og))
print(pl)
print(ppl)
print(og)
return JsonResponse({
'tweets': tweets,
'people': ppl,
'places': pl,
'org': og
})
def generate_keywords(self):
# get keywords
keywords_dict = indicoio.keywords(self.tweet)
all_else_dict = indicoio.analyze_text(
self.tweet, apis=['places', 'people', 'organizations'])
# store identified keywords
keywords = []
for key, value in keywords_dict.iteritems():
if value > 0.2:
keywords.append(key)
for key, value in all_else_dict.iteritems():
for list_item in value:
for key_in, value_in in list_item.iteritems():
if key_in == 'text':
keywords.append(value_in)
return keywords
def analyse_text(text, list_algo_fv):
"""
Method that analyses a text based on given set of algorithms.
:param text: Text to be analysed.
:param list_algo_fv: List of 'feedvay' algorithms enums
:return: Data dictionary containing algo_key as key and result as value.
**Exceptions**: Throws ``KeyError`` exception if there are any invalid keys
that do not match according to ``TextAlgorithms.mapping_feedvay_indicio``.
**Authors**: Gagandeep Singh
"""
# --- Validation ---
# Validate text
if not (isinstance(text, str) or isinstance(text, unicode)):
raise TypeError("text must a string or unicode.")
# Validate algo keys and transform to indicoio keys
list_algo_keys = [
TextAlgorithms.mapping_feedvay_indicio[algo_key_fv]
for algo_key_fv in list_algo_fv
]
# --- /Validation ---
# Call service in bulk
result_indico = indicoio.analyze_text(text,
apis=list_algo_keys,
api_key=settings.INDICOIO_API_KEY)
# Formalize results
map_indicoio_feedvay = TextAlgorithms.get_mapping_indicio_feedvay()
result = {}
for key_indico, output in result_indico.iteritems():
if key_indico == 'sentiment':
output = TextSentimentAnalysis.formalize_results(output)
result[map_indicoio_feedvay[key_indico]] = output
return result
def tweet(self, text):
if len(text) > 140:
print("Sorry, but your tweet is more than 140 characters. Try a shorter one.")
return
tts.speak("Analyzing text...")
analysis = indicoio.analyze_text(
text,
apis=['sentiment_hq', 'political', 'twitter_engagement'])
tts.speak("Okay, let's take a look at the results.")
print_goodness_message(
analysis["sentiment_hq"],
"Nice positivity!",
"Emotion seems chill.",
"That seems a little negative.",
"That seems pretty negative.")
if analysis["sentiment_hq"] <= 0.2:
tts.speak("Hey, do you want to talk about this?")
print_goodness_message(
analysis["twitter_engagement"],
"I suspect that this tweet'll be pretty popular.",
"This'll probably be somewhat popular.",
"I'm not sure if this'll appeal to many people.",
"I think this one will be unpopular.")
political_bias = get_political_bias(analysis["political"])
if political_bias[1] >= 0.40:
tts.speak("Seems that your tweet has a high {} bias.".format(political_bias[0]))
if False:
return False
else:
# Tweet it!
self.api.update_status(text)
tts.speak("And tweeted! Thanks.")
return True
def choose_post(response_json):
posts = []
posts_messages = []
data = response_json['data']
for post in data:
if 'message' not in post:
continue
likes = 0
if 'likes' in post:
likes = len(post['likes'])
posts.append(Post(post['id'], post['created_time'],post['message'], likes, None))
posts_messages.append(post['message'])
apis = ['sentiment_hq', 'text_tags', 'personality']
analyses = indicoio.analyze_text(posts_messages, apis=apis)
posts_list = []
#for analysis_type, vals in analyses.iteritems():
for post, text_tags, sentiment_hq, personality in itertools.izip(posts, *analyses.values()):
likes = post.likes
sentiment = sentiment_hq
tags = text_tags
personalities = personality
# Sentiment analysis debugging
# top_tags = sorted(tags.items(), key=operator.itemgetter(1), reverse=True)
# top_tags = top_tags[:10]
# print '\n=========================================================='
# print post.message
# for key, val in top_tags: print "%s: %0.5f" % (key, val)
posts_list.append(Post(post.id, post.date, post.message, post.likes, score_post(likes, sentiment, tags, personalities)))
posts_list.sort(key=get_score, reverse=True)
return random.choice(posts_list[:len(posts_list)/2])
def analyze(query):
tweets = get_twitter_data(query)
tweet_text = [tweet.text for tweet in tweets]
num_tweets = len(tweets)
apis = ['sentiment', 'political']
analysis = indicoio.analyze_text(tweet_text, apis)
# print(analysis['sentiment'])
avg_sentiment = sum(analysis['sentiment']) / num_tweets
avg_libertarian = sum([p['Libertarian']
for p in analysis['political']]) / num_tweets
avg_green = sum([p['Green'] for p in analysis['political']]) / num_tweets
avg_liberal = sum([p['Liberal']
for p in analysis['political']]) / num_tweets
avg_conservative = sum([p['Conservative']
for p in analysis['political']]) / num_tweets
values = (num_tweets, avg_sentiment, avg_libertarian, avg_green,
avg_liberal, avg_conservative)
return avg_sentiment
def analyze(query):
tweets = get_twitter_data(query)
tweet_text = [tweet.text for tweet in tweets]
num_tweets = len(tweets)
apis = ['sentiment', 'political']
analysis = indicoio.analyze_text(tweet_text, apis)
# print(analysis['sentiment'])
avg_sentiment = sum(analysis['sentiment']) / num_tweets
avg_libertarian = sum([p['Libertarian'] for p in analysis['political']]) / num_tweets
avg_green = sum([p['Green'] for p in analysis['political']]) / num_tweets
avg_liberal = sum([p['Liberal'] for p in analysis['political']]) / num_tweets
avg_conservative = sum([p['Conservative'] for p in analysis['political']]) / num_tweets
print(str(num_tweets) + ' tweets for search "' + query + '"')
print('Average sentiment: ' + str(avg_sentiment))
print('Average libertarian: ' + str(avg_libertarian))
print('Average green: ' + str(avg_green))
print('Average liberal: ' + str(avg_liberal))
print('Average conservative: ' + str(avg_conservative))
def test_multi_api_text(self):
test_data = "clearly an english sentence"
response = analyze_text(test_data)
self.assertTrue(isinstance(response, dict))
self.assertTrue(set(response.keys()) <= set(TEXT_APIS.keys()))
def analyze_tweets(tweet_dict):
print('done organizing tweets, analyzing now')
indico_dict = {}
tweet_batches = {}
for key in tweet_dict:
tweet_batches[key] = []
for tweet in tweet_dict[key]:
tweet_batches[key].append(tweet)
indicoio.config.api_key = key1
print('key 1')
for month in tweet_batches:
indico_dict[month] = []
returndict = (indicoio.analyze_text(tweet_batches[month], apis=['sentiment','twitter_engagement','political']))
indico_dict[month].append({'sentiment' : sum(returndict['sentiment'])/len(returndict['sentiment'])})
indico_dict[month].append({'twitter_engagement' : sum(returndict['twitter_engagement'])/len(returndict['twitter_engagement'])})
indico_dict[month].append({'political': {'Libertarian': 0, 'Liberal': 0, 'Green': 0, 'Conservative': 0}})
for mydict in returndict['political']:
indico_dict[month][2]['political']['Libertarian'] += mydict['Libertarian']
indico_dict[month][2]['political']['Liberal'] += mydict['Liberal']
indico_dict[month][2]['political']['Green'] += mydict['Green']
indico_dict[month][2]['political']['Conservative'] += mydict['Conservative']
indico_dict[month][2]['political']['Libertarian'] = indico_dict[month][2]['political']['Libertarian']/len(returndict['political'])
indico_dict[month][2]['political']['Liberal'] = indico_dict[month][2]['political']['Liberal']/len(returndict['political'])
indico_dict[month][2]['political']['Green'] = indico_dict[month][2]['political']['Green']/len(returndict['political'])
indico_dict[month][2]['political']['Conservative'] = indico_dict[month][2]['political']['Conservative']/len(returndict['political'])
indicoio.config.api_key = key2
print('key 2')
for month in tweet_batches:
returndict = (indicoio.analyze_text(tweet_batches[month], apis=['personality','personas','emotion']))
indico_dict[month].append({'personality': {'openness': 0, 'extraversion': 0, 'agreeableness': 0, 'conscientiousness': 0}})
for mydict in returndict['personality']:
indico_dict[month][3]['personality']['openness'] += mydict['openness']
indico_dict[month][3]['personality']['extraversion'] += mydict['extraversion']
indico_dict[month][3]['personality']['agreeableness'] += mydict['agreeableness']
indico_dict[month][3]['personality']['conscientiousness'] += mydict['conscientiousness']
indico_dict[month][3]['personality']['openness'] = indico_dict[month][3]['personality']['openness']/len(returndict['personality'])
indico_dict[month][3]['personality']['extraversion'] = indico_dict[month][3]['personality']['extraversion']/len(returndict['personality'])
indico_dict[month][3]['personality']['agreeableness'] = indico_dict[month][3]['personality']['agreeableness']/len(returndict['personality'])
indico_dict[month][3]['personality']['conscientiousness'] = indico_dict[month][3]['personality']['conscientiousness']/len(returndict['personality'])
indico_dict[month].append({'emotion': {'anger': 0, 'joy': 0, 'fear': 0, 'sadness': 0, 'surprise' : 0}})
for mydict in returndict['emotion']:
indico_dict[month][4]['emotion']['anger'] += mydict['anger']
indico_dict[month][4]['emotion']['joy'] += mydict['joy']
indico_dict[month][4]['emotion']['fear'] += mydict['fear']
indico_dict[month][4]['emotion']['sadness'] += mydict['sadness']
indico_dict[month][4]['emotion']['surprise'] += mydict['surprise']
indico_dict[month][4]['emotion']['anger'] = indico_dict[month][4]['emotion']['anger']/len(returndict['emotion'])
indico_dict[month][4]['emotion']['joy'] = indico_dict[month][4]['emotion']['joy']/len(returndict['emotion'])
indico_dict[month][4]['emotion']['fear'] = indico_dict[month][4]['emotion']['fear']/len(returndict['emotion'])
indico_dict[month][4]['emotion']['sadness'] = indico_dict[month][4]['emotion']['sadness']/len(returndict['emotion'])
indico_dict[month][4]['emotion']['surprise'] = indico_dict[month][4]['emotion']['surprise']/len(returndict['emotion'])
indico_dict[month].append({'personas': {'advocate': 0, 'debater': 0, 'mediator': 0, 'consul': 0, 'executive' : 0, 'adventurer' : 0, 'logistician' : 0,
'commander' : 0, 'entrepreneur' : 0, 'logician' : 0, 'protagonist' : 0, 'architect' : 0, 'campaigner' : 0, 'entertainer' : 0, 'defender' : 0, 'virtuoso' : 0}})
for mydict in returndict['personas']:
indico_dict[month][5]['personas']['advocate'] += mydict['advocate']
indico_dict[month][5]['personas']['debater'] += mydict['debater']
indico_dict[month][5]['personas']['mediator'] += mydict['mediator']
indico_dict[month][5]['personas']['consul'] += mydict['consul']
indico_dict[month][5]['personas']['executive'] += mydict['executive']
indico_dict[month][5]['personas']['adventurer'] += mydict['adventurer']
indico_dict[month][5]['personas']['logistician'] += mydict['logistician']
indico_dict[month][5]['personas']['commander'] += mydict['commander']
indico_dict[month][5]['personas']['entrepreneur'] += mydict['entrepreneur']
indico_dict[month][5]['personas']['logician'] += mydict['logician']
indico_dict[month][5]['personas']['protagonist'] += mydict['protagonist']
indico_dict[month][5]['personas']['architect'] += mydict['architect']
indico_dict[month][5]['personas']['campaigner'] += mydict['campaigner']
indico_dict[month][5]['personas']['entertainer'] += mydict['entertainer']
indico_dict[month][5]['personas']['defender'] += mydict['defender']
indico_dict[month][5]['personas']['virtuoso'] += mydict['virtuoso']
indico_dict[month][5]['personas']['advocate'] = indico_dict[month][5]['personas']['advocate']/len(returndict['personas'])
indico_dict[month][5]['personas']['debater'] = indico_dict[month][5]['personas']['debater']/len(returndict['personas'])
indico_dict[month][5]['personas']['mediator'] = indico_dict[month][5]['personas']['mediator']/len(returndict['personas'])
indico_dict[month][5]['personas']['consul'] = indico_dict[month][5]['personas']['consul']/len(returndict['personas'])
indico_dict[month][5]['personas']['executive'] = indico_dict[month][5]['personas']['executive']/len(returndict['personas'])
indico_dict[month][5]['personas']['adventurer'] = indico_dict[month][5]['personas']['adventurer']/len(returndict['personas'])
indico_dict[month][5]['personas']['logistician'] = indico_dict[month][5]['personas']['logistician']/len(returndict['personas'])
indico_dict[month][5]['personas']['commander'] = indico_dict[month][5]['personas']['commander']/len(returndict['personas'])
indico_dict[month][5]['personas']['entrepreneur'] = indico_dict[month][5]['personas']['entrepreneur']/len(returndict['personas'])
indico_dict[month][5]['personas']['logician'] = indico_dict[month][5]['personas']['logician']/len(returndict['personas'])
indico_dict[month][5]['personas']['protagonist'] = indico_dict[month][5]['personas']['protagonist']/len(returndict['personas'])
indico_dict[month][5]['personas']['architect'] = indico_dict[month][5]['personas']['architect']/len(returndict['personas'])
indico_dict[month][5]['personas']['campaigner'] = indico_dict[month][5]['personas']['campaigner']/len(returndict['personas'])
indico_dict[month][5]['personas']['entertainer'] = indico_dict[month][5]['personas']['entertainer']/len(returndict['personas'])
indico_dict[month][5]['personas']['defender'] = indico_dict[month][5]['personas']['defender']/len(returndict['personas'])
indico_dict[month][5]['personas']['virtuoso'] = indico_dict[month][5]['personas']['virtuoso']/len(returndict['personas'])
return indico_dict
def test_multi_api_text(self):
test_data = 'clearly an english sentence'
response = analyze_text(test_data, apis=config.TEXT_APIS, api_key=self.api_key)
self.assertTrue(isinstance(response, dict))
self.assertTrue(set(response.keys()) == set(config.TEXT_APIS))
def setData(): currentData = json.dumps(indicoio.analyze_text(str(request.form['politics']), apis=['political', 'sentiment_hq', 'keywords'])) data.append(currentData) sentences.append(request.form['politics']) return str(currentData)
def processText(): text = request.form['text'] r = indicoio.analyze_text(text, apis=['text_tags', 'sentiment_hq', 'keywords']); app.logger.debug(r) return json.jsonify(**r);
def analyze_tweets(tweet_dict):
print('done organizing tweets, analyzing now')
indico_dict = {}
tweet_batches = {}
for key in tweet_dict:
tweet_batches[key] = []
for tweet in tweet_dict[key]:
tweet_batches[key].append(tweet)
for month in tweet_batches:
indico_dict[month] = []
returndict = (indicoio.analyze_text(tweet_batches[month], apis=['sentiment','twitter_engagement','political','personality','personas','emotion']))
indico_dict[month].append({'sentiment' : sum(returndict['sentiment'])/len(returndict['sentiment'])})
indico_dict[month].append({'twitter_engagement' : sum(returndict['twitter_engagement'])/len(returndict['twitter_engagement'])})
indico_dict[month].append({'political': {'Libertarian': 0, 'Liberal': 0, 'Green': 0, 'Conservative': 0}})
for mydict in returndict['political']:
indico_dict[month][2]['political']['Libertarian'] += mydict['Libertarian']
indico_dict[month][2]['political']['Liberal'] += mydict['Liberal']
indico_dict[month][2]['political']['Green'] += mydict['Green']
indico_dict[month][2]['political']['Conservative'] += mydict['Conservative']
indico_dict[month][2]['political']['Libertarian'] = indico_dict[month][2]['political']['Libertarian']/len(returndict['political'])
indico_dict[month][2]['political']['Liberal'] = indico_dict[month][2]['political']['Liberal']/len(returndict['political'])
indico_dict[month][2]['political']['Green'] = indico_dict[month][2]['political']['Green']/len(returndict['political'])
indico_dict[month][2]['political']['Conservative'] = indico_dict[month][2]['political']['Conservative']/len(returndict['political'])
indico_dict[month].append({'personality': {'openness': 0, 'extraversion': 0, 'agreeableness': 0, 'conscientiousness': 0}})
for mydict in returndict['personality']:
indico_dict[month][3]['personality']['openness'] += mydict['openness']
indico_dict[month][3]['personality']['extraversion'] += mydict['extraversion']
indico_dict[month][3]['personality']['agreeableness'] += mydict['agreeableness']
indico_dict[month][3]['personality']['conscientiousness'] += mydict['conscientiousness']
indico_dict[month][3]['personality']['openness'] = indico_dict[month][3]['personality']['openness']/len(returndict['personality'])
indico_dict[month][3]['personality']['extraversion'] = indico_dict[month][3]['personality']['extraversion']/len(returndict['personality'])
indico_dict[month][3]['personality']['agreeableness'] = indico_dict[month][3]['personality']['agreeableness']/len(returndict['personality'])
indico_dict[month][3]['personality']['conscientiousness'] = indico_dict[month][3]['personality']['conscientiousness']/len(returndict['personality'])
indico_dict[month].append({'emotion': {'anger': 0, 'joy': 0, 'fear': 0, 'sadness': 0, 'surprise' : 0}})
for mydict in returndict['emotion']:
indico_dict[month][4]['emotion']['anger'] += mydict['anger']
indico_dict[month][4]['emotion']['joy'] += mydict['joy']
indico_dict[month][4]['emotion']['fear'] += mydict['fear']
indico_dict[month][4]['emotion']['sadness'] += mydict['sadness']
indico_dict[month][4]['emotion']['surprise'] += mydict['surprise']
indico_dict[month][4]['emotion']['anger'] = indico_dict[month][4]['emotion']['anger']/len(returndict['emotion'])
indico_dict[month][4]['emotion']['joy'] = indico_dict[month][4]['emotion']['joy']/len(returndict['emotion'])
indico_dict[month][4]['emotion']['fear'] = indico_dict[month][4]['emotion']['fear']/len(returndict['emotion'])
indico_dict[month][4]['emotion']['sadness'] = indico_dict[month][4]['emotion']['sadness']/len(returndict['emotion'])
indico_dict[month][4]['emotion']['surprise'] = indico_dict[month][4]['emotion']['surprise']/len(returndict['emotion'])
indico_dict[month].append({'personas': {'advocate': 0, 'debater': 0, 'mediator': 0, 'consul': 0, 'executive' : 0, 'adventurer' : 0, 'logistician' : 0,
'commander' : 0, 'entrepreneur' : 0, 'logician' : 0, 'protagonist' : 0, 'architect' : 0, 'campaigner' : 0, 'entertainer' : 0, 'defender' : 0, 'virtuoso' : 0}})
for mydict in returndict['personas']:
indico_dict[month][5]['personas']['advocate'] += mydict['advocate']
indico_dict[month][5]['personas']['debater'] += mydict['debater']
indico_dict[month][5]['personas']['mediator'] += mydict['mediator']
indico_dict[month][5]['personas']['consul'] += mydict['consul']
indico_dict[month][5]['personas']['executive'] += mydict['executive']
indico_dict[month][5]['personas']['adventurer'] += mydict['adventurer']
indico_dict[month][5]['personas']['logistician'] += mydict['logistician']
indico_dict[month][5]['personas']['commander'] += mydict['commander']
indico_dict[month][5]['personas']['entrepreneur'] += mydict['entrepreneur']
indico_dict[month][5]['personas']['logician'] += mydict['logician']
indico_dict[month][5]['personas']['protagonist'] += mydict['protagonist']
indico_dict[month][5]['personas']['architect'] += mydict['architect']
indico_dict[month][5]['personas']['campaigner'] += mydict['campaigner']
indico_dict[month][5]['personas']['entertainer'] += mydict['entertainer']
indico_dict[month][5]['personas']['defender'] += mydict['defender']
indico_dict[month][5]['personas']['virtuoso'] += mydict['virtuoso']
indico_dict[month][5]['personas']['advocate'] = indico_dict[month][5]['personas']['advocate']/len(returndict['personas'])
indico_dict[month][5]['personas']['debater'] = indico_dict[month][5]['personas']['debater']/len(returndict['personas'])
indico_dict[month][5]['personas']['mediator'] = indico_dict[month][5]['personas']['mediator']/len(returndict['personas'])
indico_dict[month][5]['personas']['consul'] = indico_dict[month][5]['personas']['consul']/len(returndict['personas'])
indico_dict[month][5]['personas']['executive'] = indico_dict[month][5]['personas']['executive']/len(returndict['personas'])
indico_dict[month][5]['personas']['adventurer'] = indico_dict[month][5]['personas']['adventurer']/len(returndict['personas'])
indico_dict[month][5]['personas']['logistician'] = indico_dict[month][5]['personas']['logistician']/len(returndict['personas'])
indico_dict[month][5]['personas']['commander'] = indico_dict[month][5]['personas']['commander']/len(returndict['personas'])
indico_dict[month][5]['personas']['entrepreneur'] = indico_dict[month][5]['personas']['entrepreneur']/len(returndict['personas'])
indico_dict[month][5]['personas']['logician'] = indico_dict[month][5]['personas']['logician']/len(returndict['personas'])
indico_dict[month][5]['personas']['protagonist'] = indico_dict[month][5]['personas']['protagonist']/len(returndict['personas'])
indico_dict[month][5]['personas']['architect'] = indico_dict[month][5]['personas']['architect']/len(returndict['personas'])
indico_dict[month][5]['personas']['campaigner'] = indico_dict[month][5]['personas']['campaigner']/len(returndict['personas'])
indico_dict[month][5]['personas']['entertainer'] = indico_dict[month][5]['personas']['entertainer']/len(returndict['personas'])
indico_dict[month][5]['personas']['defender'] = indico_dict[month][5]['personas']['defender']/len(returndict['personas'])
indico_dict[month][5]['personas']['virtuoso'] = indico_dict[month][5]['personas']['virtuoso']/len(returndict['personas'])
return indico_dict
import scrape
def parse_tweets(screen_name):
scrape.get_all_tweets(screen_name)
tweetlist = []
f = open('%s.txt' % screen_name, 'r')
for line in f:
lines = line.split('\t')
lines[0] = lines[0][:-9]
lines[1] = lines[1][2:-2]
tweetlist.append(lines)
f.close()
return tweetlist
def organize_tweets(tweets):
for tweet in tweets:
if tweet[0][5:7] == '01':
tweet[0] = 'January' + str(tweet[0][0:4])
elif tweet[0][5:7] == '02':
tweet[0] = 'February' + str(tweet[0][0:4])
elif tweet[0][5:7] == '03':
tweet[0] = 'March' + str(tweet[0][0:4])
elif tweet[0][5:7] == '04':
tweet[0] = 'April' + str(tweet[0][0:4])
elif tweet[0][5:7] == '05':
tweet[0] = 'May' + str(tweet[0][0:4])
elif tweet[0][5:7] == '06':
tweet[0] = 'June' + str(tweet[0][0:4])
elif tweet[0][5:7] == '07':
tweet[0] = 'July' + str(tweet[0][0:4])
elif tweet[0][5:7] == '08':
tweet[0] = 'August' + str(tweet[0][0:4])
elif tweet[0][5:7] == '09':
tweet[0] = 'September' + str(tweet[0][0:4])
elif tweet[0][5:7] == '10':
tweet[0] = 'October' + str(tweet[0][0:4])
elif tweet[0][5:7] == '11':
tweet[0] = 'November' + str(tweet[0][0:4])
elif tweet[0][5:7] == '12':
tweet[0] = 'December' + str(tweet[0][0:4])
tweet_dict={}
for tweet in tweets:
if tweet[0] not in tweet_dict:
tweet_dict[tweet[0]] = [tweet[1]]
else:
tweet_dict[tweet[0]].append(tweet[1])
return tweet_dict
if __name__ == '__main__':
#pass in the username of the account you want to download
organize_tweets(parse_tweets("HackSentivents"))
import tweepy #https://github.com/tweepy/tweepy
#Twitter API credentials
consumer_key = 'XuJqupkF9ZUq8oIz921Nl141O'
consumer_secret = '9z0JyLdMNRT7brH14wh32fUqxbpL0yDXu70Ez8mJFpLifmv3nl'
access_key = '853038411426521090-yviLNx5KWrXE7lXrubdjKlwOQ0WoC8R'
access_secret = 'wCxcNrBfJi2PhTVw9Xgi1DF4EFb1Z38cS7z0PAgC3Jd6D'
def get_all_tweets(screen_name):
#Twitter only allows access to a users most recent 3240 tweets with this method
#authorize twitter, initialize tweepy
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
api = tweepy.API(auth)
#initialize a list to hold all the tweepy Tweets
alltweets = []
#make initial request for most recent tweets (200 is the maximum allowed count)
new_tweets = api.user_timeline(screen_name = screen_name,count=200)
#save most recent tweets
alltweets.extend(new_tweets)
#save the id of the oldest tweet less one
oldest = alltweets[-1].id - 1
#keep grabbing tweets until there are no tweets left to grab
while len(new_tweets) > 0:
#print "getting tweets before %s" % (oldest)
#all subsiquent requests use the max_id param to prevent duplicates
new_tweets = api.user_timeline(screen_name = screen_name,count=200,max_id=oldest)
#save most recent tweets
alltweets.extend(new_tweets)
#update the id of the oldest tweet less one
oldest = alltweets[-1].id - 1
#print "...%s tweets downloaded so far" % (len(alltweets))
#transform the tweepy tweets into a 2D array that will populate the csv
outtweets = [[tweet.id_str, tweet.created_at, tweet.text.encode("utf-8")] for tweet in alltweets]
#return outtweets
#write the csv
with open('%s.txt' % screen_name, 'w') as f:
for i in outtweets:
f.write(str(i[1])+'\t')
f.write(str(i[2])+'\n')
pass
if __name__ == '__main__':
#pass in the username of the account you want to download
get_all_tweets("HackSentivents")
def test_default_multi_api_text(self):
test_data = ['clearly an english sentence']
response = analyze_text(test_data)
self.assertTrue(isinstance(response, dict))
self.assertTrue(set(response.keys()) <= set(config.TEXT_APIS))
print("Skipping " + filename + "...")
visited = True
break
if visited == False:
for file in output_dataframe['subreddit'].str.contains(filename):
if file == filename:
print("Yes!")
print("Adding " + filename + "...")
path = directory + '/' + filename
with open(path, 'r') as myfile:
data = myfile.read()
response = indicoio.analyze_text(data,
apis=[
'sentiment', 'emotion',
'personality',
'twitter_engagement'
])
# Build data dictionary
d = OrderedDict()
d['sentiment'] = response['sentiment']
d.update(response['emotion'])
d.update(response['personality'])
# d.update(response['twitter_engagement'])
d['twitter_engagement'] = response['twitter_engagement']
# Convert to DataFrame
columns = list(d.keys())
values = list(d.values())
arr_len = len(values)
def test_multi_api_text(self):
test_data = 'clearly an english sentence'
response = analyze_text(test_data)
self.assertTrue(isinstance(response, dict))
self.assertTrue(set(response.keys()) <= set(TEXT_APIS.keys()))
#organizations(data, [api_key], [cloud], [threshold])
#twitter_engagement(data, [api_key], [cloud])
#personality(data, [api_key], [cloud])
#relevance(data, queries, [api_key], [cloud])
#text_features(data, [api_key], [cloud])
#emotion(data, [api_key], [cloud], [top_n], [threshold])
#intersections(data, apis, [api_key], [cloud])
#analyze_text(data, apis, [api_key], [cloud])
# batch example
indicoio.analyze_text(
[
"Democratic candidate Hillary Clinton is excited for the upcoming election.",
"Bill Clinton joins President Obama for a birthday golf game at Marthas Vineyard."
],
apis=['sentiment_hq', 'political']
)
# splitting
import indicoio
indicoio.sentiment('This sentence is awful. This sentence is great!', split='sentence')
#custom training
from indicoio.custom import Collection
indicoio.config.api_key = '75b93ed62df0c77a6e58c8ebb1bb71f2'
