def get_text_analysis(self): cur = self.conn.cursor() query = ("SELECT tweet_id, text FROM tweets " "WHERE text_analysis::text='{}'::text;") cur.execute(query) rows = cur.fetchall() for row in rows: text = row[1] text = self.clean_text(text) id = row[0] try: analysis = indicoio.analyze_text( text, apis=['sentiment', 'political', 'keywords', 'text_tags']) json_analysis = json.dumps(analysis) # print("analysing") query = "update tweets set text_analysis=%s where tweet_id=%s;" cur.execute(query, (json_analysis, id)) self.conn.commit() except Exception as ex: template = "An exception of type {0} occurred. Arguments:\n{1!r}" message = template.format(type(ex).__name__, ex.args) print message print "Error with tweet. Id: " + id + " text: " + text if ex.args[0] == 'Input contains one or more empty strings.': query = "delete from tweets where tweet_id='%s';" % id cur.execute(query) self.conn.commit() print('tweet deleted, text= ' + text) print "done indico analysis"
def analyze_tweets(tweet_dict): indico_dict = {} for key in tweet_dict: indico_dict[key] = [] for tweet in tweet_dict[key]: indico_dict[key].append(indicoio.analyze_text(tweet, apis=['sentiment', 'political','twitter_engagement','personality','personas','emotion']))
def analyse_text(): input_text = request.form["text"] user_key = current_talking_user action = request.form.get("action") if action == "political": indico_response = indicoio.analyze_text(input_text, apis=['political']) political_party = indico_response["political"] top_political_party = sorted(political_party.keys(), key=lambda x: political_party[x], reverse=True)[0] return top_political_party else: indico_response = indicoio.analyze_text(input_text, apis=['keywords']) keywords = indico_response["keywords"] keywords_ = (sorted(keywords.keys(), key=lambda x: keywords[x], reverse=True)[:5]) return keywords_
def trendtweets(request, query): print(query) tweets = [] text = [] for tweet in tweepy.Cursor(api.search, q=query + ' -RT', lang="en").items(100): jtweet = {} jtweet['created_at'] = tweet._json['created_at'] jtweet['text'] = tweet._json['text'] text.append(jtweet['text']) #params = {'text': jtweet['text']} #try: #analysis = client.get_request(params, HODApps.ANALYZE_SENTIMENT, async=False) #sentiment = analysis['aggregate']['score'] #except: #continue jtweet['location'] = tweet._json['user']['location'] tweets.append(jtweet) print(len(tweets)) analyze = indicoio.analyze_text(text, apis=['people', 'places', 'organizations'], threshold=0.7) places = analyze['places'] people = analyze['people'] org = analyze['organizations'] print(org) pl = [] ppl = [] og = [] for place in places: if len(place) != 0: pl.append(place[0]['text']) for p in people: if len(p) != 0: ppl.append(p[0]['text']) for o in org: if len(o) != 0: og.append(o[0]['text']) pl = list(set(pl)) ppl = list(set(ppl)) og = list(set(og)) print(pl) print(ppl) print(og) return JsonResponse({ 'tweets': tweets, 'people': ppl, 'places': pl, 'org': og })
def generate_keywords(self): # get keywords keywords_dict = indicoio.keywords(self.tweet) all_else_dict = indicoio.analyze_text( self.tweet, apis=['places', 'people', 'organizations']) # store identified keywords keywords = [] for key, value in keywords_dict.iteritems(): if value > 0.2: keywords.append(key) for key, value in all_else_dict.iteritems(): for list_item in value: for key_in, value_in in list_item.iteritems(): if key_in == 'text': keywords.append(value_in) return keywords
def analyse_text(text, list_algo_fv): """ Method that analyses a text based on given set of algorithms. :param text: Text to be analysed. :param list_algo_fv: List of 'feedvay' algorithms enums :return: Data dictionary containing algo_key as key and result as value. **Exceptions**: Throws ``KeyError`` exception if there are any invalid keys that do not match according to ``TextAlgorithms.mapping_feedvay_indicio``. **Authors**: Gagandeep Singh """ # --- Validation --- # Validate text if not (isinstance(text, str) or isinstance(text, unicode)): raise TypeError("text must a string or unicode.") # Validate algo keys and transform to indicoio keys list_algo_keys = [ TextAlgorithms.mapping_feedvay_indicio[algo_key_fv] for algo_key_fv in list_algo_fv ] # --- /Validation --- # Call service in bulk result_indico = indicoio.analyze_text(text, apis=list_algo_keys, api_key=settings.INDICOIO_API_KEY) # Formalize results map_indicoio_feedvay = TextAlgorithms.get_mapping_indicio_feedvay() result = {} for key_indico, output in result_indico.iteritems(): if key_indico == 'sentiment': output = TextSentimentAnalysis.formalize_results(output) result[map_indicoio_feedvay[key_indico]] = output return result
def tweet(self, text): if len(text) > 140: print("Sorry, but your tweet is more than 140 characters. Try a shorter one.") return tts.speak("Analyzing text...") analysis = indicoio.analyze_text( text, apis=['sentiment_hq', 'political', 'twitter_engagement']) tts.speak("Okay, let's take a look at the results.") print_goodness_message( analysis["sentiment_hq"], "Nice positivity!", "Emotion seems chill.", "That seems a little negative.", "That seems pretty negative.") if analysis["sentiment_hq"] <= 0.2: tts.speak("Hey, do you want to talk about this?") print_goodness_message( analysis["twitter_engagement"], "I suspect that this tweet'll be pretty popular.", "This'll probably be somewhat popular.", "I'm not sure if this'll appeal to many people.", "I think this one will be unpopular.") political_bias = get_political_bias(analysis["political"]) if political_bias[1] >= 0.40: tts.speak("Seems that your tweet has a high {} bias.".format(political_bias[0])) if False: return False else: # Tweet it! self.api.update_status(text) tts.speak("And tweeted! Thanks.") return True
def choose_post(response_json): posts = [] posts_messages = [] data = response_json['data'] for post in data: if 'message' not in post: continue likes = 0 if 'likes' in post: likes = len(post['likes']) posts.append(Post(post['id'], post['created_time'],post['message'], likes, None)) posts_messages.append(post['message']) apis = ['sentiment_hq', 'text_tags', 'personality'] analyses = indicoio.analyze_text(posts_messages, apis=apis) posts_list = [] #for analysis_type, vals in analyses.iteritems(): for post, text_tags, sentiment_hq, personality in itertools.izip(posts, *analyses.values()): likes = post.likes sentiment = sentiment_hq tags = text_tags personalities = personality # Sentiment analysis debugging # top_tags = sorted(tags.items(), key=operator.itemgetter(1), reverse=True) # top_tags = top_tags[:10] # print '\n==========================================================' # print post.message # for key, val in top_tags: print "%s: %0.5f" % (key, val) posts_list.append(Post(post.id, post.date, post.message, post.likes, score_post(likes, sentiment, tags, personalities))) posts_list.sort(key=get_score, reverse=True) return random.choice(posts_list[:len(posts_list)/2])
def analyze(query): tweets = get_twitter_data(query) tweet_text = [tweet.text for tweet in tweets] num_tweets = len(tweets) apis = ['sentiment', 'political'] analysis = indicoio.analyze_text(tweet_text, apis) # print(analysis['sentiment']) avg_sentiment = sum(analysis['sentiment']) / num_tweets avg_libertarian = sum([p['Libertarian'] for p in analysis['political']]) / num_tweets avg_green = sum([p['Green'] for p in analysis['political']]) / num_tweets avg_liberal = sum([p['Liberal'] for p in analysis['political']]) / num_tweets avg_conservative = sum([p['Conservative'] for p in analysis['political']]) / num_tweets values = (num_tweets, avg_sentiment, avg_libertarian, avg_green, avg_liberal, avg_conservative) return avg_sentiment
def analyze(query): tweets = get_twitter_data(query) tweet_text = [tweet.text for tweet in tweets] num_tweets = len(tweets) apis = ['sentiment', 'political'] analysis = indicoio.analyze_text(tweet_text, apis) # print(analysis['sentiment']) avg_sentiment = sum(analysis['sentiment']) / num_tweets avg_libertarian = sum([p['Libertarian'] for p in analysis['political']]) / num_tweets avg_green = sum([p['Green'] for p in analysis['political']]) / num_tweets avg_liberal = sum([p['Liberal'] for p in analysis['political']]) / num_tweets avg_conservative = sum([p['Conservative'] for p in analysis['political']]) / num_tweets print(str(num_tweets) + ' tweets for search "' + query + '"') print('Average sentiment: ' + str(avg_sentiment)) print('Average libertarian: ' + str(avg_libertarian)) print('Average green: ' + str(avg_green)) print('Average liberal: ' + str(avg_liberal)) print('Average conservative: ' + str(avg_conservative))
def test_multi_api_text(self): test_data = "clearly an english sentence" response = analyze_text(test_data) self.assertTrue(isinstance(response, dict)) self.assertTrue(set(response.keys()) <= set(TEXT_APIS.keys()))
def analyze_tweets(tweet_dict): print('done organizing tweets, analyzing now') indico_dict = {} tweet_batches = {} for key in tweet_dict: tweet_batches[key] = [] for tweet in tweet_dict[key]: tweet_batches[key].append(tweet) indicoio.config.api_key = key1 print('key 1') for month in tweet_batches: indico_dict[month] = [] returndict = (indicoio.analyze_text(tweet_batches[month], apis=['sentiment','twitter_engagement','political'])) indico_dict[month].append({'sentiment' : sum(returndict['sentiment'])/len(returndict['sentiment'])}) indico_dict[month].append({'twitter_engagement' : sum(returndict['twitter_engagement'])/len(returndict['twitter_engagement'])}) indico_dict[month].append({'political': {'Libertarian': 0, 'Liberal': 0, 'Green': 0, 'Conservative': 0}}) for mydict in returndict['political']: indico_dict[month][2]['political']['Libertarian'] += mydict['Libertarian'] indico_dict[month][2]['political']['Liberal'] += mydict['Liberal'] indico_dict[month][2]['political']['Green'] += mydict['Green'] indico_dict[month][2]['political']['Conservative'] += mydict['Conservative'] indico_dict[month][2]['political']['Libertarian'] = indico_dict[month][2]['political']['Libertarian']/len(returndict['political']) indico_dict[month][2]['political']['Liberal'] = indico_dict[month][2]['political']['Liberal']/len(returndict['political']) indico_dict[month][2]['political']['Green'] = indico_dict[month][2]['political']['Green']/len(returndict['political']) indico_dict[month][2]['political']['Conservative'] = indico_dict[month][2]['political']['Conservative']/len(returndict['political']) indicoio.config.api_key = key2 print('key 2') for month in tweet_batches: returndict = (indicoio.analyze_text(tweet_batches[month], apis=['personality','personas','emotion'])) indico_dict[month].append({'personality': {'openness': 0, 'extraversion': 0, 'agreeableness': 0, 'conscientiousness': 0}}) for mydict in returndict['personality']: indico_dict[month][3]['personality']['openness'] += mydict['openness'] indico_dict[month][3]['personality']['extraversion'] += mydict['extraversion'] indico_dict[month][3]['personality']['agreeableness'] += mydict['agreeableness'] indico_dict[month][3]['personality']['conscientiousness'] += mydict['conscientiousness'] indico_dict[month][3]['personality']['openness'] = indico_dict[month][3]['personality']['openness']/len(returndict['personality']) indico_dict[month][3]['personality']['extraversion'] = indico_dict[month][3]['personality']['extraversion']/len(returndict['personality']) indico_dict[month][3]['personality']['agreeableness'] = indico_dict[month][3]['personality']['agreeableness']/len(returndict['personality']) indico_dict[month][3]['personality']['conscientiousness'] = indico_dict[month][3]['personality']['conscientiousness']/len(returndict['personality']) indico_dict[month].append({'emotion': {'anger': 0, 'joy': 0, 'fear': 0, 'sadness': 0, 'surprise' : 0}}) for mydict in returndict['emotion']: indico_dict[month][4]['emotion']['anger'] += mydict['anger'] indico_dict[month][4]['emotion']['joy'] += mydict['joy'] indico_dict[month][4]['emotion']['fear'] += mydict['fear'] indico_dict[month][4]['emotion']['sadness'] += mydict['sadness'] indico_dict[month][4]['emotion']['surprise'] += mydict['surprise'] indico_dict[month][4]['emotion']['anger'] = indico_dict[month][4]['emotion']['anger']/len(returndict['emotion']) indico_dict[month][4]['emotion']['joy'] = indico_dict[month][4]['emotion']['joy']/len(returndict['emotion']) indico_dict[month][4]['emotion']['fear'] = indico_dict[month][4]['emotion']['fear']/len(returndict['emotion']) indico_dict[month][4]['emotion']['sadness'] = indico_dict[month][4]['emotion']['sadness']/len(returndict['emotion']) indico_dict[month][4]['emotion']['surprise'] = indico_dict[month][4]['emotion']['surprise']/len(returndict['emotion']) indico_dict[month].append({'personas': {'advocate': 0, 'debater': 0, 'mediator': 0, 'consul': 0, 'executive' : 0, 'adventurer' : 0, 'logistician' : 0, 'commander' : 0, 'entrepreneur' : 0, 'logician' : 0, 'protagonist' : 0, 'architect' : 0, 'campaigner' : 0, 'entertainer' : 0, 'defender' : 0, 'virtuoso' : 0}}) for mydict in returndict['personas']: indico_dict[month][5]['personas']['advocate'] += mydict['advocate'] indico_dict[month][5]['personas']['debater'] += mydict['debater'] indico_dict[month][5]['personas']['mediator'] += mydict['mediator'] indico_dict[month][5]['personas']['consul'] += mydict['consul'] indico_dict[month][5]['personas']['executive'] += mydict['executive'] indico_dict[month][5]['personas']['adventurer'] += mydict['adventurer'] indico_dict[month][5]['personas']['logistician'] += mydict['logistician'] indico_dict[month][5]['personas']['commander'] += mydict['commander'] indico_dict[month][5]['personas']['entrepreneur'] += mydict['entrepreneur'] indico_dict[month][5]['personas']['logician'] += mydict['logician'] indico_dict[month][5]['personas']['protagonist'] += mydict['protagonist'] indico_dict[month][5]['personas']['architect'] += mydict['architect'] indico_dict[month][5]['personas']['campaigner'] += mydict['campaigner'] indico_dict[month][5]['personas']['entertainer'] += mydict['entertainer'] indico_dict[month][5]['personas']['defender'] += mydict['defender'] indico_dict[month][5]['personas']['virtuoso'] += mydict['virtuoso'] indico_dict[month][5]['personas']['advocate'] = indico_dict[month][5]['personas']['advocate']/len(returndict['personas']) indico_dict[month][5]['personas']['debater'] = indico_dict[month][5]['personas']['debater']/len(returndict['personas']) indico_dict[month][5]['personas']['mediator'] = indico_dict[month][5]['personas']['mediator']/len(returndict['personas']) indico_dict[month][5]['personas']['consul'] = indico_dict[month][5]['personas']['consul']/len(returndict['personas']) indico_dict[month][5]['personas']['executive'] = indico_dict[month][5]['personas']['executive']/len(returndict['personas']) indico_dict[month][5]['personas']['adventurer'] = indico_dict[month][5]['personas']['adventurer']/len(returndict['personas']) indico_dict[month][5]['personas']['logistician'] = indico_dict[month][5]['personas']['logistician']/len(returndict['personas']) indico_dict[month][5]['personas']['commander'] = indico_dict[month][5]['personas']['commander']/len(returndict['personas']) indico_dict[month][5]['personas']['entrepreneur'] = indico_dict[month][5]['personas']['entrepreneur']/len(returndict['personas']) indico_dict[month][5]['personas']['logician'] = indico_dict[month][5]['personas']['logician']/len(returndict['personas']) indico_dict[month][5]['personas']['protagonist'] = indico_dict[month][5]['personas']['protagonist']/len(returndict['personas']) indico_dict[month][5]['personas']['architect'] = indico_dict[month][5]['personas']['architect']/len(returndict['personas']) indico_dict[month][5]['personas']['campaigner'] = indico_dict[month][5]['personas']['campaigner']/len(returndict['personas']) indico_dict[month][5]['personas']['entertainer'] = indico_dict[month][5]['personas']['entertainer']/len(returndict['personas']) indico_dict[month][5]['personas']['defender'] = indico_dict[month][5]['personas']['defender']/len(returndict['personas']) indico_dict[month][5]['personas']['virtuoso'] = indico_dict[month][5]['personas']['virtuoso']/len(returndict['personas']) return indico_dict
def test_multi_api_text(self): test_data = 'clearly an english sentence' response = analyze_text(test_data, apis=config.TEXT_APIS, api_key=self.api_key) self.assertTrue(isinstance(response, dict)) self.assertTrue(set(response.keys()) == set(config.TEXT_APIS))
def setData(): currentData = json.dumps(indicoio.analyze_text(str(request.form['politics']), apis=['political', 'sentiment_hq', 'keywords'])) data.append(currentData) sentences.append(request.form['politics']) return str(currentData)
def processText(): text = request.form['text'] r = indicoio.analyze_text(text, apis=['text_tags', 'sentiment_hq', 'keywords']); app.logger.debug(r) return json.jsonify(**r);
def analyze_tweets(tweet_dict): print('done organizing tweets, analyzing now') indico_dict = {} tweet_batches = {} for key in tweet_dict: tweet_batches[key] = [] for tweet in tweet_dict[key]: tweet_batches[key].append(tweet) for month in tweet_batches: indico_dict[month] = [] returndict = (indicoio.analyze_text(tweet_batches[month], apis=['sentiment','twitter_engagement','political','personality','personas','emotion'])) indico_dict[month].append({'sentiment' : sum(returndict['sentiment'])/len(returndict['sentiment'])}) indico_dict[month].append({'twitter_engagement' : sum(returndict['twitter_engagement'])/len(returndict['twitter_engagement'])}) indico_dict[month].append({'political': {'Libertarian': 0, 'Liberal': 0, 'Green': 0, 'Conservative': 0}}) for mydict in returndict['political']: indico_dict[month][2]['political']['Libertarian'] += mydict['Libertarian'] indico_dict[month][2]['political']['Liberal'] += mydict['Liberal'] indico_dict[month][2]['political']['Green'] += mydict['Green'] indico_dict[month][2]['political']['Conservative'] += mydict['Conservative'] indico_dict[month][2]['political']['Libertarian'] = indico_dict[month][2]['political']['Libertarian']/len(returndict['political']) indico_dict[month][2]['political']['Liberal'] = indico_dict[month][2]['political']['Liberal']/len(returndict['political']) indico_dict[month][2]['political']['Green'] = indico_dict[month][2]['political']['Green']/len(returndict['political']) indico_dict[month][2]['political']['Conservative'] = indico_dict[month][2]['political']['Conservative']/len(returndict['political']) indico_dict[month].append({'personality': {'openness': 0, 'extraversion': 0, 'agreeableness': 0, 'conscientiousness': 0}}) for mydict in returndict['personality']: indico_dict[month][3]['personality']['openness'] += mydict['openness'] indico_dict[month][3]['personality']['extraversion'] += mydict['extraversion'] indico_dict[month][3]['personality']['agreeableness'] += mydict['agreeableness'] indico_dict[month][3]['personality']['conscientiousness'] += mydict['conscientiousness'] indico_dict[month][3]['personality']['openness'] = indico_dict[month][3]['personality']['openness']/len(returndict['personality']) indico_dict[month][3]['personality']['extraversion'] = indico_dict[month][3]['personality']['extraversion']/len(returndict['personality']) indico_dict[month][3]['personality']['agreeableness'] = indico_dict[month][3]['personality']['agreeableness']/len(returndict['personality']) indico_dict[month][3]['personality']['conscientiousness'] = indico_dict[month][3]['personality']['conscientiousness']/len(returndict['personality']) indico_dict[month].append({'emotion': {'anger': 0, 'joy': 0, 'fear': 0, 'sadness': 0, 'surprise' : 0}}) for mydict in returndict['emotion']: indico_dict[month][4]['emotion']['anger'] += mydict['anger'] indico_dict[month][4]['emotion']['joy'] += mydict['joy'] indico_dict[month][4]['emotion']['fear'] += mydict['fear'] indico_dict[month][4]['emotion']['sadness'] += mydict['sadness'] indico_dict[month][4]['emotion']['surprise'] += mydict['surprise'] indico_dict[month][4]['emotion']['anger'] = indico_dict[month][4]['emotion']['anger']/len(returndict['emotion']) indico_dict[month][4]['emotion']['joy'] = indico_dict[month][4]['emotion']['joy']/len(returndict['emotion']) indico_dict[month][4]['emotion']['fear'] = indico_dict[month][4]['emotion']['fear']/len(returndict['emotion']) indico_dict[month][4]['emotion']['sadness'] = indico_dict[month][4]['emotion']['sadness']/len(returndict['emotion']) indico_dict[month][4]['emotion']['surprise'] = indico_dict[month][4]['emotion']['surprise']/len(returndict['emotion']) indico_dict[month].append({'personas': {'advocate': 0, 'debater': 0, 'mediator': 0, 'consul': 0, 'executive' : 0, 'adventurer' : 0, 'logistician' : 0, 'commander' : 0, 'entrepreneur' : 0, 'logician' : 0, 'protagonist' : 0, 'architect' : 0, 'campaigner' : 0, 'entertainer' : 0, 'defender' : 0, 'virtuoso' : 0}}) for mydict in returndict['personas']: indico_dict[month][5]['personas']['advocate'] += mydict['advocate'] indico_dict[month][5]['personas']['debater'] += mydict['debater'] indico_dict[month][5]['personas']['mediator'] += mydict['mediator'] indico_dict[month][5]['personas']['consul'] += mydict['consul'] indico_dict[month][5]['personas']['executive'] += mydict['executive'] indico_dict[month][5]['personas']['adventurer'] += mydict['adventurer'] indico_dict[month][5]['personas']['logistician'] += mydict['logistician'] indico_dict[month][5]['personas']['commander'] += mydict['commander'] indico_dict[month][5]['personas']['entrepreneur'] += mydict['entrepreneur'] indico_dict[month][5]['personas']['logician'] += mydict['logician'] indico_dict[month][5]['personas']['protagonist'] += mydict['protagonist'] indico_dict[month][5]['personas']['architect'] += mydict['architect'] indico_dict[month][5]['personas']['campaigner'] += mydict['campaigner'] indico_dict[month][5]['personas']['entertainer'] += mydict['entertainer'] indico_dict[month][5]['personas']['defender'] += mydict['defender'] indico_dict[month][5]['personas']['virtuoso'] += mydict['virtuoso'] indico_dict[month][5]['personas']['advocate'] = indico_dict[month][5]['personas']['advocate']/len(returndict['personas']) indico_dict[month][5]['personas']['debater'] = indico_dict[month][5]['personas']['debater']/len(returndict['personas']) indico_dict[month][5]['personas']['mediator'] = indico_dict[month][5]['personas']['mediator']/len(returndict['personas']) indico_dict[month][5]['personas']['consul'] = indico_dict[month][5]['personas']['consul']/len(returndict['personas']) indico_dict[month][5]['personas']['executive'] = indico_dict[month][5]['personas']['executive']/len(returndict['personas']) indico_dict[month][5]['personas']['adventurer'] = indico_dict[month][5]['personas']['adventurer']/len(returndict['personas']) indico_dict[month][5]['personas']['logistician'] = indico_dict[month][5]['personas']['logistician']/len(returndict['personas']) indico_dict[month][5]['personas']['commander'] = indico_dict[month][5]['personas']['commander']/len(returndict['personas']) indico_dict[month][5]['personas']['entrepreneur'] = indico_dict[month][5]['personas']['entrepreneur']/len(returndict['personas']) indico_dict[month][5]['personas']['logician'] = indico_dict[month][5]['personas']['logician']/len(returndict['personas']) indico_dict[month][5]['personas']['protagonist'] = indico_dict[month][5]['personas']['protagonist']/len(returndict['personas']) indico_dict[month][5]['personas']['architect'] = indico_dict[month][5]['personas']['architect']/len(returndict['personas']) indico_dict[month][5]['personas']['campaigner'] = indico_dict[month][5]['personas']['campaigner']/len(returndict['personas']) indico_dict[month][5]['personas']['entertainer'] = indico_dict[month][5]['personas']['entertainer']/len(returndict['personas']) indico_dict[month][5]['personas']['defender'] = indico_dict[month][5]['personas']['defender']/len(returndict['personas']) indico_dict[month][5]['personas']['virtuoso'] = indico_dict[month][5]['personas']['virtuoso']/len(returndict['personas']) return indico_dict import scrape def parse_tweets(screen_name): scrape.get_all_tweets(screen_name) tweetlist = [] f = open('%s.txt' % screen_name, 'r') for line in f: lines = line.split('\t') lines[0] = lines[0][:-9] lines[1] = lines[1][2:-2] tweetlist.append(lines) f.close() return tweetlist def organize_tweets(tweets): for tweet in tweets: if tweet[0][5:7] == '01': tweet[0] = 'January' + str(tweet[0][0:4]) elif tweet[0][5:7] == '02': tweet[0] = 'February' + str(tweet[0][0:4]) elif tweet[0][5:7] == '03': tweet[0] = 'March' + str(tweet[0][0:4]) elif tweet[0][5:7] == '04': tweet[0] = 'April' + str(tweet[0][0:4]) elif tweet[0][5:7] == '05': tweet[0] = 'May' + str(tweet[0][0:4]) elif tweet[0][5:7] == '06': tweet[0] = 'June' + str(tweet[0][0:4]) elif tweet[0][5:7] == '07': tweet[0] = 'July' + str(tweet[0][0:4]) elif tweet[0][5:7] == '08': tweet[0] = 'August' + str(tweet[0][0:4]) elif tweet[0][5:7] == '09': tweet[0] = 'September' + str(tweet[0][0:4]) elif tweet[0][5:7] == '10': tweet[0] = 'October' + str(tweet[0][0:4]) elif tweet[0][5:7] == '11': tweet[0] = 'November' + str(tweet[0][0:4]) elif tweet[0][5:7] == '12': tweet[0] = 'December' + str(tweet[0][0:4]) tweet_dict={} for tweet in tweets: if tweet[0] not in tweet_dict: tweet_dict[tweet[0]] = [tweet[1]] else: tweet_dict[tweet[0]].append(tweet[1]) return tweet_dict if __name__ == '__main__': #pass in the username of the account you want to download organize_tweets(parse_tweets("HackSentivents")) import tweepy #https://github.com/tweepy/tweepy #Twitter API credentials consumer_key = 'XuJqupkF9ZUq8oIz921Nl141O' consumer_secret = '9z0JyLdMNRT7brH14wh32fUqxbpL0yDXu70Ez8mJFpLifmv3nl' access_key = '853038411426521090-yviLNx5KWrXE7lXrubdjKlwOQ0WoC8R' access_secret = 'wCxcNrBfJi2PhTVw9Xgi1DF4EFb1Z38cS7z0PAgC3Jd6D' def get_all_tweets(screen_name): #Twitter only allows access to a users most recent 3240 tweets with this method #authorize twitter, initialize tweepy auth = tweepy.OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) api = tweepy.API(auth) #initialize a list to hold all the tweepy Tweets alltweets = [] #make initial request for most recent tweets (200 is the maximum allowed count) new_tweets = api.user_timeline(screen_name = screen_name,count=200) #save most recent tweets alltweets.extend(new_tweets) #save the id of the oldest tweet less one oldest = alltweets[-1].id - 1 #keep grabbing tweets until there are no tweets left to grab while len(new_tweets) > 0: #print "getting tweets before %s" % (oldest) #all subsiquent requests use the max_id param to prevent duplicates new_tweets = api.user_timeline(screen_name = screen_name,count=200,max_id=oldest) #save most recent tweets alltweets.extend(new_tweets) #update the id of the oldest tweet less one oldest = alltweets[-1].id - 1 #print "...%s tweets downloaded so far" % (len(alltweets)) #transform the tweepy tweets into a 2D array that will populate the csv outtweets = [[tweet.id_str, tweet.created_at, tweet.text.encode("utf-8")] for tweet in alltweets] #return outtweets #write the csv with open('%s.txt' % screen_name, 'w') as f: for i in outtweets: f.write(str(i[1])+'\t') f.write(str(i[2])+'\n') pass if __name__ == '__main__': #pass in the username of the account you want to download get_all_tweets("HackSentivents")
def test_default_multi_api_text(self): test_data = ['clearly an english sentence'] response = analyze_text(test_data) self.assertTrue(isinstance(response, dict)) self.assertTrue(set(response.keys()) <= set(config.TEXT_APIS))
print("Skipping " + filename + "...") visited = True break if visited == False: for file in output_dataframe['subreddit'].str.contains(filename): if file == filename: print("Yes!") print("Adding " + filename + "...") path = directory + '/' + filename with open(path, 'r') as myfile: data = myfile.read() response = indicoio.analyze_text(data, apis=[ 'sentiment', 'emotion', 'personality', 'twitter_engagement' ]) # Build data dictionary d = OrderedDict() d['sentiment'] = response['sentiment'] d.update(response['emotion']) d.update(response['personality']) # d.update(response['twitter_engagement']) d['twitter_engagement'] = response['twitter_engagement'] # Convert to DataFrame columns = list(d.keys()) values = list(d.values()) arr_len = len(values)
def test_multi_api_text(self): test_data = 'clearly an english sentence' response = analyze_text(test_data) self.assertTrue(isinstance(response, dict)) self.assertTrue(set(response.keys()) <= set(TEXT_APIS.keys()))
#organizations(data, [api_key], [cloud], [threshold]) #twitter_engagement(data, [api_key], [cloud]) #personality(data, [api_key], [cloud]) #relevance(data, queries, [api_key], [cloud]) #text_features(data, [api_key], [cloud]) #emotion(data, [api_key], [cloud], [top_n], [threshold]) #intersections(data, apis, [api_key], [cloud]) #analyze_text(data, apis, [api_key], [cloud]) # batch example indicoio.analyze_text( [ "Democratic candidate Hillary Clinton is excited for the upcoming election.", "Bill Clinton joins President Obama for a birthday golf game at Marthas Vineyard." ], apis=['sentiment_hq', 'political'] ) # splitting import indicoio indicoio.sentiment('This sentence is awful. This sentence is great!', split='sentence') #custom training from indicoio.custom import Collection indicoio.config.api_key = '75b93ed62df0c77a6e58c8ebb1bb71f2'