def save_tweets(tweets, search_term, location_search_term=None): """ Saves given tweet data in MongoDB database. :param tweets: Tweet data :param search_term: Search term entered by user :param location_search_term: Location search term entered by user (optional) """ try: if location_search_term: # Cycle through list of Tweets for status in tweets: tweet_id = status['tweet_id'] # Check if tweet already exists in db, skip this iteration if it does id_query = len(Tweet.objects(tweet_id=tweet_id)) if id_query >= 1: continue # Define record to save to db record = Tweet( tweet_id=status['tweet_id'], tweet_time=status['created_at'], tweet_text=status['text'], tweet_user=status['user'], tweet_user_fullname=status['user_fullname'], profile_image_url=status['profile_image_url'], sentiment_type=status['sentiment'], sentiment_score=status['sentimentScore'], location_geo=status['location_geo'], location_address=status['location_address'], keyword_search_term=search_term, location_search_term=location_search_term ) # Save to DB record.save() else: # Cycle through list of processed Tweets for status in tweets: tweet_id = status['tweet_id'] # Check if tweet already exists in db, skip this iteration if it does id_query = len(Tweet.objects(tweet_id=tweet_id)) if id_query >= 1: continue # Define record to save to db record = Tweet( tweet_id=status['tweet_id'], tweet_time=status['created_at'], tweet_text=status['text'], tweet_user=status['user'], tweet_user_fullname=status['user_fullname'], profile_image_url=status['profile_image_url'], sentiment_type=status['sentiment'], sentiment_score=status['sentimentScore'], keyword_search_term=search_term ) # Save to DB record.save() except Exception: raise Exception("Database error")
def search_keyword(keyword, count, location=None): """ Performs a Twitter search for a given keyword. Will restrict keyword search to within 10 miles radius of given location (if provided). :param keyword: Keyword to search :param location: Location object from geopy to restrict results to (optional) :param count: Number of Tweets to search for :return: List of dictionaries containing parsed Tweet data from search """ # Perform OAuth connection to Twitter, creates instance of Twython twitter = twitter_auth() is_location_search = False in_db = False # List to store our data tweets = [] # Attempt to query Twitter REST API using Twython try: if location is None: search_result = twitter.search(q=keyword, lang="en", count=count) else: is_location_search = True search_result = twitter.search(q=keyword, lang="en", geocode="%s,%s,10mi" % (location.latitude, location.longitude), count=count) except Exception: raise Exception("No Twitter results returned") # Cycle through results and parse the data we want, save as dictionary and store in 'tweets' list for status in search_result['statuses']: tweet_id = status['id'] # Check if Tweet is already in DB id_query = len(Tweet.objects(tweet_id=tweet_id)) if id_query >= 1: in_db = True continue tweet = {} tweet['tweet_id'] = tweet_id tweet['text'] = status['text'].strip() tm = status['created_at'] tm = datetime.strptime(tm, '%a %b %d %H:%M:%S +0000 %Y') tweet['created_at'] = tm tweet['user'] = status['user']['screen_name'].strip() tweet['user_fullname'] = status['user']['name'] tweet['profile_image_url'] = status['user']['profile_image_url'] if is_location_search: tweet['location_geo'] = {"latitude": location.latitude, "longitude": location.longitude} tweet['location_address'] = location.address # Append parsed tweet data dictionary to results list tweets.append(tweet) # If no search results returned and data is not in our database, raise exception if not tweets and not in_db: raise Exception("No Twitter results returned") return tweets
def delete(self, username): context = self.get_context(username) muse = context.get('muse') # Fetch and clear out this user's tweets. tweets = Tweet.objects(username=username) for tweet in tweets: tweet.delete() muse.delete() return jsonify({'success':True})
def delete(self, username): context = self.get_context(username) muse = context.get('muse') # Fetch and clear out this user's tweets. tweets = Tweet.objects(username=username) for tweet in tweets: tweet.delete() muse.delete() return jsonify({'success': True})
def search_user(screen_name, count): """ Performs a Twitter search for a given username. :param screen_name: Username of Twitter user to search for :param count: Number of the user's Tweets to return :return: List of dictionaries containing parsed Tweet data from search """ # Perform OAuth connection to Twitter, creates instance of Twython twitter = twitter_auth() in_db = False # List to store our results tweets = [] # Attempt to query Twitter REST API using Twython try: search_result = twitter.get_user_timeline(screen_name=screen_name, count=count) except Exception: raise Exception("No Twitter results returned") if search_result[0]['user']['lang'] != "en": # If not English, we can't perform sentiment analysis (limitation of AlchemyAPI) raise Exception("Not an English language user") # Cycle through results and parse the data we want, save as dictionary and store in 'tweets' list for status in search_result: tweet_id = status['id'] # Check if Tweet is already in DB id_query = len(Tweet.objects(tweet_id=tweet_id)) if id_query >= 1: in_db = True continue tweet = {} tweet['tweet_id'] = tweet_id tweet['text'] = status['text'].strip() tm = status['created_at'] tm = datetime.strptime(tm, '%a %b %d %H:%M:%S +0000 %Y') tweet['created_at'] = tm tweet['user'] = screen_name[1:] tweet['user_fullname'] = status['user']['name'] tweet['profile_image_url'] = status['user']['profile_image_url'] # Add parsed tweet data dictionary to results list tweets.append(tweet) # If no search results returned and data is not in our database, raise exception if not tweets and not in_db: raise Exception("No Twitter results returned") return tweets
def delete_obsolete_tweets(): max_datetime = datetime.now().date() Tweet.objects(datetime__lt=max_datetime).delete()
def search(): """ Handles post request from index page search form, processes form and calls manager functions accordingly. Passes results to rendered search template, or redirects user to homepage if exception occurs. :return: Rendered template with search results """ from app import manager from app import helper import time page = "search" if request.method == 'POST': # Get form data with trailing spaces removed keyword = request.form['keyword'].strip() count = request.form['count'].strip() location = request.form['location'].strip() # Check if search term has been provided if not keyword: flash("Blessed are the forgetful! Looks like you didn't enter a search parameter.") return redirect(url_for('index')) # If user hasn't specified a number of Tweets to search for, use defualt (15) if not count: count = config.TWEET_SEARCH_LIMIT_DEFAULT # If count provided, check to sure it does not exceed max (100) if int(count) > config.TWEET_SEARCH_LIMIT_MAX: # If exceeds max, set count to max count = config.TWEET_SEARCH_LIMIT_MAX # Check if supplied count is negative if int(count) <= 0: flash("Please enter a valid number of Tweets to search (1-100)") return redirect(url_for('index')) user_search = False location_search_term = None # Get time t1 = time.time() # Attempt to perform search, analysis and storage try: if keyword and not location: keyword = keyword.lower() # Check if searching for a user if keyword[0] == "@": user_search = True manager.analysis_supervisor(manager.search_user(keyword, count), keyword) else: manager.analysis_supervisor(manager.search_keyword(keyword, count), keyword) if keyword and location: keyword = keyword.lower() location = location.lower() # Check if searching for a user if keyword[0] == "@": user_search = True manager.analysis_supervisor(manager.search_user(keyword, count), keyword) else: location_search_term = location location = manager.get_geo_info(location_search_term) manager.analysis_supervisor(manager.search_keyword(keyword, count, location), keyword, location_search_term) except Exception as e: # Exception handling for any errors that may occur in retrieving / analyzing / saving data e = str(e) if e == "No Twitter results returned": if location_search_term: flash("Sorry, Twitter returned no results for: \"" + keyword + "\" near " + "\"" + location_search_term + "\"") else: flash("Sorry, Twitter returned no results for: \"" + keyword + "\"") elif e == "Twython auth error": flash("Oops, it appears we're having trouble connecting to Twitter. Please try again later.") elif e == "AlchemyAPI auth error": flash("Oops, it appears we're having trouble connecting to our language processing API. " "Please try again later.") elif e == "AlchemyAPI error": flash("Oops, it appears we had trouble analyzing one or more of the results for that search") elif e == "Not an English language user": flash("Sorry, it appears that account " + keyword + " is not an English language user. Presently, " "Tweetvibe can only analyze English tweets.") elif e == "Location error": flash("Oops, it appears we had trouble identifying location " + "\"" + location_search_term + "\"") elif e == "Database error": flash("Oops, it appears we are experiencing a problem interacting with our database.") else: flash("Oops, something went wrong. A team of highly trained engineer monkeys have been dispatched to" " fix the problem. Please try again later.") # Redirect to index with flash message return redirect(url_for('index')) # Attempt to get results from database try: if location: results = Tweet.objects(Q(keyword_search_term=keyword) & Q(location_address=location.address)).order_by('-stored_at', '-tweet_time').limit(int(count)) else: results = Tweet.objects(keyword_search_term=keyword).order_by('-stored_at', '-tweet_time').limit(int(count)) except: flash("Oops, it appears we are experiencing a problem querying our database.") return redirect(url_for('index')) try: search_aggregate = helper.aggregate_sentiment(results) search_avg = helper.get_query_sentiment_avg(results) except: flash("Oops, something went wrong. A team of highly trained engineer monkeys have been dispatched to" " fix the problem. Please try again later.") return redirect(url_for('index')) # Calculate time taken to perform search and analysis t2 = time.time() time_taken = t2 - t1 if location: # Format location latitude/longitude to 2 decimal places longitude = "{:.2f}".format(float(location.longitude)) latitude = "{:.2f}".format(float(location.latitude)) hist_avg = helper.get_historical_sentiment_avg(keyword, location.address) hist_data = helper.get_historical_sentiment(keyword, location.address) hist_predominant_sentiment = helper.predominant_sentiment(hist_data) return render_template( "search.html", time_taken=time_taken, results=results, page=page, keyword=keyword, location_search_term=location_search_term, location=location.address, longitude=longitude, latitude=latitude, search_count=count, search_aggregate=search_aggregate, search_avg=search_avg, search_stats=helper.get_query_statistics(results, search_aggregate), dom_sentiment=hist_predominant_sentiment, hist_data=hist_data, hist_avg=hist_avg, overtime_data=helper.get_sentiment_overtime(keyword, location.address) ) elif user_search: hist_avg = helper.get_historical_sentiment_avg(keyword) hist_data = helper.get_historical_sentiment(keyword) hist_predominant_sentiment = helper.predominant_sentiment(hist_data) return render_template( "search.html", results=results, time_taken=time_taken, page=page, user=keyword, search_count=count, search_aggregate=search_aggregate, search_avg=search_avg, search_stats=helper.get_query_statistics(results, search_aggregate), dom_sentiment=hist_predominant_sentiment, hist_data=hist_data, hist_avg=hist_avg, overtime_data=helper.get_sentiment_overtime(keyword) ) else: hist_avg = helper.get_historical_sentiment_avg(keyword) hist_data = helper.get_historical_sentiment(keyword) hist_predominant_sentiment = helper.predominant_sentiment(hist_data) return render_template( "search.html", results=results, time_taken=time_taken, page=page, keyword=keyword, search_count=count, search_aggregate=search_aggregate, search_avg=search_avg, search_stats=helper.get_query_statistics(results, search_aggregate), dom_sentiment=hist_predominant_sentiment, hist_data=hist_data, hist_avg=hist_avg, overtime_data=helper.get_sentiment_overtime(keyword) ) else: return redirect(url_for('index'))