def update_database(): print("fetching") tweets = Tweet.objects(total_fetched=1, error_occured__ne=True) print("updating") i = 0 error_count = 0 for each_tweet in tweets: print("loop") data = update_given_tweet(each_tweet.tweet_id) if not data: error_count += 1 print("!!!!!!!!!error", error_count, "correct", i) each_tweet.error_occured = True each_tweet.save() continue elif data == 3: continue print("got data") data['fetched_timestamp'] = datetime.datetime.now() data['fresh_tweet'] = False data['update_count'] = 2 each_tweet.total_fetched = 2 each_tweet.tweets.append(data) each_tweet.save() print(i, "errors=", error_count) i += 1
def specific_tweet_fact_check(fake_news_url): from mongoengine.queryset.visitor import Q query = Tweet.objects( Q(fake_news_url=fake_news_url) & Q(fact_checked=False)).order_by( '-user__followers_count').values_list('id_str', 'fake_news_url', 'user__screen_name').first() return query
def getTweets(request): tweets_res = [] tweets = Tweet.objects() for tweet in tweets: tweets_res.append({"text":tweet.tweet_text,"time":str(tweet.tweet_time)}) return HttpResponse(json.dumps(tweets_res))
def update_fakenews_db(): urls = FakeNews.objects(fake_news_url__exists=1).values_list( 'fake_news_url', 'tweets') for fake_story, tweets in urls: update_story_db(fake_story) data_object = FakeNews.objects.get(fake_news_url=fake_story) data_object.tweets = Tweet.objects(fake_news_url=fake_story).count() data_object.save()
def post(self, request , *args , **Kwargs): # print "in post method" slack_request_msg = request.data # checking if request coming from correct workspace if slack_request_msg.get('token') != VeriFication_Token : return Response(status = status.HTTP_403_FORBIDDEN) #verification challenge if slack_request_msg.get('type') == 'url_verification' : return Response(data=slack_request_msg , status = status.HTTP_200_OK) # event listening if 'event' in slack_request_msg: msg_event = slack_request_msg.get('event') # check if msg coming from slack api if msg_event.get('subtype') == 'bot_message': return Response(status=status.HTTP_200_OK) # process user's message user = msg_event.get('user') text = msg_event.get('text') channel = msg_event.get('channel') # respond with hello msg bot_text = 'Hi <@{}> :wave:'.format(user) if 'go' in text.lower(): slack_client.api_call(method='chat.postMessage', channel=channel, text=bot_text) # tweet_obj = Tweet(tweet_id=1, tweet_text="Hi Hesham", tweet_time=datetime.datetime.now()) # tweet_obj.save() # x = Tweet.objects() # for t in x : # print t.tweet_id , t.tweet_text # print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" # to get all tweets in the timeline All_Tweets = tweepy.Cursor(api.home_timeline).items() for iter in All_Tweets : newTweet = Tweet.objects(tweet_id=iter.id) # check if this tweet dosen't exist before if not newTweet : tweet_obj = Tweet(tweet_id = iter.id,tweet_text= iter.text,tweet_time = iter.created_at) tweet_obj.save() else : break return Response(status=status.HTTP_200_OK) return Response(status=status.HTTP_200_OK)
def render_GET(self, request): request.setHeader("content-type", "application/json") tweets = Tweet.objects(entities__ne=None) data = [] for tweet in tweets: if re.search("http://pbs.twimg.com/media/", tweet.entities): media_url = ast.literal_eval(tweet.entities)['media'][0]['media_url'] data.append({'image': media_url}) return json.dumps(data)
def tweet_fact_check(): bot = TwitterBot(settings.ACCESS_KEY, settings.ACCESS_SECRET, settings.CONSUMER_KEY, settings.CONSUMER_SECRET) query = Tweet.objects( fact_checked=False).order_by('-user__followers_count').values_list( 'id_str', 'fake_news_url', 'user__screen_name').first() fake_story = FakeNews.objects.get(fake_news_url=query[1]) correct_link = fake_story.shortened_url message = fake_story.message_1 fc_tweet = bot.tweet_factcheck(query[0], query[2], message, correct_link) return query
def GetPastTweets(app, searchStrings): with app.app_context(): auth = OAuthHandler( current_app.config['TWITTER_CONSUMER_KEY'], current_app.config['TWITTER_CONSUMER_SECRET']) auth.set_access_token(current_app.config['TWITTER_ACCESS_TOKEN'], current_app.config['TWITTER_ACCESS_TOKEN_SECRET']) api = API(auth) deleteEntries = connect(current_app.config['MONGODB_SETTINGS']['DB']) deleteEntries.drop_database(current_app.config['MONGODB_SETTINGS']['DB']) listTweetIDs = [] user_query = session['userSentence'] session['completeTweetFetch'] = False session['completedMetaModel'] = False for searchString in searchStrings: # Sleep for 28 secs to avoid Twitter download rate restrictions searchTweets = [status for status in Cursor(api.search, q=searchString).items(current_app.config['MAX_FETCH_TWEETS'])] for tweet in searchTweets: tweet_id = tweet.id if listTweetIDs.__contains__(tweet_id): continue if len(Tweet.objects(tweet_id=tweet_id)) > 1: continue tweet_message = tweet.text.encode("utf-8") tweet_userhandle = tweet.user.screen_name tweet_retweet_count = tweet.retweet_count tweet_createtime = tweet.created_at tweet_location = None tweet_geo = None tweet_favoritecount = tweet.favorite_count tweet_username = tweet.user.name tweet_user_no_of_following = tweet.user.friends_count tweet_user_no_of_followers = tweet.user.followers_count tweet_positiveOrnegative = 0 tweet_polarOrneutral = 0 tweet_isRetweet = 0 oneTweet = Tweet(tweet_id=tweet_id, tweet_msg=tweet_message, tweet_likes=tweet_favoritecount, tweet_retweets=tweet_retweet_count, tweet_search_category=searchString, tweet_user_search_query=user_query, tweet_positiveOrnegative=tweet_positiveOrnegative, tweet_polarOrneutral=tweet_polarOrneutral, tweet_user_handle=tweet_userhandle, tweet_user_name=tweet_username, tweet_user_followers=tweet_user_no_of_followers, tweet_user_following=tweet_user_no_of_following, tweet_isretweet=tweet_isRetweet, tweet_time=tweet_createtime, tweet_location=tweet_location, tweet_geo=tweet_geo) oneTweet.save() listTweetIDs.append(tweet_id) if session['metamodelThread'] is None: session['metamodelThread'] = thread.start_new_thread(twittermetamodelBuilding, ()) print 'completed tweet fetch' session['completeTweetFetch'] = True pass
def update_story_db(url): bot = TwitterBot(settings.ACCESS_KEY, settings.ACCESS_SECRET, settings.CONSUMER_KEY, settings.CONSUMER_SECRET) bot.search(url) total_added = Tweet.objects(fake_news_url=url).count() print(f'{total_added} tweets mentioning {url} added.')
def twittermetamodelBuilding(user_id): ''' This function will be used to build the metamodel of the twitter data. This basically develops all the background data needed for the visualization, using Tweet data stored in DB. :return: - ''' previouscount = 0 userSearchObj = user_searches_ongoing[user_id] global TWITTER_DATA_DOWNLOADER_THREAD_INDEX global META_MODEL_BUILDER_THREAD_INDEX while(True): if userSearchObj.meta_model_complete and userSearchObj.tweet_fetch_complete: time.sleep(2) continue print userSearchObj.user_search_sentence, "From metamodel building" list_tweets = [] user_tweets = {} hashtag_rel = {} list_tweets = Tweet.objects(tweet_user_search_query=userSearchObj.user_search_sentence) tweet_msgs_lst = [] no_of_retweets = 0 no_of_likes = 0 total_posCount = 0 total_negCount = 0 metadata = {} for tweet in list_tweets: new_user = False no_of_retweets += tweet.tweet_retweets no_of_likes += tweet.tweet_likes posCount = 0 negCount = 0 if senana.classifier.classify(senana.feature_extractor(tweet.tweet_msg.split())) == 'positive': tweet.tweet_positiveOrnegative = 1 posCount = 1 total_posCount += 1 else: tweet.tweet_positiveOrnegative = 0 negCount = 1 total_negCount += 1 if user_tweets.has_key(tweet.tweet_user_handle): user = user_tweets[tweet.tweet_user_handle] user.tweet_likes += tweet.tweet_likes user.no_of_retweets += tweet.tweet_retweets user.positiveCount += posCount user.negativeCount += negCount else: user = Tweet_User(tweet.tweet_user_handle, tweet.tweet_user_name, tweet.tweet_user_following, tweet.tweet_user_followers, tweet.tweet_likes, tweet.tweet_retweets, posCount, negCount, 0, 0) user_tweets.__setitem__(tweet.tweet_user_handle, user) new_user = True tweet_msgs_lst.append(tweet.tweet_msg) words = tweet.tweet_msg.split(' ') hashtags_lst = [] for word in words: if word.__len__() > 1 and word[0] == '#': hashtag_string = word[1:] if not re.match("^[a-zA-Z0-9]*$", hashtag_string): continue hashtag_string = hashtag_string.lower() if hashtag_rel.has_key(hashtag_string): hashtag_obj = hashtag_rel[hashtag_string] hashtag_obj.no_of_retweets += tweet.tweet_retweets hashtag_obj.no_of_likes += tweet.tweet_likes hashtag_obj.tweets.append(tweet) if new_user: hashtag_obj.no_of_users += 1 hashtag_obj.negativeCount += negCount hashtag_obj.positiveCount += posCount else: hashtag_obj = Twitter_Hashtag(hashtag_string, tweet.tweet_retweets, tweet.tweet_likes, 1, posCount, negCount, 0, 0) hashtag_obj.tweets.append(tweet) hashtag_rel.__setitem__(hashtag_string, hashtag_obj) if not hashtags_lst.__contains__(hashtag_string): hashtags_lst.append(hashtag_string) if not user.hashtagsUsed.has_key(hashtag_string): user.hashtagsUsed.__setitem__(hashtag_string, hashtag_obj) # print hashtags_lst for hashtag in hashtags_lst: hashtag_obj = hashtag_rel[hashtag] hashtag_obj.hashtag_rank = hashtag_rank_calculator(hashtag_obj.tweets.__len__(), hashtag_obj.no_of_users, hashtag_obj.no_of_retweets, hashtag_obj.no_of_likes) for otherhashtag in hashtags_lst: if otherhashtag == hashtag: continue if not hashtag_obj.related_hashtags.has_key(otherhashtag): hashtag_obj.related_hashtags.__setitem__(otherhashtag, hashtag_rel[otherhashtag]) full_hashtags = '' topfiveHashtags = [] for hashtag in hashtag_rel.keys(): # full_hashtags = full_hashtags + '#' + hashtag + ' ' if topfiveHashtags.__len__() == 0: topfiveHashtags.append((hashtag, hashtag_rel[hashtag].hashtag_rank)) else: bfound = False for i in range(0, topfiveHashtags.__len__()): if topfiveHashtags[i][1] < hashtag_rel[hashtag].hashtag_rank: topfiveHashtags.insert(i, (hashtag, hashtag_rel[hashtag].hashtag_rank)) bfound = True break if not bfound: topfiveHashtags.insert(topfiveHashtags.__len__(), (hashtag, hashtag_rel[hashtag].hashtag_rank)) if topfiveHashtags.__len__() > 10: topfiveHashtags.__delitem__(10) for data in topfiveHashtags: print data full_hashtags = full_hashtags + '#' + data[0] + ' ' top10Users = [] for user in user_tweets.keys(): user_tweets[user].user_rank = user_rank_calculator(user_tweets[user].tweet_follower, user_tweets[user].tweets.__len__(), user_tweets[user].no_of_retweets, user_tweets[user].tweet_likes) if top10Users.__len__() == 0: top10Users.append((user, user_tweets[user].user_rank)) else: bfound = False for i in range(0, top10Users.__len__()): if top10Users[i][1] < user_tweets[user].user_rank: top10Users.insert(i, (user, user_tweets[user].user_rank)) bfound = True break if not bfound: top10Users.insert(top10Users.__len__(), (user, user_tweets[user].user_rank)) if top10Users.__len__() > 10: top10Users.__delitem__(10) wordlist = LanguageProcessor.Get_Common_Words_Tweets(tweet_msgs_lst, userSearchObj.configuration_parameters['max_words']) print userSearchObj.configuration_parameters['max_words'] full_hashtags = full_hashtags[:full_hashtags.__len__()-1] posNegValues = [] posNegValues.append(total_posCount) posNegValues.append(total_negCount) metadata.__setitem__('TotalTweets', list_tweets.__len__()) metadata.__setitem__('TotalRetweets', no_of_retweets) metadata.__setitem__('TotalLikes', no_of_likes) metadata.__setitem__('TotalUsers', user_tweets.__len__()) metadata.__setitem__('Hashtags', full_hashtags) metadata.__setitem__('Top5Hashtags', topfiveHashtags) metadata.__setitem__('WordList', wordlist) metadata.__setitem__('TopUsers', top10Users) metadata.__setitem__('positiveOrNegative', posNegValues) value = json.dumps(metadata) # socketIO.emit('stats', value, '/analyze') # emit('stats', value) print 'emitting stats' socket_io.emit('stats', value, namespace='/analyze') if userSearchObj.tweet_fetch_complete is True: print 'Acknowledging Tweet fetch is complete' if previouscount == list_tweets.__len__() and userSearchObj.tweet_fetch_complete is True: print 'meta model build is done' userSearchObj.meta_model_complete = True else: previouscount = list_tweets.__len__() time.sleep(2)
def find_top_hoaxers(fake_news_url): Tweet.objects(fake_news_url=fake_news_url).order_by('-followers_count')
def last_tweet_id(link): if Tweet.objects(fake_news_url=link).count() == 0: return None else: return Tweet.objects( fake_news_url=link).order_by('-created_at')[0].id_str