def update_database():
print("fetching")
tweets = Tweet.objects(total_fetched=1, error_occured__ne=True)
print("updating")
i = 0
error_count = 0
for each_tweet in tweets:
print("loop")
data = update_given_tweet(each_tweet.tweet_id)
if not data:
error_count += 1
print("!!!!!!!!!error", error_count, "correct", i)
each_tweet.error_occured = True
each_tweet.save()
continue
elif data == 3:
continue
print("got data")
data['fetched_timestamp'] = datetime.datetime.now()
data['fresh_tweet'] = False
data['update_count'] = 2
each_tweet.total_fetched = 2
each_tweet.tweets.append(data)
each_tweet.save()
print(i, "errors=", error_count)
i += 1
def specific_tweet_fact_check(fake_news_url):
from mongoengine.queryset.visitor import Q
query = Tweet.objects(
Q(fake_news_url=fake_news_url) & Q(fact_checked=False)).order_by(
'-user__followers_count').values_list('id_str', 'fake_news_url',
'user__screen_name').first()
return query
def update_database():
print("fetching")
tweets = Tweet.objects(total_fetched=1, error_occured__ne=True)
print("updating")
i = 0
error_count = 0
for each_tweet in tweets:
print("loop")
data = update_given_tweet(each_tweet.tweet_id)
if not data:
error_count += 1
print("!!!!!!!!!error", error_count, "correct", i)
each_tweet.error_occured = True
each_tweet.save()
continue
elif data == 3:
continue
print("got data")
data['fetched_timestamp'] = datetime.datetime.now()
data['fresh_tweet'] = False
data['update_count'] = 2
each_tweet.total_fetched = 2
each_tweet.tweets.append(data)
each_tweet.save()
print(i, "errors=", error_count)
i += 1
def getTweets(request):
tweets_res = []
tweets = Tweet.objects()
for tweet in tweets:
tweets_res.append({"text":tweet.tweet_text,"time":str(tweet.tweet_time)})
return HttpResponse(json.dumps(tweets_res))
def update_fakenews_db():
urls = FakeNews.objects(fake_news_url__exists=1).values_list(
'fake_news_url', 'tweets')
for fake_story, tweets in urls:
update_story_db(fake_story)
data_object = FakeNews.objects.get(fake_news_url=fake_story)
data_object.tweets = Tweet.objects(fake_news_url=fake_story).count()
data_object.save()
def post(self, request , *args , **Kwargs):
# print "in post method"
slack_request_msg = request.data
# checking if request coming from correct workspace
if slack_request_msg.get('token') != VeriFication_Token :
return Response(status = status.HTTP_403_FORBIDDEN)
#verification challenge
if slack_request_msg.get('type') == 'url_verification' :
return Response(data=slack_request_msg , status = status.HTTP_200_OK)
# event listening
if 'event' in slack_request_msg:
msg_event = slack_request_msg.get('event')
# check if msg coming from slack api
if msg_event.get('subtype') == 'bot_message':
return Response(status=status.HTTP_200_OK)
# process user's message
user = msg_event.get('user')
text = msg_event.get('text')
channel = msg_event.get('channel')
# respond with hello msg
bot_text = 'Hi <@{}> :wave:'.format(user)
if 'go' in text.lower():
slack_client.api_call(method='chat.postMessage',
channel=channel,
text=bot_text)
# tweet_obj = Tweet(tweet_id=1, tweet_text="Hi Hesham", tweet_time=datetime.datetime.now())
# tweet_obj.save()
# x = Tweet.objects()
# for t in x :
# print t.tweet_id , t.tweet_text
# print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
# to get all tweets in the timeline
All_Tweets = tweepy.Cursor(api.home_timeline).items()
for iter in All_Tweets :
newTweet = Tweet.objects(tweet_id=iter.id)
# check if this tweet dosen't exist before
if not newTweet :
tweet_obj = Tweet(tweet_id = iter.id,tweet_text= iter.text,tweet_time = iter.created_at)
tweet_obj.save()
else :
break
return Response(status=status.HTTP_200_OK)
return Response(status=status.HTTP_200_OK)
def render_GET(self, request):
request.setHeader("content-type", "application/json")
tweets = Tweet.objects(entities__ne=None)
data = []
for tweet in tweets:
if re.search("http://pbs.twimg.com/media/", tweet.entities):
media_url = ast.literal_eval(tweet.entities)['media'][0]['media_url']
data.append({'image': media_url})
return json.dumps(data)
def tweet_fact_check():
bot = TwitterBot(settings.ACCESS_KEY, settings.ACCESS_SECRET,
settings.CONSUMER_KEY, settings.CONSUMER_SECRET)
query = Tweet.objects(
fact_checked=False).order_by('-user__followers_count').values_list(
'id_str', 'fake_news_url', 'user__screen_name').first()
fake_story = FakeNews.objects.get(fake_news_url=query[1])
correct_link = fake_story.shortened_url
message = fake_story.message_1
fc_tweet = bot.tweet_factcheck(query[0], query[2], message, correct_link)
return query
def GetPastTweets(app, searchStrings):
with app.app_context():
auth = OAuthHandler( current_app.config['TWITTER_CONSUMER_KEY'], current_app.config['TWITTER_CONSUMER_SECRET'])
auth.set_access_token(current_app.config['TWITTER_ACCESS_TOKEN'], current_app.config['TWITTER_ACCESS_TOKEN_SECRET'])
api = API(auth)
deleteEntries = connect(current_app.config['MONGODB_SETTINGS']['DB'])
deleteEntries.drop_database(current_app.config['MONGODB_SETTINGS']['DB'])
listTweetIDs = []
user_query = session['userSentence']
session['completeTweetFetch'] = False
session['completedMetaModel'] = False
for searchString in searchStrings:
# Sleep for 28 secs to avoid Twitter download rate restrictions
searchTweets = [status for status in Cursor(api.search, q=searchString).items(current_app.config['MAX_FETCH_TWEETS'])]
for tweet in searchTweets:
tweet_id = tweet.id
if listTweetIDs.__contains__(tweet_id):
continue
if len(Tweet.objects(tweet_id=tweet_id)) > 1:
continue
tweet_message = tweet.text.encode("utf-8")
tweet_userhandle = tweet.user.screen_name
tweet_retweet_count = tweet.retweet_count
tweet_createtime = tweet.created_at
tweet_location = None
tweet_geo = None
tweet_favoritecount = tweet.favorite_count
tweet_username = tweet.user.name
tweet_user_no_of_following = tweet.user.friends_count
tweet_user_no_of_followers = tweet.user.followers_count
tweet_positiveOrnegative = 0
tweet_polarOrneutral = 0
tweet_isRetweet = 0
oneTweet = Tweet(tweet_id=tweet_id, tweet_msg=tweet_message, tweet_likes=tweet_favoritecount, tweet_retweets=tweet_retweet_count, tweet_search_category=searchString, tweet_user_search_query=user_query, tweet_positiveOrnegative=tweet_positiveOrnegative, tweet_polarOrneutral=tweet_polarOrneutral, tweet_user_handle=tweet_userhandle, tweet_user_name=tweet_username, tweet_user_followers=tweet_user_no_of_followers, tweet_user_following=tweet_user_no_of_following, tweet_isretweet=tweet_isRetweet, tweet_time=tweet_createtime, tweet_location=tweet_location, tweet_geo=tweet_geo)
oneTweet.save()
listTweetIDs.append(tweet_id)
if session['metamodelThread'] is None:
session['metamodelThread'] = thread.start_new_thread(twittermetamodelBuilding, ())
print 'completed tweet fetch'
session['completeTweetFetch'] = True
pass
def update_story_db(url):
bot = TwitterBot(settings.ACCESS_KEY, settings.ACCESS_SECRET,
settings.CONSUMER_KEY, settings.CONSUMER_SECRET)
bot.search(url)
total_added = Tweet.objects(fake_news_url=url).count()
print(f'{total_added} tweets mentioning {url} added.')
def twittermetamodelBuilding(user_id):
'''
This function will be used to build the metamodel of the twitter data.
This basically develops all the background data needed for the visualization, using Tweet data stored in DB.
:return: -
'''
previouscount = 0
userSearchObj = user_searches_ongoing[user_id]
global TWITTER_DATA_DOWNLOADER_THREAD_INDEX
global META_MODEL_BUILDER_THREAD_INDEX
while(True):
if userSearchObj.meta_model_complete and userSearchObj.tweet_fetch_complete:
time.sleep(2)
continue
print userSearchObj.user_search_sentence, "From metamodel building"
list_tweets = []
user_tweets = {}
hashtag_rel = {}
list_tweets = Tweet.objects(tweet_user_search_query=userSearchObj.user_search_sentence)
tweet_msgs_lst = []
no_of_retweets = 0
no_of_likes = 0
total_posCount = 0
total_negCount = 0
metadata = {}
for tweet in list_tweets:
new_user = False
no_of_retweets += tweet.tweet_retweets
no_of_likes += tweet.tweet_likes
posCount = 0
negCount = 0
if senana.classifier.classify(senana.feature_extractor(tweet.tweet_msg.split())) == 'positive':
tweet.tweet_positiveOrnegative = 1
posCount = 1
total_posCount += 1
else:
tweet.tweet_positiveOrnegative = 0
negCount = 1
total_negCount += 1
if user_tweets.has_key(tweet.tweet_user_handle):
user = user_tweets[tweet.tweet_user_handle]
user.tweet_likes += tweet.tweet_likes
user.no_of_retweets += tweet.tweet_retweets
user.positiveCount += posCount
user.negativeCount += negCount
else:
user = Tweet_User(tweet.tweet_user_handle, tweet.tweet_user_name, tweet.tweet_user_following, tweet.tweet_user_followers, tweet.tweet_likes, tweet.tweet_retweets, posCount, negCount, 0, 0)
user_tweets.__setitem__(tweet.tweet_user_handle, user)
new_user = True
tweet_msgs_lst.append(tweet.tweet_msg)
words = tweet.tweet_msg.split(' ')
hashtags_lst = []
for word in words:
if word.__len__() > 1 and word[0] == '#':
hashtag_string = word[1:]
if not re.match("^[a-zA-Z0-9]*$", hashtag_string):
continue
hashtag_string = hashtag_string.lower()
if hashtag_rel.has_key(hashtag_string):
hashtag_obj = hashtag_rel[hashtag_string]
hashtag_obj.no_of_retweets += tweet.tweet_retweets
hashtag_obj.no_of_likes += tweet.tweet_likes
hashtag_obj.tweets.append(tweet)
if new_user:
hashtag_obj.no_of_users += 1
hashtag_obj.negativeCount += negCount
hashtag_obj.positiveCount += posCount
else:
hashtag_obj = Twitter_Hashtag(hashtag_string, tweet.tweet_retweets, tweet.tweet_likes, 1, posCount, negCount, 0, 0)
hashtag_obj.tweets.append(tweet)
hashtag_rel.__setitem__(hashtag_string, hashtag_obj)
if not hashtags_lst.__contains__(hashtag_string):
hashtags_lst.append(hashtag_string)
if not user.hashtagsUsed.has_key(hashtag_string):
user.hashtagsUsed.__setitem__(hashtag_string, hashtag_obj)
# print hashtags_lst
for hashtag in hashtags_lst:
hashtag_obj = hashtag_rel[hashtag]
hashtag_obj.hashtag_rank = hashtag_rank_calculator(hashtag_obj.tweets.__len__(), hashtag_obj.no_of_users, hashtag_obj.no_of_retweets, hashtag_obj.no_of_likes)
for otherhashtag in hashtags_lst:
if otherhashtag == hashtag:
continue
if not hashtag_obj.related_hashtags.has_key(otherhashtag):
hashtag_obj.related_hashtags.__setitem__(otherhashtag, hashtag_rel[otherhashtag])
full_hashtags = ''
topfiveHashtags = []
for hashtag in hashtag_rel.keys():
# full_hashtags = full_hashtags + '#' + hashtag + ' '
if topfiveHashtags.__len__() == 0:
topfiveHashtags.append((hashtag, hashtag_rel[hashtag].hashtag_rank))
else:
bfound = False
for i in range(0, topfiveHashtags.__len__()):
if topfiveHashtags[i][1] < hashtag_rel[hashtag].hashtag_rank:
topfiveHashtags.insert(i, (hashtag, hashtag_rel[hashtag].hashtag_rank))
bfound = True
break
if not bfound:
topfiveHashtags.insert(topfiveHashtags.__len__(), (hashtag, hashtag_rel[hashtag].hashtag_rank))
if topfiveHashtags.__len__() > 10:
topfiveHashtags.__delitem__(10)
for data in topfiveHashtags:
print data
full_hashtags = full_hashtags + '#' + data[0] + ' '
top10Users = []
for user in user_tweets.keys():
user_tweets[user].user_rank = user_rank_calculator(user_tweets[user].tweet_follower, user_tweets[user].tweets.__len__(), user_tweets[user].no_of_retweets, user_tweets[user].tweet_likes)
if top10Users.__len__() == 0:
top10Users.append((user, user_tweets[user].user_rank))
else:
bfound = False
for i in range(0, top10Users.__len__()):
if top10Users[i][1] < user_tweets[user].user_rank:
top10Users.insert(i, (user, user_tweets[user].user_rank))
bfound = True
break
if not bfound:
top10Users.insert(top10Users.__len__(), (user, user_tweets[user].user_rank))
if top10Users.__len__() > 10:
top10Users.__delitem__(10)
wordlist = LanguageProcessor.Get_Common_Words_Tweets(tweet_msgs_lst, userSearchObj.configuration_parameters['max_words'])
print userSearchObj.configuration_parameters['max_words']
full_hashtags = full_hashtags[:full_hashtags.__len__()-1]
posNegValues = []
posNegValues.append(total_posCount)
posNegValues.append(total_negCount)
metadata.__setitem__('TotalTweets', list_tweets.__len__())
metadata.__setitem__('TotalRetweets', no_of_retweets)
metadata.__setitem__('TotalLikes', no_of_likes)
metadata.__setitem__('TotalUsers', user_tweets.__len__())
metadata.__setitem__('Hashtags', full_hashtags)
metadata.__setitem__('Top5Hashtags', topfiveHashtags)
metadata.__setitem__('WordList', wordlist)
metadata.__setitem__('TopUsers', top10Users)
metadata.__setitem__('positiveOrNegative', posNegValues)
value = json.dumps(metadata)
# socketIO.emit('stats', value, '/analyze')
# emit('stats', value)
print 'emitting stats'
socket_io.emit('stats', value, namespace='/analyze')
if userSearchObj.tweet_fetch_complete is True:
print 'Acknowledging Tweet fetch is complete'
if previouscount == list_tweets.__len__() and userSearchObj.tweet_fetch_complete is True:
print 'meta model build is done'
userSearchObj.meta_model_complete = True
else:
previouscount = list_tweets.__len__()
time.sleep(2)
def find_top_hoaxers(fake_news_url):
Tweet.objects(fake_news_url=fake_news_url).order_by('-followers_count')
def last_tweet_id(link):
if Tweet.objects(fake_news_url=link).count() == 0:
return None
else:
return Tweet.objects(
fake_news_url=link).order_by('-created_at')[0].id_str
