def get_stats(self, request, format=None):
stats = {
"assignments": {
# "all_time": {},
"past_week": {},
},
"triage": {
# "all_time": {},
"past_week": {},
}
}
for user in User.objects.all():
# stats["assignments"]["all_time"][user.id] = get_social_assignment_stats_for_user(user)
stats["assignments"]["past_week"][
user.id] = get_social_assignment_stats_for_user(
user, sincePastNDays=7)
for social_column in get_social_columns(
SocialPlatformChoice.TWITTER).order_by("priority").all():
# stats["triage"]["all_time"][social_column.id] = get_stats_for_column(social_column)
stats["triage"]["past_week"][
social_column.id] = get_stats_for_column(social_column,
sincePastNDays=7)
return Response(stats)
def fetch_tweets(startIndex,
stopIndex,
sinceId=None,
maxId=None,
columnIds=[]):
columns = []
tweets = {}
for social_column in get_social_columns(SocialPlatformChoice.TWITTER,
columnIds):
column_tweets = get_tweets_for_column(social_column, sinceId, maxId,
startIndex, stopIndex)
column_tweet_ids = []
for tweet in column_tweets:
tweets[tweet["tweet_id"]] = TweetsSerializer(tweet).data
column_tweet_ids.append(tweet["tweet_id"])
columns.append({
"id": social_column.id,
"tweet_ids": column_tweet_ids,
})
return {
"columns": columns,
"tweets": tweets,
}
def fill_in_missing_tweets(since_id, max_id):
if since_id >= max_id:
logger.warning(
"since_id {} is out of range of max_id {} in fill_in_missing_tweets - it should be a lower number!"
.format(since_id, max_id))
return None
api = get_tweepy_api_auth()
if api is None:
logger.warning(
"No Twitter credentials available! Please generate them by-hand.")
return None
tweets = []
total_tweets_added = 0
for column in get_social_columns(SocialPlatformChoice.TWITTER):
tweets_added = 0
q = column_search_phrase_to_twitter_search_query(column)
for status in tweepy_rate_limit_handled(
tweepy.Cursor(api.search,
q=q,
result_type="recent",
tweet_mode="extended",
include_entities=True,
since_id=since_id,
max_id=max_id).items()):
tweet, created = save_tweet(status._json,
source=TweetSource.BACKFILL,
status=TweetStatus.OK)
tweets.append(tweet)
tweets_added += 1
total_tweets_added += 1
logger.info("Filled in {} missing tweets for the query '{}'".format(
tweets_added, q))
# Sort all of our backfilled tweets by their id (i.e. newest tweets first) so our thread resolution code can maximise use of the local database
tweetsByAge = sorted(tweets, key=lambda t: t.tweet_id, reverse=True)
for tweet in tweetsByAge:
if is_a_reply(tweet.data) is False:
save_tweet(tweet.data,
source=TweetSource.BACKFILL,
status=TweetStatus.OK)
else:
# Save the tweet anyway so we can send the en masse notice of new tweets
save_tweet(tweet.data,
source=TweetSource.BACKFILL,
status=TweetStatus.DIRTY)
logger.info(
"Sending tweet {} to the queue to be processed for backfill".
format(tweet.tweet_id))
task_process_tweet_reply.apply_async(
args=[tweet.data, TweetSource.BACKFILL, False])
notify_of_saved_tweets(tweets)
return total_tweets_added
def fetch_tweets_for_columns(columnPositions, columnIds=[]):
columns = []
tweets = {}
for social_column in get_social_columns(SocialPlatformChoice.TWITTER,
columnIds):
hasColumnPosition = columnPositions is not None and str(
social_column.id) in columnPositions
if hasColumnPosition is True:
sinceId = int(columnPositions[str(
social_column.id)]["stopTweet"]) - 1
column_tweets = get_tweets_for_column_by_tweet_ids(
social_column, sinceId)
else:
column_tweets = get_tweets_for_column(social_column,
startIndex=0,
stopIndex=20)
column_tweet_ids = []
column_tweet_ids_buffered = []
for tweet in column_tweets:
tweets[tweet["tweet_id"]] = TweetsSerializer(tweet).data
if hasColumnPosition is True and int(tweet["tweet_id"]) > int(
columnPositions[str(social_column.id)]["firstTweet"]):
column_tweet_ids_buffered.append(tweet["tweet_id"])
else:
column_tweet_ids.append(tweet["tweet_id"])
columns.append({
"id": social_column.id,
"tweet_ids": column_tweet_ids,
"tweet_ids_buffered": column_tweet_ids_buffered,
})
return {
"columns": columns,
"tweets": tweets,
}
def open_tweet_stream():
# https://stackoverflow.com/a/33660005/7368493
class MyStreamListener(tweepy.StreamListener):
def on_status(self, status):
logger.debug("Sending tweet {} to the queue to be processed from streaming".format(status._json["id_str"]))
task_process_tweet_reply.apply_async(args=[status._json, TweetSource.STREAMING, True])
def on_error(self, status_code):
if status_code == 420:
logger.warning("Streaming got status {}. Disconnecting from stream.".format(status_code))
# Fire off tasks to restart streaming (delayed by 2s)
celery_restart_streaming(wait=10)
# Returning False in on_error disconnects the stream
return False
# Returning non-False reconnects the stream, with backoff
logger.warning("Streaming got status {}. Taking no action.".format(status_code))
def on_limit(self, track):
logger.warning("Received an on limit message from Twitter.")
def on_timeout(self):
logger.critical("Streaming connection to Twitter has timed out.")
# Fire off tasks to restart streaming (delayed by 2s)
celery_restart_streaming(wait=2)
# Returning False in on_timeout disconnects the stream
return False
def on_disconnect(self, notice):
"""Called when twitter sends a disconnect notice
Disconnect codes are listed here:
https://dev.twitter.com/docs/streaming-apis/messages#Disconnect_messages_disconnect
"""
logger.critical("Received a disconnect notice from Twitter: {}".format(notice))
# Fire off tasks to restart streaming (delayed by 2s)
celery_restart_streaming(wait=2)
# Returning False in on_disconnect disconnects the stream
return False
def on_warning(self, notice):
logger.critical("Received disconnection warning notice from Twitter. {}".format(notice))
def on_connect(self):
logger.info("on_connect")
websockets.send_channel_message("notifications.send", {
"message": "Real-time tweet streaming has connected.",
"options": {
"variant": NotificationVariants.INFO
}
})
websockets.send_channel_message("tweets.streaming_state", {
"connected": True,
})
def on_data(self, raw_data):
logger.info("on_data")
return super(MyStreamListener, self).on_data(raw_data)
def on_delete(self, status_id, user_id):
"""Called when a delete notice arrives for a status"""
logger.warning("on_delete: {}, {}".format(status_id, user_id))
return
def keep_alive(self):
"""Called when a keep-alive arrived"""
logger.info("keep_alive")
return
# Create Twitter app credentials + config store if it doesn't exist
t = get_twitter_app()
if t is None:
t = SocialPlatforms(platform=SocialPlatformChoice.TWITTER)
t.save()
t = get_twitter_app()
# Begin streaming!
api = get_tweepy_api_auth()
if api is None:
logger.critical("No Twitter credentials available! Please generate them by-hand.")
return None
try:
myStreamListener = MyStreamListener()
myStream = tweepy.Stream(auth=api.auth, listener=myStreamListener)
track = []
[track.extend(column.search_phrases) for column in get_social_columns(SocialPlatformChoice.TWITTER)]
if len(track) == 0:
logger.info("No search phrases are set - we won't try to stream tweets")
else:
logger.info("track")
logger.info(track)
# Fill in any gaps in tweets since streaming last stopped
sinceId = get_latest_tweet_id_for_streaming()
logger.info("Tweet streaming about to start. Queueing up fill in missing tweets task since {}.".format(sinceId))
if sinceId is not None:
task_fill_missing_tweets.apply_async(args=[sinceId], countdown=5)
else:
logger.warning("Got sinceId of None when trying to start task_fill_missing_tweets")
# Begin streaming!
myStream.filter(track=track, stall_warnings=True)
logger.warning("Oops, looks like tweet streaming has ended unexpectedly.")
websockets.send_channel_message("notifications.send", {
"message": "Real-time tweet streaming has disconnected.",
"options": {
"variant": NotificationVariants.ERROR,
"autoHideDuration": None
}
})
websockets.send_channel_message("tweets.streaming_state", {
"connected": False,
})
except Exception as e:
logger.error("Exception {}: '{}' during streaming".format(type(e), str(e)))
websockets.send_channel_message("notifications.send", {
"message": "Real-time tweet streaming has encountered an exception. Trying to restart.",
"options": {
"variant": NotificationVariants.ERROR,
"autoHideDuration": 10000
}
})
# Fire off tasks to restart streaming
celery_restart_streaming(wait=5)
def get_twitter_columns():
cols = get_social_columns(
SocialPlatformChoice.TWITTER).order_by("priority").all()
return SocialColumnsSerializerWithTweetCountSerializer(cols,
many=True).data