def set_muzzled(self, request): qp = request.query_params muzzled = True if "muzzled" in qp and qp["muzzled"] == "1" else False with transaction.atomic(): t = get_twitter_app() if t is None: raise ScremsongException( "We haven't authenticated against Twitter yet.") t.settings = {**t.settings, "muzzled": muzzled} t.save() # Update settings for connected clients websockets.send_channel_message( "socialplatforms.settings", {"settings": { str(SocialPlatformChoice.TWITTER): t.settings }}) # Send a notification to all connected clients message = "We've been muzzled by Twitter! Replying, favouriting, and retweeting will now open tweets in a new tab. See WhatsApp for more information. **And please reload Scremsong!**" if muzzled is True else "It's all good. We've been unmuzzled by Twitter. Scremsong is returning to normal operations 🎉 **And please reload Scremsong!**" websockets.send_channel_message( "notifications.send", { "message": message, "options": { "variant": NotificationVariants.WARNING } }) return Response({"settings": t.settings})
def set_user_accepting_assignments(self, request, format=None): qp = request.query_params user_id = int(qp["user_id"]) if "user_id" in qp else None is_accepting_assignments = True if "is_accepting_assignments" in qp and qp[ "is_accepting_assignments"] == "true" else False profile = Profile.objects.get(user_id=user_id) profile.is_accepting_assignments = is_accepting_assignments if is_accepting_assignments is True: profile.offline_reason = None else: profile.offline_reason = ProfileOfflineReason.USER_CHOICE profile.save() websockets.send_channel_message( "reviewers.set_status", { "user_id": user_id, "is_accepting_assignments": is_accepting_assignments }) if is_accepting_assignments is True: message = "{} has come online and is ready to receive assignments!".format( UserSerializer(profile.user).data["name"]) else: message = "{} has gone offline".format( UserSerializer(profile.user).data["name"]) websockets.send_channel_message("notifications.send", { "message": message, "options": { "variant": NotificationVariants.INFO } }) return Response({"OK": True})
def reply_to_tweet(inReplyToTweetId, replyText): try: api = get_tweepy_api_auth() status = api.update_status(status=replyText, in_reply_to_status_id=inReplyToTweetId) reply, created = save_tweet(status._json, source=TweetSource.REPLYING, status=TweetStatus.OK) websockets.send_channel_message("tweets.update_tweets", { "tweets": { reply.tweet_id: TweetsSerializer(reply).data }, }) except tweepy.RateLimitError as e: logger.warning( "Got a RateLimitError from Tweepy while sending a reply to {}". format(inReplyToTweetId)) raise e except tweepy.TweepError as e: logger.warning( "Got a TweepError {} from Tweepy while sending a reply to {}". format(e.api_code, inReplyToTweetId)) raise e
def receive_json(self, content): if "type" not in content: return None if content["type"] == settings.MSG_TYPE_USER_CHANGE_SETTINGS: websockets.send_channel_message("user.change_settings", {"settings": content["settings"]})
def unassign_triager(self, request, format=None): qp = request.query_params columnId = qp["columnId"] if "columnId" in qp else None column = SocialColumns.objects.get(id=columnId) if column.assigned_to_id is not None: websockets.send_user_channel_message( "notifications.send", { "message": "You have been unassigned from triaging the column \"{}\"". format(" ".join(column.search_phrases)), "options": { "variant": NotificationVariants.INFO } }, column.assigned_to.username) column.assigned_to_id = None column.save() websockets.send_channel_message( "columns.update", { "columns": [ SocialColumnsSerializerWithTweetCountSerializer( column).data ], }) return Response({"OK": True})
def disconnect(self, close_code): # Leave the group async_to_sync(self.channel_layer.group_discard)(self.group_name, self.channel_name) # Mark the user as offline if self.user.id is not None: profile = Profile.objects.get(user_id=self.user.id) if profile.is_accepting_assignments is True: profile.is_accepting_assignments = False profile.offline_reason = ProfileOfflineReason.DISCONNECTED profile.save() websockets.send_channel_message( "reviewers.set_status", { "user_id": self.user.id, "is_accepting_assignments": False }) # Let all connected clients know that the user has gone offline websockets.send_channel_message( "notifications.send", { "message": "{} has disconnected and gone offline".format( UserSerializer(self.user).data["name"]), "options": { "variant": NotificationVariants.INFO } }) logger.debug('scremsong disconnect channel=%s user=%s', self.channel_name, self.user)
def connect(self): self.user = self.scope["user"] self.group_name = 'scremsong_%s' % self.scope['url_route']['kwargs'][ 'group_name'] # For all Scremsong users self.user_group_name = 'user_%s' % self.user # Just for this user if self.user.is_anonymous is False and self.user.is_authenticated: # Join the general Scremsong group async_to_sync(self.channel_layer.group_add)(self.group_name, self.channel_name) # Join the specific channel for this user (lets us send user-specific messages) async_to_sync(self.channel_layer.group_add)(self.user_group_name, self.channel_name) self.accept() # Send a message back to the client on a successful connection self.send_json(build_on_connect_data_payload(self.user)) # If we're reconnecting from a recent disconnection pop the user back online if self.user.profile.is_accepting_assignments is False and self.user.profile.offline_reason == ProfileOfflineReason.DISCONNECTED: Profile.objects.filter(user_id=self.user.id).update( is_accepting_assignments=True, offline_reason=None) self.user.profile.is_accepting_assignments = True self.user.profile.offline_reason = None # reviewers.user_connected takes care of sending our updated profile object # Send a message to all connected clients that a new user has come online # We send the whole object to deal with brand new registered users coming online for the first time websockets.send_channel_message( "reviewers.user_connected", {"user": ReviewerUserSerializer(self.user).data}) # Let all connected clients that the user has come online if self.user.profile.is_accepting_assignments is True: message = "{} has come online and is ready to receive assignments!".format( UserSerializer(self.user).data["name"]) else: message = "{} has come online but isn't ready to accept assignments yet".format( UserSerializer(self.user).data["name"]) websockets.send_channel_message( "notifications.send", { "message": message, "options": { "variant": NotificationVariants.INFO } }) logger.debug( 'scremsong connect channel=%s group=%s group=%s user=%s', self.channel_name, self.group_name, self.user_group_name, self.user) else: # Setting a code doesn't actually seem to work # https://github.com/django/channels/issues/414 self.close(code=4000)
def notify_of_saved_tweets(tweets): if len(tweets) > 0: response = { "tweets": {}, } for tweet in tweets: response["tweets"][tweet.tweet_id] = TweetsSerializer(tweet).data websockets.send_channel_message("tweets.new_tweets", response)
def on_connect(self): logger.info("on_connect") websockets.send_channel_message("notifications.send", { "message": "Real-time tweet streaming has connected.", "options": { "variant": NotificationVariants.INFO } }) websockets.send_channel_message("tweets.streaming_state", { "connected": True, })
def unassign_reviewer(self, request, format=None): qp = request.query_params assignmentId = int( qp["assignmentId"]) if "assignmentId" in qp else None assignment = SocialAssignments.objects.get(id=assignmentId) assignment.delete() websockets.send_channel_message("reviewers.unassign", { "assignmentId": assignmentId, }) return Response({"OK": True})
def assign_reviewer(self, request, format=None): qp = request.query_params tweetId = qp["tweetId"] if "tweetId" in qp else None reviewerId = qp["reviewerId"] if "reviewerId" in qp else None try: status = get_status_from_db(tweetId) if status is None: raise ScremsongException( "Could not find tweet {} in local db".format(tweetId)) parents, parent = resolve_tweet_parents(status) parent, tweets, relationships = resolve_tweet_thread_for_parent( parent) replyTweetIds = [ tweetId for tweetId in list(tweets.keys()) if tweetId != parent["data"]["id_str"] ] assignment, created = SocialAssignments.objects.update_or_create( platform=SocialPlatformChoice.TWITTER, social_id=parent["data"]["id_str"], defaults={ "user_id": reviewerId, "assigned_by": request.user, "thread_relationships": relationships, "thread_tweets": replyTweetIds }) websockets.send_channel_message( "reviewers.assign", { "assignment": SocialAssignmentSerializer(assignment).data, "tweets": set_tweet_object_state_en_masse(tweets, TweetState.ASSIGNED), }) return Response({"OK": True}) except ScremsongException as e: return Response( { "error": "Could not assign tweet {}. Failed to resolve and update tweet thread. Message: {}" .format(tweetId, str(e)) }, status=status.HTTP_400_BAD_REQUEST)
def mark_read(self, request, format=None): qp = request.query_params assignmentId = int( qp["assignmentId"]) if "assignmentId" in qp else None assignment = SocialAssignments.objects.get(id=assignmentId) assignment.last_read_on = getCreationDateOfNewestTweetInAssignment( assignment) assignment.save() websockets.send_channel_message( "reviewers.assignment_metdata_changed", { "assignment": SocialAssignmentSerializer(assignment).data, }) return Response({"OK": True})
def restore(self, request, format=None): qp = request.query_params assignmentId = int( qp["assignmentId"]) if "assignmentId" in qp else None assignment = SocialAssignments.objects.get(id=assignmentId) assignment.close_reason = None assignment.state = SocialAssignmentState.PENDING assignment.save() websockets.send_channel_message( "reviewers.assignment_metdata_changed", { "assignment": SocialAssignmentSerializer(assignment).data, }) return Response({"OK": True})
def bulk_reassign_reviewer(self, request, format=None): qp = request.query_params currentReviewerId = int( qp["currentReviewerId"]) if "currentReviewerId" in qp else None newReviewerId = int( qp["newReviewerId"]) if "newReviewerId" in qp else None if currentReviewerId is not None and newReviewerId is not None: try: assignmentsUpdated = [] tweetsUpdated = {} assignments = SocialAssignments.objects.filter( user_id=currentReviewerId).filter( state=SocialAssignmentState.PENDING) with transaction.atomic(): for assignment in assignments: assignment.user_id = newReviewerId assignment.assigned_by = request.user assignment.save() assignmentsUpdated.append( SocialAssignmentSerializer(assignment).data) parent = get_status_from_db(assignment.social_id) parent, tweets, relationships = resolve_tweet_thread_for_parent( parent) tweetsUpdated = { **tweetsUpdated, **set_tweet_object_state_en_masse( tweets, TweetState.ASSIGNED) } websockets.send_channel_message("reviewers.bulk_assign", { "assignments": assignmentsUpdated, "tweets": tweetsUpdated, }) return Response({"OK": True}) except ScremsongException as e: return Response( { "error": "Could not bulk reassign assignments for {} to {}. Failed to resolve and update tweet thread. Message: {}" .format(currentReviewerId, newReviewerId, str(e)) }, status=status.HTTP_400_BAD_REQUEST)
def retweet_tweet(tweetId): def ws_send_updated_tweet(tweet): websockets.send_channel_message("tweets.update_tweets", { "tweets": { tweet.tweet_id: TweetsSerializer(tweet).data }, }) tweet = Tweets.objects.get(tweet_id=tweetId) # The client shouldn't be able to retweet a tweet they've already retweeted. # Fail quietly and send out new state to all connected clients. if tweet.data["retweeted"] is True: ws_send_updated_tweet(tweet) return try: api = get_tweepy_api_auth() status = api.retweet(tweetId) tweet.data = status._json["retweeted_status"] tweet.save() retweet, created = save_tweet(status._json, source=TweetSource.RETWEETING, status=TweetStatus.OK) websockets.send_channel_message( "tweets.update_tweets", { "tweets": { tweet.tweet_id: TweetsSerializer(tweet).data, retweet.tweet_id: TweetsSerializer(retweet).data }, }) except tweepy.TweepError as e: if e.api_code == 327: # The tweet was already retweeted somewhere else (e.g. another Twitter client). Update local state tweet and respond as if we succeeded. tweet.data["retweeted"] = True tweet.save() ws_send_updated_tweet(tweet) else: # Uh oh, some other error code was returned # NB: tweepy.api can return certain errors via retry_errors raise e
def set_state(self, request, format=None): qp = request.query_params tweetId = qp["tweetId"] if "tweetId" in qp else None tweetState = qp["tweetState"] if "tweetState" in qp else None if TweetState.has_value(tweetState): tweet = Tweets.objects.get(tweet_id=tweetId) tweet.state = tweetState tweet.save() websockets.send_channel_message("tweets.set_state", { "tweetStates": [{ "tweetId": tweetId, "tweetState": tweetState, }] }) return Response({})
def task_open_tweet_stream(self): from scremsong.app.twitter_streaming import open_tweet_stream open_tweet_stream() logger.warning("Unexpectedly done streaming tweets!") websockets.send_channel_message( "notifications.send", { "message": "Real-time tweet streaming has disconnected (death).", "options": { "variant": NotificationVariants.ERROR, "autoHideDuration": None } }) websockets.send_channel_message("tweets.streaming_state", { "connected": False, }) return True
def close(self, request, format=None): qp = request.query_params assignmentId = int( qp["assignmentId"]) if "assignmentId" in qp else None reason = str(qp["reason"]) if "reason" in qp else None if SocialAssignmentCloseReason.has_value(reason) is True: assignment = SocialAssignments.objects.get(id=assignmentId) assignment.close_reason = reason assignment.state = SocialAssignmentState.CLOSED assignment.last_read_on = getCreationDateOfNewestTweetInAssignment( assignment) assignment.save() websockets.send_channel_message( "reviewers.assignment_metdata_changed", { "assignment": SocialAssignmentSerializer(assignment).data, }) return Response({"OK": True})
def reassign_reviewer(self, request, format=None): qp = request.query_params assignmentId = int( qp["assignmentId"]) if "assignmentId" in qp else None newReviewerId = int( qp["newReviewerId"]) if "newReviewerId" in qp else None if assignmentId is not None and newReviewerId is not None: try: assignment = get_or_none(SocialAssignments, id=assignmentId) assignment.user_id = newReviewerId assignment.assigned_by = request.user assignment.save() parent = get_status_from_db(assignment.social_id) parent, tweets, relationships = resolve_tweet_thread_for_parent( parent) websockets.send_channel_message( "reviewers.assign", { "assignment": SocialAssignmentSerializer(assignment).data, "tweets": set_tweet_object_state_en_masse( tweets, TweetState.ASSIGNED), }) return Response({"OK": True}) except ScremsongException as e: return Response( { "error": "Could not reassign assignment {}. Failed to resolve and update tweet thread. Message: {}" .format(assignmentId, str(e)) }, status=status.HTTP_400_BAD_REQUEST)
def ws_send_updated_tweet(tweet): websockets.send_channel_message("tweets.update_tweets", { "tweets": { tweet.tweet_id: TweetsSerializer(tweet).data }, })
def task_collect_twitter_rate_limit_info(self): # logger.warning("task_collect_twitter_rate_limit_info() is disabled when an election is not running") # return True if is_rate_limit_collection_task_running( excludeTaskId=self.request.id) is True: logger.warning( "Abandoning starting Twitter rate limit collection - an identical task already exists" ) return True from scremsong.app.models import TwitterRateLimitInfo from scremsong.app.twitter import are_we_rate_limited, get_tweepy_api_auth api = get_tweepy_api_auth() while True: status = api.rate_limit_status() resources = status["resources"] r = TwitterRateLimitInfo(data=resources) r.save() websockets.send_channel_message("tweets.rate_limit_resources", { "resources": resources, }) rateLimitedResources = are_we_rate_limited(resources, bufferPercentage=0.2) if len(rateLimitedResources.keys()) > 0: resourceNames = [ resource_name for resource_name in rateLimitedResources.keys() ] websockets.send_channel_message( "notifications.send", { "message": "We've been rate limited by Twitter for {}.".format( ", ".join(resourceNames)), "options": { "variant": NotificationVariants.ERROR, "autoHideDuration": 20000, } }) websockets.send_channel_message( "tweets.rate_limit_state", { "state": TwitterRateLimitState.RATE_LIMITED, }) else: rateLimitedResources = are_we_rate_limited(resources, bufferPercentage=0.20) if len(rateLimitedResources.keys()) > 0: websockets.send_channel_message( "notifications.send", { "message": "Twitter rate limit approaching for {}.".format( ", ".join(resourceNames)), "options": { "variant": NotificationVariants.WARNING, "autoHideDuration": 20000, } }) websockets.send_channel_message( "tweets.rate_limit_state", { "state": TwitterRateLimitState.WARNING, }) else: websockets.send_channel_message( "tweets.rate_limit_state", { "state": TwitterRateLimitState.EVERYTHING_OK, }) sleep(30) logger.warning("Unexpectedly done collecting Twitter rate limit info!") return True
def open_tweet_stream(): # https://stackoverflow.com/a/33660005/7368493 class MyStreamListener(tweepy.StreamListener): def on_status(self, status): logger.debug("Sending tweet {} to the queue to be processed from streaming".format(status._json["id_str"])) task_process_tweet_reply.apply_async(args=[status._json, TweetSource.STREAMING, True]) def on_error(self, status_code): if status_code == 420: logger.warning("Streaming got status {}. Disconnecting from stream.".format(status_code)) # Fire off tasks to restart streaming (delayed by 2s) celery_restart_streaming(wait=10) # Returning False in on_error disconnects the stream return False # Returning non-False reconnects the stream, with backoff logger.warning("Streaming got status {}. Taking no action.".format(status_code)) def on_limit(self, track): logger.warning("Received an on limit message from Twitter.") def on_timeout(self): logger.critical("Streaming connection to Twitter has timed out.") # Fire off tasks to restart streaming (delayed by 2s) celery_restart_streaming(wait=2) # Returning False in on_timeout disconnects the stream return False def on_disconnect(self, notice): """Called when twitter sends a disconnect notice Disconnect codes are listed here: https://dev.twitter.com/docs/streaming-apis/messages#Disconnect_messages_disconnect """ logger.critical("Received a disconnect notice from Twitter: {}".format(notice)) # Fire off tasks to restart streaming (delayed by 2s) celery_restart_streaming(wait=2) # Returning False in on_disconnect disconnects the stream return False def on_warning(self, notice): logger.critical("Received disconnection warning notice from Twitter. {}".format(notice)) def on_connect(self): logger.info("on_connect") websockets.send_channel_message("notifications.send", { "message": "Real-time tweet streaming has connected.", "options": { "variant": NotificationVariants.INFO } }) websockets.send_channel_message("tweets.streaming_state", { "connected": True, }) def on_data(self, raw_data): logger.info("on_data") return super(MyStreamListener, self).on_data(raw_data) def on_delete(self, status_id, user_id): """Called when a delete notice arrives for a status""" logger.warning("on_delete: {}, {}".format(status_id, user_id)) return def keep_alive(self): """Called when a keep-alive arrived""" logger.info("keep_alive") return # Create Twitter app credentials + config store if it doesn't exist t = get_twitter_app() if t is None: t = SocialPlatforms(platform=SocialPlatformChoice.TWITTER) t.save() t = get_twitter_app() # Begin streaming! api = get_tweepy_api_auth() if api is None: logger.critical("No Twitter credentials available! Please generate them by-hand.") return None try: myStreamListener = MyStreamListener() myStream = tweepy.Stream(auth=api.auth, listener=myStreamListener) track = [] [track.extend(column.search_phrases) for column in get_social_columns(SocialPlatformChoice.TWITTER)] if len(track) == 0: logger.info("No search phrases are set - we won't try to stream tweets") else: logger.info("track") logger.info(track) # Fill in any gaps in tweets since streaming last stopped sinceId = get_latest_tweet_id_for_streaming() logger.info("Tweet streaming about to start. Queueing up fill in missing tweets task since {}.".format(sinceId)) if sinceId is not None: task_fill_missing_tweets.apply_async(args=[sinceId], countdown=5) else: logger.warning("Got sinceId of None when trying to start task_fill_missing_tweets") # Begin streaming! myStream.filter(track=track, stall_warnings=True) logger.warning("Oops, looks like tweet streaming has ended unexpectedly.") websockets.send_channel_message("notifications.send", { "message": "Real-time tweet streaming has disconnected.", "options": { "variant": NotificationVariants.ERROR, "autoHideDuration": None } }) websockets.send_channel_message("tweets.streaming_state", { "connected": False, }) except Exception as e: logger.error("Exception {}: '{}' during streaming".format(type(e), str(e))) websockets.send_channel_message("notifications.send", { "message": "Real-time tweet streaming has encountered an exception. Trying to restart.", "options": { "variant": NotificationVariants.ERROR, "autoHideDuration": 10000 } }) # Fire off tasks to restart streaming celery_restart_streaming(wait=5)
def process_new_tweet_reply(status, tweetSource, sendWebSocketEvent): # Deal with tweets arriving / being processed out of order. # If it's already part of an assignment then it's been processed and clients have been notified. if is_tweet_part_of_an_assignment(status["id_str"]) is True: logger.warning( "Got tweet {} that was already part of an assignment (process_new_tweet_reply)" .format(status["id_str"])) # Only send a web socket event if we're not handling this elsewhere (e.g. backfilling) if sendWebSocketEvent is True: notify_of_saved_tweet(get_tweet_from_db(status["id_str"])) return True # OK! This means that this is a newly arrived tweet, so we need to work out if it's part of an already existing assignment. # Start by saving the tweet as dirty (i.e. we need it in the database for thread resolution, but haven't finished processing it yet) tweet, created = save_tweet(status, source=tweetSource, status=TweetStatus.DIRTY) try: parents, parent = resolve_tweet_parents(status) # If the parent is tweet is part of an assignment then we need to go and # run a refresh to get us all new replies in the thread. # This gets us replies that we don't get via the stream, as well as our own replies. if is_tweet_part_of_an_assignment(parent["id_str"]) is True: parent, tweets, relationships = resolve_tweet_thread_for_parent( parent) replyTweetIds = [ tweetId for tweetId in list(tweets.keys()) if tweetId != parent["data"]["id_str"] ] assignment, created = SocialAssignments.objects.update_or_create( platform=SocialPlatformChoice.TWITTER, social_id=parent["data"]["id_str"], defaults={ "thread_relationships": relationships, "thread_tweets": replyTweetIds, "last_updated_on": timezone.now() }) # Adding a new tweet reopens the assignment if the user is accepting assignments or unassigns it if they're offline and the assignment is closed # if is_user_accepting_assignments(assignment.user_id) is False and is_from_demsausage(tweet) is False: # logger.info("Processing tweet {}: User is offline, so delete the assignment".format(status["id_str"])) # assignmentId = assignment.id # assignment.delete() # websockets.send_channel_message("reviewers.unassign", { # "assignmentId": assignmentId, # }) # else: # Adding a new tweet reopens the assignment if the user is accepting assignments if assignment.state == SocialAssignmentState.CLOSED: if is_from_demsausage(tweet) is False: logger.info( "Processing tweet {}: Set it to pending".format( status["id_str"])) assignment.state = SocialAssignmentState.PENDING assignment.save() websockets.send_user_channel_message( "notifications.send", { "message": "One of your assignments has had a new reply arrive - it's been added back into your queue again", "options": { "variant": NotificationVariants.INFO } }, assignment.user.username) else: websockets.send_user_channel_message( "notifications.send", { "message": "One of your assignments has had a new reply arrive", "options": { "variant": NotificationVariants.INFO } }, assignment.user.username) websockets.send_channel_message( "reviewers.assignment_updated", { "assignment": SocialAssignmentSerializer(assignment).data, "tweets": tweets, }) tweet.state = TweetState.ASSIGNED # Once we're done processing the tweet, or if its parent is not part of an assignment, # then we just carry on and save the tweet has processed and send a notification. tweet.status = TweetStatus.OK tweet.save() if sendWebSocketEvent is True: notify_of_saved_tweet(tweet) return True except Exception as e: # Mark tweets as dirty if we failed to resolve thread relationships (or if something else terrible happened) tweet = Tweets.objects.update_or_create( tweet_id=tweet.tweet_id, defaults={"status": TweetStatus.DIRTY}) raise e logger.error("Failed to process new tweet {}".format(status["id_str"])) return False