def find_old_ids(): # type: () -> None recips = ', '.join(str(id) for id in recipient_ids) is_topic_muted = build_topic_mute_checker(user_profile) query = ''' SELECT zerver_usermessage.id, zerver_message.recipient_id, zerver_message.subject FROM zerver_usermessage INNER JOIN zerver_message ON ( zerver_message.id = zerver_usermessage.message_id ) WHERE ( zerver_usermessage.user_profile_id = %s AND zerver_usermessage.message_id <= %s AND (zerver_usermessage.flags & 1) = 0 AND zerver_message.recipient_id in (%s) ) ''' % (user_profile.id, pointer, recips) print(''' EXPLAIN analyze''' + query.rstrip() + ';') cursor.execute(query) rows = cursor.fetchall() for (um_id, recipient_id, topic) in rows: if not is_topic_muted(recipient_id, topic): user_message_ids.append(um_id) print('rows found: %d' % (len(user_message_ids), ))
def show_all_unread(user_profile: UserProfile) -> None: unreads = get_unread_messages(user_profile) stream_ids = {row['stream_id'] for row in unreads} muted_stream_ids = get_muted_streams(user_profile, stream_ids) is_topic_muted = build_topic_mute_checker(user_profile) for row in unreads: row['stream_muted'] = row['stream_id'] in muted_stream_ids row['topic_muted'] = is_topic_muted(row['recipient_id'], row['topic']) row['before'] = row['message_id'] < user_profile.pointer for row in unreads: print(row)
def get_raw_unread_data(user_profile: UserProfile) -> RawUnreadMessagesResult: excluded_recipient_ids = get_inactive_recipient_ids(user_profile) user_msgs = UserMessage.objects.filter(user_profile=user_profile).exclude( message__recipient_id__in=excluded_recipient_ids).extra( where=[UserMessage.where_unread()]).values( 'message_id', 'message__sender_id', MESSAGE__TOPIC, 'message__recipient_id', 'message__recipient__type', 'message__recipient__type_id', 'flags', ).order_by("-message_id") # Limit unread messages for performance reasons. user_msgs = list(user_msgs[:MAX_UNREAD_MESSAGES]) rows = list(reversed(user_msgs)) muted_stream_ids = get_muted_stream_ids(user_profile) topic_mute_checker = build_topic_mute_checker(user_profile) def is_row_muted(stream_id: int, recipient_id: int, topic: str) -> bool: if stream_id in muted_stream_ids: return True if topic_mute_checker(recipient_id, topic): return True return False huddle_cache = {} # type: Dict[int, str] def get_huddle_users(recipient_id: int) -> str: if recipient_id in huddle_cache: return huddle_cache[recipient_id] user_ids_string = huddle_users(recipient_id) huddle_cache[recipient_id] = user_ids_string return user_ids_string pm_dict = {} stream_dict = {} unmuted_stream_msgs = set() huddle_dict = {} mentions = set() for row in rows: message_id = row['message_id'] msg_type = row['message__recipient__type'] recipient_id = row['message__recipient_id'] sender_id = row['message__sender_id'] if msg_type == Recipient.STREAM: stream_id = row['message__recipient__type_id'] topic = row[MESSAGE__TOPIC] stream_dict[message_id] = dict( stream_id=stream_id, topic=topic, sender_id=sender_id, ) if not is_row_muted(stream_id, recipient_id, topic): unmuted_stream_msgs.add(message_id) elif msg_type == Recipient.PERSONAL: if sender_id == user_profile.id: other_user_id = row['message__recipient__type_id'] else: other_user_id = sender_id # The `sender_id` field here is misnamed. It's really # just the other participant in a PM conversation. For # most unread PM messages, the other user is also the sender, # but that's not true for certain messages sent from the # API. Unfortunately, it's difficult now to rename the # field without breaking mobile. pm_dict[message_id] = dict(sender_id=other_user_id, ) elif msg_type == Recipient.HUDDLE: user_ids_string = get_huddle_users(recipient_id) huddle_dict[message_id] = dict(user_ids_string=user_ids_string, ) # TODO: Add support for alert words here as well. is_mentioned = (row['flags'] & UserMessage.flags.mentioned) != 0 is_wildcard_mentioned = (row['flags'] & UserMessage.flags.wildcard_mentioned) != 0 if is_mentioned: mentions.add(message_id) if is_wildcard_mentioned: if msg_type == Recipient.STREAM: stream_id = row['message__recipient__type_id'] topic = row[MESSAGE__TOPIC] if not is_row_muted(stream_id, recipient_id, topic): mentions.add(message_id) else: # nocoverage # TODO: Test wildcard mentions in PMs. mentions.add(message_id) return dict( pm_dict=pm_dict, stream_dict=stream_dict, muted_stream_ids=muted_stream_ids, unmuted_stream_msgs=unmuted_stream_msgs, huddle_dict=huddle_dict, mentions=mentions, )
def get_raw_unread_data(user_profile: UserProfile) -> RawUnreadMessagesResult: excluded_recipient_ids = get_inactive_recipient_ids(user_profile) user_msgs = UserMessage.objects.filter(user_profile=user_profile).exclude( message__recipient_id__in=excluded_recipient_ids).extra( where=[UserMessage.where_unread()]).values( 'message_id', 'message__sender_id', MESSAGE__TOPIC, 'message__recipient_id', 'message__recipient__type', 'message__recipient__type_id', 'flags', ).order_by("-message_id") # Limit unread messages for performance reasons. user_msgs = list(user_msgs[:MAX_UNREAD_MESSAGES]) rows = list(reversed(user_msgs)) muted_stream_ids = get_muted_stream_ids(user_profile) topic_mute_checker = build_topic_mute_checker(user_profile) def is_row_muted(stream_id: int, recipient_id: int, topic: str) -> bool: if stream_id in muted_stream_ids: return True if topic_mute_checker(recipient_id, topic): return True return False huddle_cache = {} # type: Dict[int, str] def get_huddle_users(recipient_id: int) -> str: if recipient_id in huddle_cache: return huddle_cache[recipient_id] user_ids_string = huddle_users(recipient_id) huddle_cache[recipient_id] = user_ids_string return user_ids_string pm_dict = {} stream_dict = {} unmuted_stream_msgs = set() huddle_dict = {} mentions = set() for row in rows: message_id = row['message_id'] msg_type = row['message__recipient__type'] recipient_id = row['message__recipient_id'] sender_id = row['message__sender_id'] if msg_type == Recipient.STREAM: stream_id = row['message__recipient__type_id'] topic = row[MESSAGE__TOPIC] stream_dict[message_id] = dict( stream_id=stream_id, topic=topic, sender_id=sender_id, ) if not is_row_muted(stream_id, recipient_id, topic): unmuted_stream_msgs.add(message_id) elif msg_type == Recipient.PERSONAL: pm_dict[message_id] = dict(sender_id=sender_id, ) elif msg_type == Recipient.HUDDLE: user_ids_string = get_huddle_users(recipient_id) huddle_dict[message_id] = dict(user_ids_string=user_ids_string, ) # TODO: Add support for alert words here as well. is_mentioned = (row['flags'] & UserMessage.flags.mentioned) != 0 is_wildcard_mentioned = (row['flags'] & UserMessage.flags.wildcard_mentioned) != 0 if is_mentioned: mentions.add(message_id) if is_wildcard_mentioned: if msg_type == Recipient.STREAM: stream_id = row['message__recipient__type_id'] topic = row[MESSAGE__TOPIC] if not is_row_muted(stream_id, recipient_id, topic): mentions.add(message_id) else: # nocoverage # TODO: Test wildcard mentions in PMs. mentions.add(message_id) return dict( pm_dict=pm_dict, stream_dict=stream_dict, muted_stream_ids=muted_stream_ids, unmuted_stream_msgs=unmuted_stream_msgs, huddle_dict=huddle_dict, mentions=mentions, )
def extract_unread_data_from_um_rows( rows: List[Dict[str, Any]], user_profile: Optional[UserProfile]) -> RawUnreadMessagesResult: pm_dict: Dict[int, Any] = {} stream_dict: Dict[int, Any] = {} unmuted_stream_msgs: Set[int] = set() huddle_dict: Dict[int, Any] = {} mentions: Set[int] = set() total_unreads = 0 raw_unread_messages: RawUnreadMessagesResult = dict( pm_dict=pm_dict, stream_dict=stream_dict, muted_stream_ids=[], unmuted_stream_msgs=unmuted_stream_msgs, huddle_dict=huddle_dict, mentions=mentions, old_unreads_missing=False, ) if user_profile is None: return raw_unread_messages muted_stream_ids = get_muted_stream_ids(user_profile) raw_unread_messages["muted_stream_ids"] = muted_stream_ids topic_mute_checker = build_topic_mute_checker(user_profile) def is_row_muted(stream_id: int, recipient_id: int, topic: str) -> bool: if stream_id in muted_stream_ids: return True if topic_mute_checker(recipient_id, topic): return True # Messages sent by muted users are never unread, so we don't # need any logic related to muted users here. return False huddle_cache: Dict[int, str] = {} def get_huddle_users(recipient_id: int) -> str: if recipient_id in huddle_cache: return huddle_cache[recipient_id] user_ids_string = huddle_users(recipient_id) huddle_cache[recipient_id] = user_ids_string return user_ids_string for row in rows: total_unreads += 1 message_id = row["message_id"] msg_type = row["message__recipient__type"] recipient_id = row["message__recipient_id"] sender_id = row["message__sender_id"] if msg_type == Recipient.STREAM: stream_id = row["message__recipient__type_id"] topic = row[MESSAGE__TOPIC] stream_dict[message_id] = dict( stream_id=stream_id, topic=topic, sender_id=sender_id, ) if not is_row_muted(stream_id, recipient_id, topic): unmuted_stream_msgs.add(message_id) elif msg_type == Recipient.PERSONAL: if sender_id == user_profile.id: other_user_id = row["message__recipient__type_id"] else: other_user_id = sender_id # The `sender_id` field here is misnamed. It's really # just the other participant in a PM conversation. For # most unread PM messages, the other user is also the sender, # but that's not true for certain messages sent from the # API. Unfortunately, it's difficult now to rename the # field without breaking mobile. pm_dict[message_id] = dict(sender_id=other_user_id, ) elif msg_type == Recipient.HUDDLE: user_ids_string = get_huddle_users(recipient_id) huddle_dict[message_id] = dict(user_ids_string=user_ids_string, ) # TODO: Add support for alert words here as well. is_mentioned = (row["flags"] & UserMessage.flags.mentioned) != 0 is_wildcard_mentioned = (row["flags"] & UserMessage.flags.wildcard_mentioned) != 0 if is_mentioned: mentions.add(message_id) if is_wildcard_mentioned: if msg_type == Recipient.STREAM: stream_id = row["message__recipient__type_id"] topic = row[MESSAGE__TOPIC] if not is_row_muted(stream_id, recipient_id, topic): mentions.add(message_id) else: # nocoverage # TODO: Test wildcard mentions in PMs. mentions.add(message_id) # Record whether the user had more than MAX_UNREAD_MESSAGES total # unreads -- that's a state where Zulip's behavior will start to # be erroneous, and clients should display a warning. raw_unread_messages[ "old_unreads_missing"] = total_unreads == MAX_UNREAD_MESSAGES return raw_unread_messages
def extract_unread_data_from_um_rows( rows: List[Dict[str, Any]], user_profile: Optional[UserProfile] ) -> RawUnreadMessagesResult: pm_dict: Dict[int, Any] = {} stream_dict: Dict[int, Any] = {} unmuted_stream_msgs: Set[int] = set() huddle_dict: Dict[int, Any] = {} mentions: Set[int] = set() raw_unread_messages: RawUnreadMessagesResult = dict( pm_dict=pm_dict, stream_dict=stream_dict, muted_stream_ids=[], unmuted_stream_msgs=unmuted_stream_msgs, huddle_dict=huddle_dict, mentions=mentions, ) if user_profile is None: return raw_unread_messages # nocoverage muted_stream_ids = get_muted_stream_ids(user_profile) raw_unread_messages['muted_stream_ids'] = muted_stream_ids topic_mute_checker = build_topic_mute_checker(user_profile) def is_row_muted(stream_id: int, recipient_id: int, topic: str) -> bool: if stream_id in muted_stream_ids: return True if topic_mute_checker(recipient_id, topic): return True return False huddle_cache: Dict[int, str] = {} def get_huddle_users(recipient_id: int) -> str: if recipient_id in huddle_cache: return huddle_cache[recipient_id] user_ids_string = huddle_users(recipient_id) huddle_cache[recipient_id] = user_ids_string return user_ids_string for row in rows: message_id = row['message_id'] msg_type = row['message__recipient__type'] recipient_id = row['message__recipient_id'] sender_id = row['message__sender_id'] if msg_type == Recipient.STREAM: stream_id = row['message__recipient__type_id'] topic = row[MESSAGE__TOPIC] stream_dict[message_id] = dict( stream_id=stream_id, topic=topic, sender_id=sender_id, ) if not is_row_muted(stream_id, recipient_id, topic): unmuted_stream_msgs.add(message_id) elif msg_type == Recipient.PERSONAL: if sender_id == user_profile.id: other_user_id = row['message__recipient__type_id'] else: other_user_id = sender_id # The `sender_id` field here is misnamed. It's really # just the other participant in a PM conversation. For # most unread PM messages, the other user is also the sender, # but that's not true for certain messages sent from the # API. Unfortunately, it's difficult now to rename the # field without breaking mobile. pm_dict[message_id] = dict( sender_id=other_user_id, ) elif msg_type == Recipient.HUDDLE: user_ids_string = get_huddle_users(recipient_id) huddle_dict[message_id] = dict( user_ids_string=user_ids_string, ) # TODO: Add support for alert words here as well. is_mentioned = (row['flags'] & UserMessage.flags.mentioned) != 0 is_wildcard_mentioned = (row['flags'] & UserMessage.flags.wildcard_mentioned) != 0 if is_mentioned: mentions.add(message_id) if is_wildcard_mentioned: if msg_type == Recipient.STREAM: stream_id = row['message__recipient__type_id'] topic = row[MESSAGE__TOPIC] if not is_row_muted(stream_id, recipient_id, topic): mentions.add(message_id) else: # nocoverage # TODO: Test wildcard mentions in PMs. mentions.add(message_id) return raw_unread_messages
def fix_pre_pointer(cursor, user_profile): # type: (CursorObj, UserProfile) -> None pointer = user_profile.pointer if not pointer: return is_topic_muted = build_topic_mute_checker(user_profile) recipient_ids = [] def find_non_muted_recipients(): # type: () -> None query = ''' SELECT zerver_subscription.recipient_id FROM zerver_subscription INNER JOIN zerver_recipient ON ( zerver_recipient.id = zerver_subscription.recipient_id ) WHERE ( zerver_subscription.user_profile_id = '%s' AND zerver_recipient.type = 2 AND zerver_subscription.in_home_view AND zerver_subscription.active ) ''' cursor.execute(query, [user_profile.id]) rows = cursor.fetchall() for row in rows: recipient_ids.append(row[0]) print(recipient_ids) get_timing( 'find_non_muted_recipients', find_non_muted_recipients ) if not recipient_ids: return user_message_ids = [] def find_old_ids(): # type: () -> None recips = ', '.join(str(id) for id in recipient_ids) query = ''' SELECT zerver_usermessage.id, zerver_recipient.type_id, subject FROM zerver_usermessage INNER JOIN zerver_message ON ( zerver_message.id = zerver_usermessage.message_id ) INNER JOIN zerver_recipient ON ( zerver_recipient.id = zerver_message.recipient_id ) WHERE ( zerver_usermessage.user_profile_id = %s AND zerver_usermessage.message_id <= %s AND (zerver_usermessage.flags & 1) = 0 AND zerver_message.recipient_id in (%s) ) ''' % (user_profile.id, pointer, recips) print(''' EXPLAIN analyze''' + query.rstrip() + ';') cursor.execute(query) rows = cursor.fetchall() for (um_id, stream_id, topic) in rows: if not is_topic_muted(stream_id, topic): user_message_ids.append(um_id) print('rows found: %d' % (len(user_message_ids),)) get_timing( 'finding pre-pointer messages that are not muted', find_old_ids ) if not user_message_ids: return def fix(): # type: () -> None update_unread_flags(cursor, user_message_ids) get_timing( 'fixing unread messages for pre-pointer non-muted messages', fix )
def get_raw_unread_data(user_profile): # type: (UserProfile) -> Dict[str, Any] excluded_recipient_ids = get_inactive_recipient_ids(user_profile) user_msgs = UserMessage.objects.filter( user_profile=user_profile ).exclude( message__recipient_id__in=excluded_recipient_ids ).extra( where=[UserMessage.where_unread()] ).values( 'message_id', 'message__sender_id', 'message__subject', 'message__recipient_id', 'message__recipient__type', 'message__recipient__type_id', 'flags', ).order_by("-message_id") # Limit unread messages for performance reasons. user_msgs = list(user_msgs[:MAX_UNREAD_MESSAGES]) rows = list(reversed(user_msgs)) muted_stream_ids = get_muted_stream_ids(user_profile) topic_mute_checker = build_topic_mute_checker(user_profile) def is_row_muted(stream_id, recipient_id, topic): # type: (int, int, Text) -> bool if stream_id in muted_stream_ids: return True if topic_mute_checker(recipient_id, topic): return True return False huddle_cache = {} # type: Dict[int, str] def get_huddle_users(recipient_id): # type: (int) -> str if recipient_id in huddle_cache: return huddle_cache[recipient_id] user_ids_string = huddle_users(recipient_id) huddle_cache[recipient_id] = user_ids_string return user_ids_string pm_dict = {} stream_dict = {} unmuted_stream_msgs = set() huddle_dict = {} mentions = set() for row in rows: message_id = row['message_id'] msg_type = row['message__recipient__type'] recipient_id = row['message__recipient_id'] sender_id = row['message__sender_id'] if msg_type == Recipient.STREAM: stream_id = row['message__recipient__type_id'] topic = row['message__subject'] stream_dict[message_id] = dict( stream_id=stream_id, topic=topic, sender_id=sender_id, ) if not is_row_muted(stream_id, recipient_id, topic): unmuted_stream_msgs.add(message_id) elif msg_type == Recipient.PERSONAL: pm_dict[message_id] = dict( sender_id=sender_id, ) elif msg_type == Recipient.HUDDLE: user_ids_string = get_huddle_users(recipient_id) huddle_dict[message_id] = dict( user_ids_string=user_ids_string, ) is_mentioned = (row['flags'] & UserMessage.flags.mentioned) != 0 if is_mentioned: mentions.add(message_id) return dict( pm_dict=pm_dict, stream_dict=stream_dict, muted_stream_ids=muted_stream_ids, unmuted_stream_msgs=unmuted_stream_msgs, huddle_dict=huddle_dict, mentions=mentions, )
def get_raw_unread_data(user_profile: UserProfile) -> RawUnreadMessagesResult: excluded_recipient_ids = get_inactive_recipient_ids(user_profile) user_msgs = UserMessage.objects.filter( user_profile=user_profile ).exclude( message__recipient_id__in=excluded_recipient_ids ).extra( where=[UserMessage.where_unread()] ).values( 'message_id', 'message__sender_id', MESSAGE__TOPIC, 'message__recipient_id', 'message__recipient__type', 'message__recipient__type_id', 'flags', ).order_by("-message_id") # Limit unread messages for performance reasons. user_msgs = list(user_msgs[:MAX_UNREAD_MESSAGES]) rows = list(reversed(user_msgs)) muted_stream_ids = get_muted_stream_ids(user_profile) topic_mute_checker = build_topic_mute_checker(user_profile) def is_row_muted(stream_id: int, recipient_id: int, topic: str) -> bool: if stream_id in muted_stream_ids: return True if topic_mute_checker(recipient_id, topic): return True return False huddle_cache = {} # type: Dict[int, str] def get_huddle_users(recipient_id: int) -> str: if recipient_id in huddle_cache: return huddle_cache[recipient_id] user_ids_string = huddle_users(recipient_id) huddle_cache[recipient_id] = user_ids_string return user_ids_string pm_dict = {} stream_dict = {} unmuted_stream_msgs = set() huddle_dict = {} mentions = set() for row in rows: message_id = row['message_id'] msg_type = row['message__recipient__type'] recipient_id = row['message__recipient_id'] sender_id = row['message__sender_id'] if msg_type == Recipient.STREAM: stream_id = row['message__recipient__type_id'] topic = row[MESSAGE__TOPIC] stream_dict[message_id] = dict( stream_id=stream_id, topic=topic, sender_id=sender_id, ) if not is_row_muted(stream_id, recipient_id, topic): unmuted_stream_msgs.add(message_id) elif msg_type == Recipient.PERSONAL: pm_dict[message_id] = dict( sender_id=sender_id, ) elif msg_type == Recipient.HUDDLE: user_ids_string = get_huddle_users(recipient_id) huddle_dict[message_id] = dict( user_ids_string=user_ids_string, ) is_mentioned = (row['flags'] & UserMessage.flags.mentioned) != 0 if is_mentioned: mentions.add(message_id) return dict( pm_dict=pm_dict, stream_dict=stream_dict, muted_stream_ids=muted_stream_ids, unmuted_stream_msgs=unmuted_stream_msgs, huddle_dict=huddle_dict, mentions=mentions, )
def get_unread_message_ids_per_recipient(user_profile): # type: (UserProfile) -> UnreadMessagesResult excluded_recipient_ids = get_inactive_recipient_ids(user_profile) user_msgs = UserMessage.objects.filter(user_profile=user_profile).exclude( message__recipient_id__in=excluded_recipient_ids).extra( where=[UserMessage.where_unread()]).values( 'message_id', 'message__sender_id', 'message__subject', 'message__recipient_id', 'message__recipient__type', 'message__recipient__type_id', 'flags', ).order_by("-message_id") # Limit unread messages for performance reasons. user_msgs = list(user_msgs[:MAX_UNREAD_MESSAGES]) rows = list(reversed(user_msgs)) muted_recipient_ids = get_muted_recipient_ids(user_profile) topic_mute_checker = build_topic_mute_checker(user_profile) def is_row_muted(row): # type: (Dict[str, Any]) -> bool recipient_id = row['message__recipient_id'] if recipient_id in muted_recipient_ids: return True topic_name = row['message__subject'] if topic_mute_checker(recipient_id, topic_name): return True return False active_stream_rows = [row for row in rows if not is_row_muted(row)] count = len(active_stream_rows) pm_msgs = [ dict( sender_id=row['message__sender_id'], message_id=row['message_id'], ) for row in rows if row['message__recipient__type'] == Recipient.PERSONAL ] pm_objects = aggregate_dict( input_rows=pm_msgs, lookup_fields=[ 'sender_id', ], input_field='message_id', output_field='unread_message_ids', ) stream_msgs = [ dict( stream_id=row['message__recipient__type_id'], topic=row['message__subject'], message_id=row['message_id'], ) for row in rows if row['message__recipient__type'] == Recipient.STREAM ] stream_objects = aggregate_dict( input_rows=stream_msgs, lookup_fields=[ 'stream_id', 'topic', ], input_field='message_id', output_field='unread_message_ids', ) huddle_msgs = [ dict( recipient_id=row['message__recipient_id'], message_id=row['message_id'], ) for row in rows if row['message__recipient__type'] == Recipient.HUDDLE ] huddle_objects = aggregate_dict( input_rows=huddle_msgs, lookup_fields=[ 'recipient_id', ], input_field='message_id', output_field='unread_message_ids', ) for huddle in huddle_objects: huddle['user_ids_string'] = huddle_users(huddle['recipient_id']) del huddle['recipient_id'] mentioned_message_ids = [ row['message_id'] for row in rows if (row['flags'] & UserMessage.flags.mentioned) != 0 ] result = dict(pms=pm_objects, streams=stream_objects, huddles=huddle_objects, mentions=mentioned_message_ids, count=count) # type: UnreadMessagesResult return result