def get_raw_db_rows(needed_ids: List[int]) -> List[Dict[str, Any]]: # This is a special purpose function optimized for # callers like get_messages_backend(). fields = [ 'id', DB_TOPIC_NAME, 'pub_date', 'last_edit_time', 'edit_history', 'content', 'rendered_content', 'rendered_content_version', 'recipient_id', 'recipient__type', 'recipient__type_id', 'sender_id', 'sending_client__name', 'sender__realm_id', ] messages = Message.objects.filter(id__in=needed_ids).values(*fields) submessages = SubMessage.get_raw_db_rows(needed_ids) sew_messages_and_submessages(messages, submessages) reactions = Reaction.get_raw_db_rows(needed_ids) return sew_messages_and_reactions(messages, reactions)
def to_dict_uncached_helper(message, apply_markdown): # type: (Message, bool) -> Dict[str, Any] return MessageDict.build_message_dict( apply_markdown = apply_markdown, message = message, message_id = message.id, last_edit_time = message.last_edit_time, edit_history = message.edit_history, content = message.content, subject = message.subject, pub_date = message.pub_date, rendered_content = message.rendered_content, rendered_content_version = message.rendered_content_version, sender_id = message.sender.id, sender_email = message.sender.email, sender_realm_id = message.sender.realm_id, sender_realm_domain = message.sender.realm.domain, sender_full_name = message.sender.full_name, sender_short_name = message.sender.short_name, sender_avatar_source = message.sender.avatar_source, sender_is_mirror_dummy = message.sender.is_mirror_dummy, sending_client_name = message.sending_client.name, recipient_id = message.recipient.id, recipient_type = message.recipient.type, recipient_type_id = message.recipient.type_id, reactions = Reaction.get_raw_db_rows([message.id]) )
def get_raw_db_rows(needed_ids): # type: (List[int]) -> List[Dict[str, Any]] # This is a special purpose function optimized for # callers like get_messages_backend(). fields = [ 'id', 'subject', 'pub_date', 'last_edit_time', 'edit_history', 'content', 'rendered_content', 'rendered_content_version', 'recipient_id', 'recipient__type', 'recipient__type_id', 'sender_id', 'sending_client__name', 'sender__realm_id', ] messages = Message.objects.filter(id__in=needed_ids).values(*fields) """Adding one-many or Many-Many relationship in values results in N X results. Link: https://docs.djangoproject.com/en/1.8/ref/models/querysets/#values """ reactions = Reaction.get_raw_db_rows(needed_ids) return sew_messages_and_reactions(messages, reactions)
def to_dict_uncached_helper(message: Message) -> Dict[str, Any]: return MessageDict.build_message_dict( message = message, message_id = message.id, last_edit_time = message.last_edit_time, edit_history = message.edit_history, content = message.content, subject = message.subject, pub_date = message.pub_date, rendered_content = message.rendered_content, rendered_content_version = message.rendered_content_version, sender_id = message.sender.id, sender_realm_id = message.sender.realm_id, sending_client_name = message.sending_client.name, recipient_id = message.recipient.id, recipient_type = message.recipient.type, recipient_type_id = message.recipient.type_id, reactions = Reaction.get_raw_db_rows([message.id]) )
def build_reactions(realm_id: int, total_reactions: List[ZerverFieldsT], reactions: List[ZerverFieldsT], message_id: int, user_id_mapper: IdMapper, zerver_realmemoji: List[ZerverFieldsT]) -> None: realmemoji = {} for realm_emoji in zerver_realmemoji: realmemoji[realm_emoji['name']] = realm_emoji['id'] # For the unicode emoji codes, we use equivalent of # function 'emoji_name_to_emoji_code' in 'zerver/lib/emoji' here for mattermost_reaction in reactions: emoji_name = mattermost_reaction['emoji_name'] username = mattermost_reaction["user"] # Check in unicode emoji if emoji_name in name_to_codepoint: emoji_code = name_to_codepoint[emoji_name] reaction_type = Reaction.UNICODE_EMOJI # Check in realm emoji elif emoji_name in realmemoji: emoji_code = realmemoji[emoji_name] reaction_type = Reaction.REALM_EMOJI else: # nocoverage continue if not user_id_mapper.has(username): continue reaction_id = NEXT_ID('reaction') reaction = Reaction(id=reaction_id, emoji_code=emoji_code, emoji_name=emoji_name, reaction_type=reaction_type) reaction_dict = model_to_dict(reaction, exclude=['message', 'user_profile']) reaction_dict['message'] = message_id reaction_dict['user_profile'] = user_id_mapper.get(username) total_reactions.append(reaction_dict)
def build_reactions(reaction_list: List[ZerverFieldsT], reactions: List[ZerverFieldsT], added_users: AddedUsersT, message_id: int, reaction_id: int, name_to_codepoint: ZerverFieldsT, zerver_realmemoji: List[ZerverFieldsT]) -> int: realmemoji = {} for realm_emoji in zerver_realmemoji: realmemoji[realm_emoji['name']] = realm_emoji['id'] # For the unicode emoji codes, we use equivalent of # function 'emoji_name_to_emoji_code' in 'zerver/lib/emoji' here for slack_reaction in reactions: emoji_name = slack_reaction['name'] # Check in unicode emoji if emoji_name in name_to_codepoint: emoji_code = name_to_codepoint[emoji_name] reaction_type = Reaction.UNICODE_EMOJI # Check in realm emoji elif emoji_name in realmemoji: emoji_code = realmemoji[emoji_name] reaction_type = Reaction.REALM_EMOJI else: continue for user in slack_reaction['users']: reaction = Reaction(id=reaction_id, emoji_code=emoji_code, emoji_name=emoji_name, reaction_type=reaction_type) reaction_dict = model_to_dict(reaction, exclude=['message', 'user_profile']) reaction_dict['message'] = message_id reaction_dict['user_profile'] = added_users[user] reaction_id += 1 reaction_list.append(reaction_dict) return reaction_id
def test_events_sent_after_transaction_commits(self) -> None: """ Tests that `send_event` is hooked to `transaction.on_commit`. This is important, because we don't want to end up holding locks on message rows for too long if the event queue runs into a problem. """ hamlet = self.example_user("hamlet") self.send_stream_message(hamlet, "Scotland") message = self.get_last_message() reaction = Reaction( user_profile=hamlet, message=message, emoji_name="whatever", emoji_code="whatever", reaction_type="whatever", ) with self.tornado_redirected_to_list([], expected_num_events=1): with mock.patch("zerver.lib.actions.send_event") as m: m.side_effect = AssertionError( "Events should be sent only after the transaction commits." ) notify_reaction_update(hamlet, message, reaction, "stuff")
def bulk_create_reactions(all_messages: List[Message]) -> None: reactions: List[Reaction] = [] num_messages = int(0.2 * len(all_messages)) messages = random.sample(all_messages, num_messages) message_ids = [message.id for message in messages] message_to_users = get_message_to_users(message_ids) for message_id in message_ids: msg_user_ids = message_to_users[message_id] if msg_user_ids: # Now let between 1 and 7 users react. # # Ideally, we'd make exactly 1 reaction more common than # this algorithm generates. max_num_users = min(7, len(msg_user_ids)) num_users = random.randrange(1, max_num_users + 1) user_ids = random.sample(msg_user_ids, num_users) for user_id in user_ids: # each user does between 1 and 3 emojis num_emojis = random.choice([1, 2, 3]) emojis = random.sample(DEFAULT_EMOJIS, num_emojis) for emoji_name, emoji_code in emojis: reaction = Reaction( user_profile_id=user_id, message_id=message_id, emoji_name=emoji_name, emoji_code=emoji_code, reaction_type=Reaction.UNICODE_EMOJI, ) reactions.append(reaction) Reaction.objects.bulk_create(reactions)
def build_reactions(reaction_list: List[ZerverFieldsT], reactions: List[ZerverFieldsT], slack_user_id_to_zulip_user_id: SlackToZulipUserIDT, message_id: int, name_to_codepoint: ZerverFieldsT, zerver_realmemoji: List[ZerverFieldsT]) -> None: realmemoji = {} for realm_emoji in zerver_realmemoji: realmemoji[realm_emoji['name']] = realm_emoji['id'] # For the unicode emoji codes, we use equivalent of # function 'emoji_name_to_emoji_code' in 'zerver/lib/emoji' here for slack_reaction in reactions: emoji_name = slack_reaction['name'] if emoji_name in name_to_codepoint: emoji_code = name_to_codepoint[emoji_name] reaction_type = Reaction.UNICODE_EMOJI elif emoji_name in realmemoji: emoji_code = realmemoji[emoji_name] reaction_type = Reaction.REALM_EMOJI else: continue for slack_user_id in slack_reaction['users']: reaction_id = NEXT_ID('reaction') reaction = Reaction(id=reaction_id, emoji_code=emoji_code, emoji_name=emoji_name, reaction_type=reaction_type) reaction_dict = model_to_dict(reaction, exclude=['message', 'user_profile']) reaction_dict['message'] = message_id reaction_dict['user_profile'] = slack_user_id_to_zulip_user_id[ slack_user_id] reaction_list.append(reaction_dict)
def _add_random_reactions_to_message( message: Message, emojis: List[Tuple[str, str]], users: Optional[List[UserProfile]] = None, prob_reaction: float = 0.075, prob_upvote: float = 0.5, prob_repeat: float = 0.5) -> List[Reaction]: '''Randomly add emoji reactions to each message from a list. Algorithm: Give the message at least one reaction with probability `prob_reaction`. Once the first reaction is added, have another user upvote it with probability `prob_upvote`, provided there is another recipient of the message left to upvote. Repeat the process for a different emoji with probability `prob_repeat`. If the number of emojis or users is small, there is a chance the above process will produce multiple reactions with the same user and emoji, so group the reactions by emoji code and user profile and then return one reaction from each group. ''' for p in (prob_reaction, prob_repeat, prob_upvote): # Prevent p=1 since for prob_repeat and prob_upvote, this will # lead to an infinite loop. if p >= 1 or p < 0: raise ValueError('Probability argument must be between 0 and 1.') # Avoid performing database queries if there will be no reactions. compute_next_reaction: bool = random.random() < prob_reaction if not compute_next_reaction: return [] if users is None: users = [] user_ids: Sequence[int] = [user.id for user in users] if not user_ids: user_ids = UserMessage.objects.filter(message=message) \ .values_list("user_profile_id", flat=True) if not user_ids: return [] emojis = list(emojis) reactions = [] while compute_next_reaction: # We do this O(users) operation only if we've decided to do a # reaction, to avoid performance issues with large numbers of # users. users_available = set(user_ids) (emoji_name, emoji_code) = random.choice(emojis) while True: # Handle corner case where all the users have reacted. if not users_available: break user_id = random.choice(list(users_available)) reactions.append(Reaction( user_profile_id=user_id, message=message, emoji_name=emoji_name, emoji_code=emoji_code, reaction_type=Reaction.UNICODE_EMOJI )) users_available.remove(user_id) # Add an upvote with the defined probability. if not random.random() < prob_upvote: break # Repeat with a possibly different random emoji with the # defined probability. compute_next_reaction = random.random() < prob_repeat # Avoid returning duplicate reactions by deduplicating on # (user_profile_id, emoji_code). grouped_reactions = defaultdict(list) for reaction in reactions: k = (str(reaction.user_profile_id), str(reaction.emoji_code)) grouped_reactions[k].append(reaction) return [reactions[0] for reactions in grouped_reactions.values()]
def generate_and_send_messages(data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int]) -> int: (tot_messages, personals_pairs, options, output, random_seed) = data random.seed(random_seed) with open(os.path.join(get_or_create_dev_uuid_var_path('test-backend'), "test_messages.json")) as infile: dialog = ujson.load(infile) random.shuffle(dialog) texts = itertools.cycle(dialog) recipient_streams: List[int] = [ klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] recipient_huddles: List[int] = [h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)] huddle_members: Dict[int, List[int]] = {} for h in recipient_huddles: huddle_members[h] = [s.user_profile.id for s in Subscription.objects.filter(recipient_id=h)] # Generate different topics for each stream possible_topics = dict() for stream_id in recipient_streams: possible_topics[stream_id] = generate_topics(options["max_topics"]) message_batch_size = options['batch_size'] num_messages = 0 random_max = 1000000 recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {} messages = [] messages_add_reaction = [] while num_messages < tot_messages: saved_data: Dict[str, Any] = {} message = Message() message.sending_client = get_client('populate_db') message.content = next(texts) randkey = random.randint(1, random_max) if (num_messages > 0 and random.randint(1, random_max) * 100. / random_max < options["stickyness"]): # Use an old recipient message_type, recipient_id, saved_data = recipients[num_messages - 1] if message_type == Recipient.PERSONAL: personals_pair = saved_data['personals_pair'] random.shuffle(personals_pair) elif message_type == Recipient.STREAM: message.subject = saved_data['subject'] message.recipient = get_recipient_by_id(recipient_id) elif message_type == Recipient.HUDDLE: message.recipient = get_recipient_by_id(recipient_id) elif (randkey <= random_max * options["percent_huddles"] / 100.): message_type = Recipient.HUDDLE message.recipient = get_recipient_by_id(random.choice(recipient_huddles)) elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.): message_type = Recipient.PERSONAL personals_pair = random.choice(personals_pairs) random.shuffle(personals_pair) elif (randkey <= random_max * 1.0): message_type = Recipient.STREAM message.recipient = get_recipient_by_id(random.choice(recipient_streams)) if message_type == Recipient.HUDDLE: sender_id = random.choice(huddle_members[message.recipient.id]) message.sender = get_user_profile_by_id(sender_id) elif message_type == Recipient.PERSONAL: message.recipient = Recipient.objects.get(type=Recipient.PERSONAL, type_id=personals_pair[0]) message.sender = get_user_profile_by_id(personals_pair[1]) saved_data['personals_pair'] = personals_pair elif message_type == Recipient.STREAM: # Pick a random subscriber to the stream message.sender = random.choice(Subscription.objects.filter( recipient=message.recipient)).user_profile message.subject = random.choice(possible_topics[message.recipient.id]) saved_data['subject'] = message.subject message.date_sent = choose_date_sent(num_messages, tot_messages, options['threads']) messages.append(message) messages_add_reaction.append(message) recipients[num_messages] = (message_type, message.recipient.id, saved_data) num_messages += 1 if (num_messages % message_batch_size) == 0: # Send the batch and empty the list: send_messages(messages) messages = [] if len(messages) > 0: # If there are unsent messages after exiting the loop, send them: send_messages(messages) reactions_message = [] add_emojis = ["1f44d", "1f642", "1f60a"] for rmessage in messages_add_reaction: if random.random() > 0.9: reactedmessage = Reaction(user_profile=rmessage.sender, message=rmessage, emoji_name="+1", emoji_code=add_emojis[random.randint(0, 2)], reaction_type="unicode_emoji") reactions_message.append(reactedmessage) Reaction.objects.bulk_create(reactions_message) return tot_messages