def generate_and_send_messages( data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int] ) -> int: (tot_messages, personals_pairs, options, output, random_seed) = data random.seed(random_seed) with open( os.path.join(get_or_create_dev_uuid_var_path('test-backend'), "test_messages.json"), "rb") as infile: dialog = orjson.loads(infile.read()) random.shuffle(dialog) texts = itertools.cycle(dialog) recipient_streams: List[int] = [ klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] recipient_huddles: List[int] = [ h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE) ] huddle_members: Dict[int, List[int]] = {} for h in recipient_huddles: huddle_members[h] = [ s.user_profile.id for s in Subscription.objects.filter(recipient_id=h) ] # Generate different topics for each stream possible_topics = dict() for stream_id in recipient_streams: possible_topics[stream_id] = generate_topics(options["max_topics"]) message_batch_size = options['batch_size'] num_messages = 0 random_max = 1000000 recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {} messages = [] while num_messages < tot_messages: saved_data: Dict[str, Any] = {} message = Message() message.sending_client = get_client('populate_db') message.content = next(texts) randkey = random.randint(1, random_max) if (num_messages > 0 and random.randint(1, random_max) * 100. / random_max < options["stickyness"]): # Use an old recipient message_type, recipient_id, saved_data = recipients[num_messages - 1] if message_type == Recipient.PERSONAL: personals_pair = saved_data['personals_pair'] random.shuffle(personals_pair) elif message_type == Recipient.STREAM: message.subject = saved_data['subject'] message.recipient = get_recipient_by_id(recipient_id) elif message_type == Recipient.HUDDLE: message.recipient = get_recipient_by_id(recipient_id) elif (randkey <= random_max * options["percent_huddles"] / 100.): message_type = Recipient.HUDDLE message.recipient = get_recipient_by_id( random.choice(recipient_huddles)) elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.): message_type = Recipient.PERSONAL personals_pair = random.choice(personals_pairs) random.shuffle(personals_pair) elif (randkey <= random_max * 1.0): message_type = Recipient.STREAM message.recipient = get_recipient_by_id( random.choice(recipient_streams)) if message_type == Recipient.HUDDLE: sender_id = random.choice(huddle_members[message.recipient.id]) message.sender = get_user_profile_by_id(sender_id) elif message_type == Recipient.PERSONAL: message.recipient = Recipient.objects.get( type=Recipient.PERSONAL, type_id=personals_pair[0]) message.sender = get_user_profile_by_id(personals_pair[1]) saved_data['personals_pair'] = personals_pair elif message_type == Recipient.STREAM: # Pick a random subscriber to the stream message.sender = random.choice( Subscription.objects.filter( recipient=message.recipient)).user_profile message.subject = random.choice( possible_topics[message.recipient.id]) saved_data['subject'] = message.subject message.date_sent = choose_date_sent(num_messages, tot_messages, options['threads']) messages.append(message) recipients[num_messages] = (message_type, message.recipient.id, saved_data) num_messages += 1 if (num_messages % message_batch_size) == 0: # Send the batch and empty the list: send_messages(messages) messages = [] if len(messages) > 0: # If there are unsent messages after exiting the loop, send them: send_messages(messages) return tot_messages
def generate_and_send_messages( data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int] ) -> int: (tot_messages, personals_pairs, options, output, random_seed) = data random.seed(random_seed) with open( os.path.join(get_or_create_dev_uuid_var_path("test-backend"), "test_messages.json"), "rb") as infile: dialog = orjson.loads(infile.read()) random.shuffle(dialog) texts = itertools.cycle(dialog) # We need to filter out streams from the analytics realm as we don't want to generate # messages to its streams - and they might also have no subscribers, which would break # our message generation mechanism below. stream_ids = Stream.objects.filter(realm=get_realm("zulip")).values_list( "id", flat=True) recipient_streams: List[int] = [ recipient.id for recipient in Recipient.objects.filter(type=Recipient.STREAM, type_id__in=stream_ids) ] recipient_huddles: List[int] = [ h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE) ] huddle_members: Dict[int, List[int]] = {} for h in recipient_huddles: huddle_members[h] = [ s.user_profile.id for s in Subscription.objects.filter(recipient_id=h) ] # Generate different topics for each stream possible_topics = {} for stream_id in recipient_streams: possible_topics[stream_id] = generate_topics(options["max_topics"]) message_batch_size = options["batch_size"] num_messages = 0 random_max = 1000000 recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {} messages: List[Message] = [] while num_messages < tot_messages: saved_data: Dict[str, Any] = {} message = Message() message.sending_client = get_client("populate_db") message.content = next(texts) randkey = random.randint(1, random_max) if (num_messages > 0 and random.randint(1, random_max) * 100.0 / random_max < options["stickyness"]): # Use an old recipient message_type, recipient_id, saved_data = recipients[num_messages - 1] if message_type == Recipient.PERSONAL: personals_pair = saved_data["personals_pair"] random.shuffle(personals_pair) elif message_type == Recipient.STREAM: message.subject = saved_data["subject"] message.recipient = get_recipient_by_id(recipient_id) elif message_type == Recipient.HUDDLE: message.recipient = get_recipient_by_id(recipient_id) elif randkey <= random_max * options["percent_huddles"] / 100.0: message_type = Recipient.HUDDLE message.recipient = get_recipient_by_id( random.choice(recipient_huddles)) elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.0): message_type = Recipient.PERSONAL personals_pair = random.choice(personals_pairs) random.shuffle(personals_pair) elif randkey <= random_max * 1.0: message_type = Recipient.STREAM message.recipient = get_recipient_by_id( random.choice(recipient_streams)) if message_type == Recipient.HUDDLE: sender_id = random.choice(huddle_members[message.recipient.id]) message.sender = get_user_profile_by_id(sender_id) elif message_type == Recipient.PERSONAL: message.recipient = Recipient.objects.get( type=Recipient.PERSONAL, type_id=personals_pair[0]) message.sender = get_user_profile_by_id(personals_pair[1]) saved_data["personals_pair"] = personals_pair elif message_type == Recipient.STREAM: # Pick a random subscriber to the stream message.sender = random.choice( Subscription.objects.filter( recipient=message.recipient)).user_profile message.subject = random.choice( possible_topics[message.recipient.id]) saved_data["subject"] = message.subject message.date_sent = choose_date_sent(num_messages, tot_messages, options["threads"]) messages.append(message) recipients[num_messages] = (message_type, message.recipient.id, saved_data) num_messages += 1 if (num_messages % message_batch_size) == 0: # Send the batch and empty the list: send_messages(messages) messages = [] if len(messages) > 0: # If there are unsent messages after exiting the loop, send them: send_messages(messages) return tot_messages