def construct_message(
self, scheduled_message: ScheduledMessage) -> Dict[str, Any]:
message = Message()
original_sender = scheduled_message.sender
message.content = scheduled_message.content
message.recipient = scheduled_message.recipient
message.subject = scheduled_message.subject
message.pub_date = timezone_now()
message.sending_client = scheduled_message.sending_client
delivery_type = scheduled_message.delivery_type
if delivery_type == ScheduledMessage.SEND_LATER:
message.sender = original_sender
elif delivery_type == ScheduledMessage.REMIND:
message.sender = get_user(settings.REMINDER_BOT,
original_sender.realm)
whos_reminding = ('%s asked me to do a reminder about:\n' %
(original_sender.full_name))
message.content = whos_reminding + message.content
return {
'message': message,
'stream': scheduled_message.stream,
'realm': scheduled_message.realm
}
def render_message_backend(request, user_profile, content=REQ()):
# type: (HttpRequest, UserProfile, Text) -> HttpResponse
message = Message()
message.sender = user_profile
message.content = content
message.sending_client = request.client
rendered_content = render_markdown(message, content, realm=user_profile.realm)
return json_success({"rendered": rendered_content})
def render_message_backend(
request: HttpRequest, user_profile: UserProfile,
content: str = REQ()) -> HttpResponse:
message = Message()
message.sender = user_profile
message.content = content
message.sending_client = request.client
rendering_result = render_markdown(message,
content,
realm=user_profile.realm)
return json_success({"rendered": rendering_result.rendered_content})
def render_message_backend(
request: HttpRequest, user_profile: UserProfile, content: str = REQ()
) -> HttpResponse:
message = Message()
message.sender = user_profile
message.content = content
client = RequestNotes.get_notes(request).client
assert client is not None
message.sending_client = client
rendering_result = render_markdown(message, content, realm=user_profile.realm)
return json_success(request, data={"rendered": rendering_result.rendered_content})
def construct_message(
self, scheduled_message: ScheduledMessage) -> Dict[str, Any]:
message = Message()
message.sender = scheduled_message.sender
message.content = scheduled_message.content
message.recipient = scheduled_message.recipient
message.subject = scheduled_message.subject
message.pub_date = timezone_now()
message.sending_client = scheduled_message.sending_client
return {
'message': message,
'stream': scheduled_message.stream,
'realm': scheduled_message.realm
}
def construct_message(self, scheduled_message: ScheduledMessage) -> Dict[str, Any]:
message = Message()
original_sender = scheduled_message.sender
message.content = scheduled_message.content
message.recipient = scheduled_message.recipient
message.subject = scheduled_message.subject
message.pub_date = timezone_now()
message.sending_client = scheduled_message.sending_client
delivery_type = scheduled_message.delivery_type
if delivery_type == ScheduledMessage.SEND_LATER:
message.sender = original_sender
elif delivery_type == ScheduledMessage.REMIND:
message.sender = get_user(settings.REMINDER_BOT, original_sender.realm)
return {'message': message, 'stream': scheduled_message.stream,
'realm': scheduled_message.realm}
def construct_message(
self, scheduled_message: ScheduledMessage) -> SendMessageRequest:
message = Message()
original_sender = scheduled_message.sender
message.content = scheduled_message.content
message.recipient = scheduled_message.recipient
message.subject = scheduled_message.subject
message.date_sent = timezone_now()
message.sending_client = scheduled_message.sending_client
delivery_type = scheduled_message.delivery_type
if delivery_type == ScheduledMessage.SEND_LATER:
message.sender = original_sender
elif delivery_type == ScheduledMessage.REMIND:
message.sender = get_user_by_delivery_email(
settings.NOTIFICATION_BOT, original_sender.realm)
return build_message_send_dict(message=message,
stream=scheduled_message.stream,
realm=scheduled_message.realm)
def construct_message(
self, scheduled_message: ScheduledMessage) -> Dict[str, Any]:
message = Message()
original_sender = scheduled_message.sender
message.content = scheduled_message.content
message.recipient = scheduled_message.recipient
message.subject = scheduled_message.subject
message.date_sent = timezone_now()
message.sending_client = scheduled_message.sending_client
delivery_type = scheduled_message.delivery_type
if delivery_type == ScheduledMessage.SEND_LATER:
message.sender = original_sender
elif delivery_type == ScheduledMessage.REMIND:
message.sender = get_user_by_delivery_email(
settings.NOTIFICATION_BOT, original_sender.realm)
return {
'message': message,
'stream': scheduled_message.stream,
'realm': scheduled_message.realm
}
def generate_and_send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open(
os.path.join(get_or_create_dev_uuid_var_path("test-backend"),
"test_messages.json"), "rb") as infile:
dialog = orjson.loads(infile.read())
random.shuffle(dialog)
texts = itertools.cycle(dialog)
# We need to filter out streams from the analytics realm as we don't want to generate
# messages to its streams - and they might also have no subscribers, which would break
# our message generation mechanism below.
stream_ids = Stream.objects.filter(realm=get_realm("zulip")).values_list(
"id", flat=True)
recipient_streams: List[int] = [
recipient.id
for recipient in Recipient.objects.filter(type=Recipient.STREAM,
type_id__in=stream_ids)
]
recipient_huddles: List[int] = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
]
huddle_members: Dict[int, List[int]] = {}
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
# Generate different topics for each stream
possible_topics = {}
for stream_id in recipient_streams:
possible_topics[stream_id] = generate_topics(options["max_topics"])
message_batch_size = options["batch_size"]
num_messages = 0
random_max = 1000000
recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {}
messages: List[Message] = []
while num_messages < tot_messages:
saved_data: Dict[str, Any] = {}
message = Message()
message.sending_client = get_client("populate_db")
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100.0 / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data["personals_pair"]
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data["subject"]
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif randkey <= random_max * options["percent_huddles"] / 100.0:
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.0):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif randkey <= random_max * 1.0:
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data["personals_pair"] = personals_pair
elif message_type == Recipient.STREAM:
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = random.choice(
possible_topics[message.recipient.id])
saved_data["subject"] = message.subject
message.date_sent = choose_date_sent(num_messages, tot_messages,
options["threads"])
messages.append(message)
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
if (num_messages % message_batch_size) == 0:
# Send the batch and empty the list:
send_messages(messages)
messages = []
if len(messages) > 0:
# If there are unsent messages after exiting the loop, send them:
send_messages(messages)
return tot_messages
def send_messages(data):
# type: (Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int]) -> int
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open("var/test_messages.json", "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
] # type: List[int]
recipient_huddles = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100. / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + Text(random.randint(1, 3))
saved_data['subject'] = message.subject
message.pub_date = timezone_now()
do_send_messages([{'message': message}])
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
return tot_messages
def send_messages(data):
# type: (Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any]]) -> int
(tot_messages, personals_pairs, options, output) = data
random.seed(os.getpid())
texts = open("zilencer/management/commands/test_messages.txt", "r").readlines()
offset = random.randint(0, len(texts))
recipient_streams = [klass.id for klass in
Recipient.objects.filter(type=Recipient.STREAM)] # type: List[int]
recipient_huddles = [h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [s.user_profile.id for s in
Subscription.objects.filter(recipient_id=h)]
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
length = random.randint(1, 5)
lines = (t.strip() for t in texts[offset: offset + length])
message.content = '\n'.join(lines)
offset += length
offset = offset % len(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0 and
random.randint(1, random_max) * 100. / random_max < options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages - 1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(random.choice(recipient_huddles))
elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(type=Recipient.PERSONAL,
type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + text_type(random.randint(1, 3))
saved_data['subject'] = message.subject
message.pub_date = now()
do_send_message(message)
recipients[num_messages] = (message_type, message.recipient.id, saved_data)
num_messages += 1
return tot_messages
def send_messages(data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open("var/test_messages.json", "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [klass.id for klass in
Recipient.objects.filter(type=Recipient.STREAM)] # type: List[int]
recipient_huddles = [h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [s.user_profile.id for s in
Subscription.objects.filter(recipient_id=h)]
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0 and
random.randint(1, random_max) * 100. / random_max < options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages - 1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(random.choice(recipient_huddles))
elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(type=Recipient.PERSONAL,
type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + str(random.randint(1, 3))
saved_data['subject'] = message.subject
# Spoofing time not supported with threading
if options['threads'] != 1:
message.pub_date = timezone_now()
else:
# Distrubutes 80% of messages starting from 5 days ago, over a period
# of 3 days. Then, distributes remaining messages over past 24 hours.
spoofed_date = timezone_now() - timezone_timedelta(days = 5)
if (num_messages < tot_messages * 0.8):
# Maximum of 3 days ahead, convert to minutes
time_ahead = 3 * 24 * 60
time_ahead //= int(tot_messages * 0.8)
else:
time_ahead = 24 * 60
time_ahead //= int(tot_messages * 0.2)
spoofed_minute = random.randint(time_ahead * num_messages, time_ahead * (num_messages + 1))
spoofed_date += timezone_timedelta(minutes = spoofed_minute)
message.pub_date = spoofed_date
# We disable USING_RABBITMQ here, so that deferred work is
# executed in do_send_message_messages, rather than being
# queued. This is important, because otherwise, if run-dev.py
# wasn't running when populate_db was run, a developer can end
# up with queued events that reference objects from a previous
# life of the database, which naturally throws exceptions.
settings.USING_RABBITMQ = False
do_send_messages([{'message': message}])
settings.USING_RABBITMQ = True
recipients[num_messages] = (message_type, message.recipient.id, saved_data)
num_messages += 1
return tot_messages
def generate_and_send_messages(data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open(os.path.join(get_or_create_dev_uuid_var_path('test-backend'),
"test_messages.json")) as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams: List[int] = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
recipient_huddles: List[int] = [h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)]
huddle_members: Dict[int, List[int]] = {}
for h in recipient_huddles:
huddle_members[h] = [s.user_profile.id for s in
Subscription.objects.filter(recipient_id=h)]
# Generate different topics for each stream
possible_topics = dict()
for stream_id in recipient_streams:
possible_topics[stream_id] = generate_topics(options["max_topics"])
message_batch_size = options['batch_size']
num_messages = 0
random_max = 1000000
recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {}
messages = []
messages_add_reaction = []
while num_messages < tot_messages:
saved_data: Dict[str, Any] = {}
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0 and
random.randint(1, random_max) * 100. / random_max < options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages - 1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(random.choice(recipient_huddles))
elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(type=Recipient.PERSONAL,
type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
# Pick a random subscriber to the stream
message.sender = random.choice(Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = random.choice(possible_topics[message.recipient.id])
saved_data['subject'] = message.subject
message.date_sent = choose_date_sent(num_messages, tot_messages, options['threads'])
messages.append(message)
messages_add_reaction.append(message)
recipients[num_messages] = (message_type, message.recipient.id, saved_data)
num_messages += 1
if (num_messages % message_batch_size) == 0:
# Send the batch and empty the list:
send_messages(messages)
messages = []
if len(messages) > 0:
# If there are unsent messages after exiting the loop, send them:
send_messages(messages)
reactions_message = []
add_emojis = ["1f44d", "1f642", "1f60a"]
for rmessage in messages_add_reaction:
if random.random() > 0.9:
reactedmessage = Reaction(user_profile=rmessage.sender,
message=rmessage,
emoji_name="+1",
emoji_code=add_emojis[random.randint(0, 2)],
reaction_type="unicode_emoji")
reactions_message.append(reactedmessage)
Reaction.objects.bulk_create(reactions_message)
return tot_messages
def send_messages(data):
# type: (Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int]) -> int
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open("var/test_messages.json", "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [klass.id for klass in
Recipient.objects.filter(type=Recipient.STREAM)] # type: List[int]
recipient_huddles = [h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [s.user_profile.id for s in
Subscription.objects.filter(recipient_id=h)]
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0 and
random.randint(1, random_max) * 100. / random_max < options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages - 1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(random.choice(recipient_huddles))
elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(type=Recipient.PERSONAL,
type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + Text(random.randint(1, 3))
saved_data['subject'] = message.subject
message.pub_date = timezone_now()
do_send_messages([{'message': message}])
recipients[num_messages] = (message_type, message.recipient.id, saved_data)
num_messages += 1
return tot_messages
def send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open("var/test_messages.json", "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
] # type: List[int]
recipient_huddles = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100. / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + str(random.randint(1, 3))
saved_data['subject'] = message.subject
# Spoofing time not supported with threading
if options['threads'] != 1:
message.pub_date = timezone_now()
else:
# Distrubutes 80% of messages starting from 5 days ago, over a period
# of 3 days. Then, distributes remaining messages over past 24 hours.
spoofed_date = timezone_now() - timezone_timedelta(days=5)
if (num_messages < tot_messages * 0.8):
# Maximum of 3 days ahead, convert to minutes
time_ahead = 3 * 24 * 60
time_ahead //= int(tot_messages * 0.8)
else:
time_ahead = 24 * 60
time_ahead //= int(tot_messages * 0.2)
spoofed_minute = random.randint(time_ahead * num_messages,
time_ahead * (num_messages + 1))
spoofed_date += timezone_timedelta(minutes=spoofed_minute)
message.pub_date = spoofed_date
# We disable USING_RABBITMQ here, so that deferred work is
# executed in do_send_message_messages, rather than being
# queued. This is important, because otherwise, if run-dev.py
# wasn't running when populate_db was run, a developer can end
# up with queued events that reference objects from a previous
# life of the database, which naturally throws exceptions.
settings.USING_RABBITMQ = False
do_send_messages([{'message': message}])
settings.USING_RABBITMQ = True
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
return tot_messages
def do_update_message(
user_profile: UserProfile,
target_message: Message,
new_stream: Optional[Stream],
topic_name: Optional[str],
propagate_mode: Optional[str],
send_notification_to_old_thread: bool,
send_notification_to_new_thread: bool,
content: Optional[str],
rendering_result: Optional[MessageRenderingResult],
prior_mention_user_ids: Set[int],
mention_data: Optional[MentionData] = None,
) -> int:
"""
The main function for message editing. A message edit event can
modify:
* the message's content (in which case the caller will have
set both content and rendered_content),
* the topic, in which case the caller will have set topic_name
* or both message's content and the topic
* or stream and/or topic, in which case the caller will have set
new_stream and/or topic_name.
With topic edits, propagate_mode determines whether other message
also have their topics edited.
"""
timestamp = timezone_now()
target_message.last_edit_time = timestamp
event: Dict[str, Any] = {
"type": "update_message",
"user_id": user_profile.id,
"edit_timestamp": datetime_to_timestamp(timestamp),
"message_id": target_message.id,
"rendering_only": False,
}
edit_history_event: EditHistoryEvent = {
"user_id": user_profile.id,
"timestamp": event["edit_timestamp"],
}
changed_messages = [target_message]
realm = user_profile.realm
stream_being_edited = None
if target_message.is_stream_message():
stream_id = target_message.recipient.type_id
stream_being_edited = get_stream_by_id_in_realm(stream_id, realm)
event["stream_name"] = stream_being_edited.name
event["stream_id"] = stream_being_edited.id
ums = UserMessage.objects.filter(message=target_message.id)
if content is not None:
assert rendering_result is not None
# mention_data is required if there's a content edit.
assert mention_data is not None
# add data from group mentions to mentions_user_ids.
for group_id in rendering_result.mentions_user_group_ids:
members = mention_data.get_group_members(group_id)
rendering_result.mentions_user_ids.update(members)
update_user_message_flags(rendering_result, ums)
# One could imagine checking realm.allow_edit_history here and
# modifying the events based on that setting, but doing so
# doesn't really make sense. We need to send the edit event
# to clients regardless, and a client already had access to
# the original/pre-edit content of the message anyway. That
# setting must be enforced on the client side, and making a
# change here simply complicates the logic for clients parsing
# edit history events.
event["orig_content"] = target_message.content
event["orig_rendered_content"] = target_message.rendered_content
edit_history_event["prev_content"] = target_message.content
edit_history_event[
"prev_rendered_content"] = target_message.rendered_content
edit_history_event[
"prev_rendered_content_version"] = target_message.rendered_content_version
target_message.content = content
target_message.rendered_content = rendering_result.rendered_content
target_message.rendered_content_version = markdown_version
event["content"] = content
event["rendered_content"] = rendering_result.rendered_content
event[
"prev_rendered_content_version"] = target_message.rendered_content_version
event["is_me_message"] = Message.is_status_message(
content, rendering_result.rendered_content)
# target_message.has_image and target_message.has_link will have been
# already updated by Markdown rendering in the caller.
target_message.has_attachment = check_attachment_reference_change(
target_message, rendering_result)
if target_message.is_stream_message():
if topic_name is not None:
new_topic_name = topic_name
else:
new_topic_name = target_message.topic_name()
stream_topic: Optional[StreamTopicTarget] = StreamTopicTarget(
stream_id=stream_id,
topic_name=new_topic_name,
)
else:
stream_topic = None
info = get_recipient_info(
realm_id=realm.id,
recipient=target_message.recipient,
sender_id=target_message.sender_id,
stream_topic=stream_topic,
possible_wildcard_mention=mention_data.message_has_wildcards(),
)
event["online_push_user_ids"] = list(info["online_push_user_ids"])
event["pm_mention_push_disabled_user_ids"] = list(
info["pm_mention_push_disabled_user_ids"])
event["pm_mention_email_disabled_user_ids"] = list(
info["pm_mention_email_disabled_user_ids"])
event["stream_push_user_ids"] = list(info["stream_push_user_ids"])
event["stream_email_user_ids"] = list(info["stream_email_user_ids"])
event["muted_sender_user_ids"] = list(info["muted_sender_user_ids"])
event["prior_mention_user_ids"] = list(prior_mention_user_ids)
event["presence_idle_user_ids"] = filter_presence_idle_user_ids(
info["active_user_ids"])
event["all_bot_user_ids"] = list(info["all_bot_user_ids"])
if rendering_result.mentions_wildcard:
event["wildcard_mention_user_ids"] = list(
info["wildcard_mention_user_ids"])
else:
event["wildcard_mention_user_ids"] = []
do_update_mobile_push_notification(
target_message,
prior_mention_user_ids,
rendering_result.mentions_user_ids,
info["stream_push_user_ids"],
)
if topic_name is not None or new_stream is not None:
assert propagate_mode is not None
orig_topic_name = target_message.topic_name()
event["propagate_mode"] = propagate_mode
if new_stream is not None:
assert content is None
assert target_message.is_stream_message()
assert stream_being_edited is not None
edit_history_event["prev_stream"] = stream_being_edited.id
edit_history_event["stream"] = new_stream.id
event[ORIG_TOPIC] = orig_topic_name
assert new_stream.recipient_id is not None
target_message.recipient_id = new_stream.recipient_id
event["new_stream_id"] = new_stream.id
event["propagate_mode"] = propagate_mode
# When messages are moved from one stream to another, some
# users may lose access to those messages, including guest
# users and users not subscribed to the new stream (if it is a
# private stream). For those users, their experience is as
# though the messages were deleted, and we should send a
# delete_message event to them instead.
subs_to_old_stream = get_active_subscriptions_for_stream_id(
stream_id,
include_deactivated_users=True).select_related("user_profile")
subs_to_new_stream = list(
get_active_subscriptions_for_stream_id(
new_stream.id,
include_deactivated_users=True).select_related("user_profile"))
old_stream_sub_ids = [
user.user_profile_id for user in subs_to_old_stream
]
new_stream_sub_ids = [
user.user_profile_id for user in subs_to_new_stream
]
# Get users who aren't subscribed to the new_stream.
subs_losing_usermessages = [
sub for sub in subs_to_old_stream
if sub.user_profile_id not in new_stream_sub_ids
]
# Users who can longer access the message without some action
# from administrators.
subs_losing_access = [
sub for sub in subs_losing_usermessages
if sub.user_profile.is_guest or not new_stream.is_public()
]
ums = ums.exclude(user_profile_id__in=[
sub.user_profile_id for sub in subs_losing_usermessages
])
subs_gaining_usermessages = []
if not new_stream.is_history_public_to_subscribers():
# For private streams, with history not public to subscribers,
# We find out users who are not present in the msgs' old stream
# and create new UserMessage for these users so that they can
# access this message.
subs_gaining_usermessages += [
user_id for user_id in new_stream_sub_ids
if user_id not in old_stream_sub_ids
]
if topic_name is not None:
topic_name = truncate_topic(topic_name)
target_message.set_topic_name(topic_name)
# These fields have legacy field names.
event[ORIG_TOPIC] = orig_topic_name
event[TOPIC_NAME] = topic_name
event[TOPIC_LINKS] = topic_links(target_message.sender.realm_id,
topic_name)
edit_history_event["prev_topic"] = orig_topic_name
edit_history_event["topic"] = topic_name
update_edit_history(target_message, timestamp, edit_history_event)
delete_event_notify_user_ids: List[int] = []
if propagate_mode in ["change_later", "change_all"]:
assert topic_name is not None or new_stream is not None
assert stream_being_edited is not None
# Other messages should only get topic/stream fields in their edit history.
topic_only_edit_history_event: EditHistoryEvent = {
"user_id": edit_history_event["user_id"],
"timestamp": edit_history_event["timestamp"],
}
if topic_name is not None:
topic_only_edit_history_event["prev_topic"] = edit_history_event[
"prev_topic"]
topic_only_edit_history_event["topic"] = edit_history_event[
"topic"]
if new_stream is not None:
topic_only_edit_history_event["prev_stream"] = edit_history_event[
"prev_stream"]
topic_only_edit_history_event["stream"] = edit_history_event[
"stream"]
messages_list = update_messages_for_topic_edit(
acting_user=user_profile,
edited_message=target_message,
propagate_mode=propagate_mode,
orig_topic_name=orig_topic_name,
topic_name=topic_name,
new_stream=new_stream,
old_stream=stream_being_edited,
edit_history_event=topic_only_edit_history_event,
last_edit_time=timestamp,
)
changed_messages += messages_list
if new_stream is not None:
assert stream_being_edited is not None
changed_message_ids = [msg.id for msg in changed_messages]
if subs_gaining_usermessages:
ums_to_create = []
for message_id in changed_message_ids:
for user_profile_id in subs_gaining_usermessages:
# The fact that the user didn't have a UserMessage originally means we can infer that the user
# was not mentioned in the original message (even if mention syntax was present, it would not
# take effect for a user who was not subscribed). If we were editing the message's content, we
# would rerender the message and then use the new stream's data to determine whether this is
# a mention of a subscriber; but as we are not doing so, we choose to preserve the "was this
# mention syntax an actual mention" decision made during the original rendering for implementation
# simplicity. As a result, the only flag to consider applying here is read.
um = UserMessageLite(
user_profile_id=user_profile_id,
message_id=message_id,
flags=UserMessage.flags.read,
)
ums_to_create.append(um)
bulk_insert_ums(ums_to_create)
# Delete UserMessage objects for users who will no
# longer have access to these messages. Note: This could be
# very expensive, since it's N guest users x M messages.
UserMessage.objects.filter(
user_profile_id__in=[
sub.user_profile_id for sub in subs_losing_usermessages
],
message_id__in=changed_message_ids,
).delete()
delete_event: DeleteMessagesEvent = {
"type": "delete_message",
"message_ids": changed_message_ids,
"message_type": "stream",
"stream_id": stream_being_edited.id,
"topic": orig_topic_name,
}
delete_event_notify_user_ids = [
sub.user_profile_id for sub in subs_losing_access
]
send_event(user_profile.realm, delete_event,
delete_event_notify_user_ids)
# Reset the Attachment.is_*_public caches for all messages
# moved to another stream with different access permissions.
if new_stream.invite_only != stream_being_edited.invite_only:
Attachment.objects.filter(
messages__in=changed_message_ids).update(
is_realm_public=None, )
ArchivedAttachment.objects.filter(
messages__in=changed_message_ids).update(
is_realm_public=None, )
if new_stream.is_web_public != stream_being_edited.is_web_public:
Attachment.objects.filter(
messages__in=changed_message_ids).update(
is_web_public=None, )
ArchivedAttachment.objects.filter(
messages__in=changed_message_ids).update(
is_web_public=None, )
# This does message.save(update_fields=[...])
save_message_for_edit_use_case(message=target_message)
realm_id: Optional[int] = None
if stream_being_edited is not None:
realm_id = stream_being_edited.realm_id
event["message_ids"] = update_to_dict_cache(changed_messages, realm_id)
def user_info(um: UserMessage) -> Dict[str, Any]:
return {
"id": um.user_profile_id,
"flags": um.flags_list(),
}
# The following blocks arranges that users who are subscribed to a
# stream and can see history from before they subscribed get
# live-update when old messages are edited (e.g. if the user does
# a topic edit themself).
#
# We still don't send an update event to users who are not
# subscribed to this stream and don't have a UserMessage row. This
# means if a non-subscriber is viewing the narrow, they won't get
# a real-time updates. This is a balance between sending
# message-edit notifications for every public stream to every user
# in the organization (too expansive, and also not what we do for
# newly sent messages anyway) and having magical live-updates
# where possible.
users_to_be_notified = list(map(user_info, ums))
if stream_being_edited is not None:
if stream_being_edited.is_history_public_to_subscribers():
subscriptions = get_active_subscriptions_for_stream_id(
stream_id, include_deactivated_users=False)
# We exclude long-term idle users, since they by
# definition have no active clients.
subscriptions = subscriptions.exclude(
user_profile__long_term_idle=True)
# Remove duplicates by excluding the id of users already
# in users_to_be_notified list. This is the case where a
# user both has a UserMessage row and is a current
# Subscriber
subscriptions = subscriptions.exclude(
user_profile_id__in=[um.user_profile_id for um in ums])
if new_stream is not None:
assert delete_event_notify_user_ids is not None
subscriptions = subscriptions.exclude(
user_profile_id__in=delete_event_notify_user_ids)
# All users that are subscribed to the stream must be
# notified when a message is edited
subscriber_ids = set(
subscriptions.values_list("user_profile_id", flat=True))
if new_stream is not None:
# TODO: Guest users don't see the new moved topic
# unless breadcrumb message for new stream is
# enabled. Excluding these users from receiving this
# event helps us avoid a error traceback for our
# clients. We should figure out a way to inform the
# guest users of this new topic if sending a 'message'
# event for these messages is not an option.
#
# Don't send this event to guest subs who are not
# subscribers of the old stream but are subscribed to
# the new stream; clients will be confused.
old_stream_unsubbed_guests = [
sub for sub in subs_to_new_stream
if sub.user_profile.is_guest
and sub.user_profile_id not in subscriber_ids
]
subscriptions = subscriptions.exclude(user_profile_id__in=[
sub.user_profile_id for sub in old_stream_unsubbed_guests
])
subscriber_ids = set(
subscriptions.values_list("user_profile_id", flat=True))
users_to_be_notified += list(
map(subscriber_info, sorted(list(subscriber_ids))))
# UserTopic updates and the content of notifications depend on
# whether we've moved the entire topic, or just part of it. We
# make that determination here.
moved_all_visible_messages = False
if topic_name is not None or new_stream is not None:
assert stream_being_edited is not None
if propagate_mode == "change_all":
moved_all_visible_messages = True
else:
# With other propagate modes, if the user in fact moved
# all messages in the stream, we want to explain it was a
# full-topic move.
#
# For security model reasons, we don't want to allow a
# user to take any action that would leak information
# about older messages they cannot access (E.g. the only
# remaining messages are in a stream without shared
# history). The bulk_access_messages call below addresses
# that concern.
#
# bulk_access_messages is inefficient for this task, since
# we just want to do the exists() version of this
# query. But it's nice to reuse code, and this bulk
# operation is likely cheaper than a `GET /messages`
# unless the topic has thousands of messages of history.
assert stream_being_edited.recipient_id is not None
unmoved_messages = messages_for_topic(
stream_being_edited.recipient_id,
orig_topic_name,
)
visible_unmoved_messages = bulk_access_messages(
user_profile, unmoved_messages, stream=stream_being_edited)
moved_all_visible_messages = len(visible_unmoved_messages) == 0
# Migrate muted topic configuration in the following circumstances:
#
# * If propagate_mode is change_all, do so unconditionally.
#
# * If propagate_mode is change_later or change_one, do so when
# the acting user has moved the entire topic (as visible to them).
#
# This rule corresponds to checking moved_all_visible_messages.
#
# We may want more complex behavior in cases where one appears to
# be merging topics (E.g. there are existing messages in the
# target topic).
if moved_all_visible_messages:
assert stream_being_edited is not None
assert topic_name is not None or new_stream is not None
for muting_user in get_users_muting_topic(stream_being_edited.id,
orig_topic_name):
# TODO: Ideally, this would be a bulk update operation,
# because we are doing database operations in a loop here.
#
# This loop is only acceptable in production because it is
# rare for more than a few users to have muted an
# individual topic that is being moved; as of this
# writing, no individual topic in Zulip Cloud had been
# muted by more than 100 users.
if new_stream is not None and muting_user.id in delete_event_notify_user_ids:
# If the messages are being moved to a stream the user
# cannot access, then we treat this as the
# messages/topic being deleted for this user. This is
# important for security reasons; we don't want to
# give users a UserTopic row in a stream they cannot
# access. Unmute the topic for such users.
do_unmute_topic(muting_user, stream_being_edited,
orig_topic_name)
else:
# Otherwise, we move the muted topic record for the user.
# We call remove_topic_mute rather than do_unmute_topic to
# avoid sending two events with new muted topics in
# immediate succession; this is correct only because
# muted_topics events always send the full set of topics.
remove_topic_mute(muting_user, stream_being_edited.id,
orig_topic_name)
do_mute_topic(
muting_user,
new_stream
if new_stream is not None else stream_being_edited,
topic_name if topic_name is not None else orig_topic_name,
ignore_duplicate=True,
)
send_event(user_profile.realm, event, users_to_be_notified)
if len(
changed_messages
) > 0 and new_stream is not None and stream_being_edited is not None:
# Notify users that the topic was moved.
changed_messages_count = len(changed_messages)
old_thread_notification_string = None
if send_notification_to_old_thread:
if moved_all_visible_messages:
old_thread_notification_string = gettext_lazy(
"This topic was moved to {new_location} by {user}.")
elif changed_messages_count == 1:
old_thread_notification_string = gettext_lazy(
"A message was moved from this topic to {new_location} by {user}."
)
else:
old_thread_notification_string = gettext_lazy(
"{changed_messages_count} messages were moved from this topic to {new_location} by {user}."
)
new_thread_notification_string = None
if send_notification_to_new_thread:
if moved_all_visible_messages:
new_thread_notification_string = gettext_lazy(
"This topic was moved here from {old_location} by {user}.")
elif changed_messages_count == 1:
new_thread_notification_string = gettext_lazy(
"A message was moved here from {old_location} by {user}.")
else:
new_thread_notification_string = gettext_lazy(
"{changed_messages_count} messages were moved here from {old_location} by {user}."
)
send_message_moved_breadcrumbs(
user_profile,
stream_being_edited,
orig_topic_name,
old_thread_notification_string,
new_stream,
topic_name,
new_thread_notification_string,
changed_messages_count,
)
if (topic_name is not None and new_stream is None and content is None
and len(changed_messages) > 0):
assert stream_being_edited is not None
maybe_send_resolve_topic_notifications(
user_profile=user_profile,
stream=stream_being_edited,
old_topic=orig_topic_name,
new_topic=topic_name,
changed_messages=changed_messages,
)
return len(changed_messages)
def generate_and_send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open(
os.path.join(get_or_create_dev_uuid_var_path('test-backend'),
"test_messages.json"), "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
] # type: List[int]
recipient_huddles = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
message_batch_size = options['batch_size']
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
messages = []
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100. / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + str(random.randint(1, 3))
saved_data['subject'] = message.subject
message.date_sent = choose_date_sent(num_messages, tot_messages,
options['threads'])
messages.append(message)
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
if (num_messages % message_batch_size) == 0:
# Send the batch and empty the list:
send_messages(messages)
messages = []
if len(messages) > 0:
# If there are unsent messages after exiting the loop, send them:
send_messages(messages)
return tot_messages
def restore_saved_messages():
# type: () -> None
old_messages = [] # type: List[Dict[str, Any]]
duplicate_suppression_hash = {} # type: Dict[str, bool]
stream_dict = {} # type: Dict[Tuple[text_type, text_type], Tuple[text_type, text_type]]
user_set = set() # type: Set[Tuple[text_type, text_type, text_type, bool]]
email_set = set([u.email for u in UserProfile.objects.all()]) # type: Set[text_type]
realm_set = set() # type: Set[text_type]
# Initial client_set is nonempty temporarily because we don't have
# clients in logs at all right now -- later we can start with nothing.
client_set = set(["populate_db", "website", "zephyr_mirror"])
huddle_user_set = set() # type: Set[Tuple[text_type, ...]]
# First, determine all the objects our messages will need.
print(datetime.datetime.now(), "Creating realms/streams/etc...")
def process_line(line):
# type: (str) -> None
old_message_json = line.strip()
# Due to populate_db's shakespeare mode, we have a lot of
# duplicate messages in our log that only differ in their
# logged ID numbers (same timestamp, content, etc.). With
# sqlite, bulk creating those messages won't work properly: in
# particular, the first 100 messages will actually only result
# in 20 rows ending up in the target table, which screws up
# the below accounting where for handling changing
# subscriptions, we assume that the Nth row populate_db
# created goes with the Nth non-subscription row of the input
# So suppress the duplicates when using sqlite.
if "sqlite" in settings.DATABASES["default"]["ENGINE"]:
tmp_message = ujson.loads(old_message_json)
tmp_message['id'] = '1'
duplicate_suppression_key = ujson.dumps(tmp_message)
if duplicate_suppression_key in duplicate_suppression_hash:
return
duplicate_suppression_hash[duplicate_suppression_key] = True
old_message = ujson.loads(old_message_json)
message_type = old_message["type"]
# Lower case emails and domains; it will screw up
# deduplication if we don't
def fix_email(email):
# type: (text_type) -> text_type
return email.strip().lower()
if message_type in ["stream", "huddle", "personal"]:
old_message["sender_email"] = fix_email(old_message["sender_email"])
# Fix the length on too-long messages before we start processing them
if len(old_message["content"]) > MAX_MESSAGE_LENGTH:
old_message["content"] = "[ This message was deleted because it was too long ]"
if message_type in ["subscription_added", "subscription_removed"]:
old_message["domain"] = old_message["domain"].lower()
old_message["user"] = fix_email(old_message["user"])
elif message_type == "subscription_property":
old_message["user"] = fix_email(old_message["user"])
elif message_type == "user_email_changed":
old_message["old_email"] = fix_email(old_message["old_email"])
old_message["new_email"] = fix_email(old_message["new_email"])
elif message_type.startswith("user_"):
old_message["user"] = fix_email(old_message["user"])
elif message_type.startswith("enable_"):
old_message["user"] = fix_email(old_message["user"])
if message_type == 'personal':
old_message["recipient"][0]["email"] = fix_email(old_message["recipient"][0]["email"])
elif message_type == "huddle":
for i in range(len(old_message["recipient"])):
old_message["recipient"][i]["email"] = fix_email(old_message["recipient"][i]["email"])
old_messages.append(old_message)
if message_type in ["subscription_added", "subscription_removed"]:
stream_name = old_message["name"].strip() # type: text_type
canon_stream_name = stream_name.lower()
if canon_stream_name not in stream_dict:
stream_dict[(old_message["domain"], canon_stream_name)] = \
(old_message["domain"], stream_name)
elif message_type == "user_created":
user_set.add((old_message["user"], old_message["full_name"], old_message["short_name"], False))
elif message_type == "realm_created":
realm_set.add(old_message["domain"])
if message_type not in ["stream", "huddle", "personal"]:
return
sender_email = old_message["sender_email"]
domain = split_email_to_domain(sender_email)
realm_set.add(domain)
if old_message["sender_email"] not in email_set:
user_set.add((old_message["sender_email"],
old_message["sender_full_name"],
old_message["sender_short_name"],
False))
if 'sending_client' in old_message:
client_set.add(old_message['sending_client'])
if message_type == 'stream':
stream_name = old_message["recipient"].strip()
canon_stream_name = stream_name.lower()
if canon_stream_name not in stream_dict:
stream_dict[(domain, canon_stream_name)] = (domain, stream_name)
elif message_type == 'personal':
u = old_message["recipient"][0]
if u["email"] not in email_set:
user_set.add((u["email"], u["full_name"], u["short_name"], False))
email_set.add(u["email"])
elif message_type == 'huddle':
for u in old_message["recipient"]:
user_set.add((u["email"], u["full_name"], u["short_name"], False))
if u["email"] not in email_set:
user_set.add((u["email"], u["full_name"], u["short_name"], False))
email_set.add(u["email"])
huddle_user_set.add(tuple(sorted(set(u["email"] for u in old_message["recipient"]))))
else:
raise ValueError('Bad message type')
event_glob = os.path.join(settings.EVENT_LOG_DIR, 'events.*')
for filename in sorted(glob.glob(event_glob)):
with open(filename, "r") as message_log:
for line in message_log.readlines():
process_line(line)
stream_recipients = {} # type: Dict[Tuple[int, text_type], Recipient]
user_recipients = {} # type: Dict[text_type, Recipient]
huddle_recipients = {} # type: Dict[text_type, Recipient]
# Then, create the objects our messages need.
print(datetime.datetime.now(), "Creating realms...")
bulk_create_realms(realm_set)
realms = {} # type: Dict[text_type, Realm]
for realm in Realm.objects.all():
realms[realm.domain] = realm
print(datetime.datetime.now(), "Creating clients...")
bulk_create_clients(client_set)
clients = {} # type: Dict[text_type, Client]
for client in Client.objects.all():
clients[client.name] = client
print(datetime.datetime.now(), "Creating streams...")
bulk_create_streams(realms, list(stream_dict.values()))
streams = {} # type: Dict[int, Stream]
for stream in Stream.objects.all():
streams[stream.id] = stream
for recipient in Recipient.objects.filter(type=Recipient.STREAM):
stream_recipients[(streams[recipient.type_id].realm_id,
streams[recipient.type_id].name.lower())] = recipient
print(datetime.datetime.now(), "Creating users...")
bulk_create_users(realms, user_set, tos_version=settings.TOS_VERSION)
users = {} # type: Dict[text_type, UserProfile]
users_by_id = {} # type: Dict[int, UserProfile]
for user_profile in UserProfile.objects.select_related().all():
users[user_profile.email] = user_profile
users_by_id[user_profile.id] = user_profile
for recipient in Recipient.objects.filter(type=Recipient.PERSONAL):
user_recipients[users_by_id[recipient.type_id].email] = recipient
print(datetime.datetime.now(), "Creating huddles...")
bulk_create_huddles(users, huddle_user_set)
huddles_by_id = {} # type: Dict[int, Huddle]
for huddle in Huddle.objects.all():
huddles_by_id[huddle.id] = huddle
for recipient in Recipient.objects.filter(type=Recipient.HUDDLE):
huddle_recipients[huddles_by_id[recipient.type_id].huddle_hash] = recipient
# TODO: Add a special entry type in the log that is a subscription
# change and import those as we go to make subscription changes
# take effect!
print(datetime.datetime.now(), "Importing subscriptions...")
subscribers = {} # type: Dict[int, Set[int]]
for s in Subscription.objects.select_related().all():
if s.active:
subscribers.setdefault(s.recipient.id, set()).add(s.user_profile.id)
# Then create all the messages, without talking to the DB!
print(datetime.datetime.now(), "Importing messages, part 1...")
first_message_id = None
if Message.objects.exists():
first_message_id = Message.objects.all().order_by("-id")[0].id + 1
messages_to_create = [] # type: List[Message]
for idx, old_message in enumerate(old_messages):
message_type = old_message["type"]
if message_type not in ["stream", "huddle", "personal"]:
continue
message = Message()
sender_email = old_message["sender_email"]
domain = split_email_to_domain(sender_email)
realm = realms[domain]
message.sender = users[sender_email]
type_hash = {"stream": Recipient.STREAM,
"huddle": Recipient.HUDDLE,
"personal": Recipient.PERSONAL}
if 'sending_client' in old_message:
message.sending_client = clients[old_message['sending_client']]
elif sender_email in ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]:
message.sending_client = clients['populate_db']
elif realm.domain == "zulip.com":
message.sending_client = clients["website"]
elif realm.domain == "mit.edu":
message.sending_client = clients['zephyr_mirror']
else:
message.sending_client = clients['populate_db']
message.type = type_hash[message_type]
message.content = old_message["content"]
message.subject = old_message["subject"]
message.pub_date = timestamp_to_datetime(old_message["timestamp"])
if message.type == Recipient.PERSONAL:
message.recipient = user_recipients[old_message["recipient"][0]["email"]]
elif message.type == Recipient.STREAM:
message.recipient = stream_recipients[(realm.id,
old_message["recipient"].lower())]
elif message.type == Recipient.HUDDLE:
huddle_hash = get_huddle_hash([users[u["email"]].id
for u in old_message["recipient"]])
message.recipient = huddle_recipients[huddle_hash]
else:
raise ValueError('Bad message type')
messages_to_create.append(message)
print(datetime.datetime.now(), "Importing messages, part 2...")
Message.objects.bulk_create(messages_to_create)
messages_to_create = []
# Finally, create all the UserMessage objects
print(datetime.datetime.now(), "Importing usermessages, part 1...")
personal_recipients = {} # type: Dict[int, bool]
for r in Recipient.objects.filter(type = Recipient.PERSONAL):
personal_recipients[r.id] = True
all_messages = Message.objects.all() # type: Sequence[Message]
user_messages_to_create = [] # type: List[UserMessage]
messages_by_id = {} # type: Dict[int, Message]
for message in all_messages:
messages_by_id[message.id] = message
if len(messages_by_id) == 0:
print(datetime.datetime.now(), "No old messages to replay")
return
if first_message_id is None:
first_message_id = min(messages_by_id.keys())
tot_user_messages = 0
pending_subs = {} # type: Dict[Tuple[int, int], bool]
current_message_id = first_message_id
pending_colors = {} # type: Dict[Tuple[text_type, text_type], text_type]
for old_message in old_messages:
message_type = old_message["type"]
if message_type == 'subscription_added':
stream_key = (realms[old_message["domain"]].id, old_message["name"].strip().lower())
subscribers.setdefault(stream_recipients[stream_key].id,
set()).add(users[old_message["user"]].id)
pending_subs[(stream_recipients[stream_key].id,
users[old_message["user"]].id)] = True
continue
elif message_type == "subscription_removed":
stream_key = (realms[old_message["domain"]].id, old_message["name"].strip().lower())
user_id = users[old_message["user"]].id
subscribers.setdefault(stream_recipients[stream_key].id, set())
try:
subscribers[stream_recipients[stream_key].id].remove(user_id)
except KeyError:
print("Error unsubscribing %s from %s: not subscribed" % (
old_message["user"], old_message["name"]))
pending_subs[(stream_recipients[stream_key].id,
users[old_message["user"]].id)] = False
continue
elif message_type == "user_activated" or message_type == "user_created":
# These are rare, so just handle them the slow way
user_profile = users[old_message["user"]]
join_date = timestamp_to_datetime(old_message['timestamp'])
do_activate_user(user_profile, log=False, join_date=join_date)
# Update the cache of users to show this user as activated
users_by_id[user_profile.id] = user_profile
users[old_message["user"]] = user_profile
continue
elif message_type == "user_deactivated":
user_profile = users[old_message["user"]]
do_deactivate_user(user_profile, log=False)
continue
elif message_type == "user_change_password":
# Just handle these the slow way
user_profile = users[old_message["user"]]
do_change_password(user_profile, old_message["pwhash"], log=False,
hashed_password=True)
continue
elif message_type == "user_change_full_name":
# Just handle these the slow way
user_profile = users[old_message["user"]]
user_profile.full_name = old_message["full_name"]
user_profile.save(update_fields=["full_name"])
continue
elif message_type == "enable_desktop_notifications_changed":
# Just handle these the slow way
user_profile = users[old_message["user"]]
user_profile.enable_desktop_notifications = (old_message["enable_desktop_notifications"] != "false")
user_profile.save(update_fields=["enable_desktop_notifications"])
continue
elif message_type == "enable_sounds_changed":
user_profile = users[old_message["user"]]
user_profile.enable_sounds = (old_message["enable_sounds"] != "false")
user_profile.save(update_fields=["enable_sounds"])
elif message_type == "enable_offline_email_notifications_changed":
user_profile = users[old_message["user"]]
user_profile.enable_offline_email_notifications = (
old_message["enable_offline_email_notifications"] != "false")
user_profile.save(update_fields=["enable_offline_email_notifications"])
continue
elif message_type == "enable_offline_push_notifications_changed":
user_profile = users[old_message["user"]]
user_profile.enable_offline_push_notifications = (
old_message["enable_offline_push_notifications"] != "false")
user_profile.save(update_fields=["enable_offline_push_notifications"])
continue
elif message_type == "default_streams":
set_default_streams(get_realm(old_message["domain"]),
old_message["streams"])
continue
elif message_type == "subscription_property":
property_name = old_message.get("property")
if property_name == "stream_color" or property_name == "color":
color = old_message.get("color", old_message.get("value"))
pending_colors[(old_message["user"],
old_message["stream_name"].lower())] = color
elif property_name in ["in_home_view", "notifications"]:
# TODO: Handle this
continue
else:
raise RuntimeError("Unknown property %s" % (property_name,))
continue
elif message_type == "realm_created":
# No action required
continue
elif message_type in ["user_email_changed", "update_onboarding", "update_message"]:
# TODO: Handle these
continue
if message_type not in ["stream", "huddle", "personal"]:
raise RuntimeError("Unexpected message type %s" % (message_type,))
message = messages_by_id[current_message_id]
current_message_id += 1
if message.recipient_id not in subscribers:
# Nobody received this message -- probably due to our
# subscriptions being out-of-date.
continue
recipient_user_ids = set() # type: Set[int]
for user_profile_id in subscribers[message.recipient_id]:
recipient_user_ids.add(user_profile_id)
if message.recipient_id in personal_recipients:
# Include the sender in huddle recipients
recipient_user_ids.add(message.sender_id)
for user_profile_id in recipient_user_ids:
if users_by_id[user_profile_id].is_active:
um = UserMessage(user_profile_id=user_profile_id,
message=message)
user_messages_to_create.append(um)
if len(user_messages_to_create) > 100000:
tot_user_messages += len(user_messages_to_create)
UserMessage.objects.bulk_create(user_messages_to_create)
user_messages_to_create = []
print(datetime.datetime.now(), "Importing usermessages, part 2...")
tot_user_messages += len(user_messages_to_create)
UserMessage.objects.bulk_create(user_messages_to_create)
print(datetime.datetime.now(), "Finalizing subscriptions...")
current_subs = {} # type: Dict[Tuple[int, int], bool]
current_subs_obj = {} # type: Dict[Tuple[int, int], Subscription]
for s in Subscription.objects.select_related().all():
current_subs[(s.recipient_id, s.user_profile_id)] = s.active
current_subs_obj[(s.recipient_id, s.user_profile_id)] = s
subscriptions_to_add = [] # type: List[Subscription]
subscriptions_to_change = [] # type: List[Tuple[Tuple[int, int], bool]]
for pending_sub in pending_subs.keys():
(recipient_id, user_profile_id) = pending_sub
current_state = current_subs.get(pending_sub)
if pending_subs[pending_sub] == current_state:
# Already correct in the database
continue
elif current_state is not None:
subscriptions_to_change.append((pending_sub, pending_subs[pending_sub]))
continue
s = Subscription(recipient_id=recipient_id,
user_profile_id=user_profile_id,
active=pending_subs[pending_sub])
subscriptions_to_add.append(s)
Subscription.objects.bulk_create(subscriptions_to_add)
for (sub_tuple, active) in subscriptions_to_change:
current_subs_obj[sub_tuple].active = active
current_subs_obj[sub_tuple].save(update_fields=["active"])
subs = {} # type: Dict[Tuple[int, int], Subscription]
for sub in Subscription.objects.all():
subs[(sub.user_profile_id, sub.recipient_id)] = sub
# TODO: do restore of subscription colors -- we're currently not
# logging changes so there's little point in having the code :(
print(datetime.datetime.now(), "Finished importing %s messages (%s usermessages)" % \
(len(all_messages), tot_user_messages))
site = Site.objects.get_current()
site.domain = 'zulip.com'
site.save()
print(datetime.datetime.now(), "Filling in user pointers...")
# Set restored pointers to the very latest messages
for user_profile in UserProfile.objects.all():
try:
top = UserMessage.objects.filter(
user_profile_id=user_profile.id).order_by("-message")[0]
user_profile.pointer = top.message_id
except IndexError:
user_profile.pointer = -1
user_profile.save(update_fields=["pointer"])
print(datetime.datetime.now(), "Done replaying old messages")
