def fetch_user_profile_cross_realm(response: TableData, config: Config, context: Context) -> None:
realm = context['realm']
if realm.string_id == settings.SYSTEM_BOT_REALM:
response['zerver_userprofile_crossrealm'] = []
else:
response['zerver_userprofile_crossrealm'] = [dict(email=x.email, id=x.id) for x in [
get_system_bot(settings.NOTIFICATION_BOT),
get_system_bot(settings.EMAIL_GATEWAY_BOT),
get_system_bot(settings.WELCOME_BOT),
]]
def zulip_server_error(report: Dict[str, Any]) -> None:
email_subject = '%(node)s: %(message)s' % (report)
logger_str = logger_repr(report)
user_info = user_info_str(report)
deployment = deployment_repr(report)
if report['has_request']:
request_repr = (
"Request info:\n~~~~\n"
"- path: %(path)s\n"
"- %(method)s: %(data)s\n") % (report)
for field in ["REMOTE_ADDR", "QUERY_STRING", "SERVER_NAME"]:
val = report.get(field.lower())
if field == "QUERY_STRING":
val = clean_data_from_query_parameters(str(val))
request_repr += "- %s: \"%s\"\n" % (field, val)
request_repr += "~~~~"
else:
request_repr = "Request info: none"
message = ("%s\nError generated by %s\n\n~~~~ pytb\n%s\n\n~~~~\n%s\n%s"
% (logger_str, user_info, report['stack_trace'], deployment, request_repr))
realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(realm, settings.ERROR_BOT, "stream", "errors",
format_email_subject(email_subject), message)
def handle(self, *args: Any, **options: Any) -> None:
if Realm.objects.count() > 0:
print("Database already initialized; doing nothing.")
return
realm = Realm.objects.create(string_id=settings.INTERNAL_BOT_DOMAIN.split('.')[0])
names = [(settings.FEEDBACK_BOT_NAME, settings.FEEDBACK_BOT)]
create_users(realm, names, bot_type=UserProfile.DEFAULT_BOT)
get_client("website")
get_client("API")
internal_bots = [(bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,))
for bot in settings.INTERNAL_BOTS]
create_users(realm, internal_bots, bot_type=UserProfile.DEFAULT_BOT)
# Set the owners for these bots to the bots themselves
bots = UserProfile.objects.filter(email__in=[bot_info[1] for bot_info in internal_bots])
for bot in bots:
bot.bot_owner = bot
bot.save()
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
self.stdout.write("Successfully populated database with initial data.\n")
self.stdout.write("Please run ./manage.py generate_realm_creation_link "
"to generate link for creating organization")
def extract_and_upload_attachments(message: message.Message, realm: Realm) -> str:
user_profile = get_system_bot(settings.EMAIL_GATEWAY_BOT)
attachment_links = []
payload = message.get_payload()
if not isinstance(payload, list):
# This is not a multipart message, so it can't contain attachments.
return ""
for part in payload:
content_type = part.get_content_type()
filename = part.get_filename()
if filename:
attachment = part.get_payload(decode=True)
if isinstance(attachment, bytes):
s3_url = upload_message_file(filename, len(attachment), content_type,
attachment,
user_profile,
target_realm=realm)
formatted_link = "[%s](%s)" % (filename, s3_url)
attachment_links.append(formatted_link)
else:
logger.warning("Payload is not bytes (invalid attachment %s in message from %s)." %
(filename, message.get("From")))
return "\n".join(attachment_links)
def report_to_zulip(error_message: str) -> None:
if settings.ERROR_BOT is None:
return
error_bot = get_system_bot(settings.ERROR_BOT)
error_stream = Stream.objects.get(name="errors", realm=error_bot.realm)
send_zulip(settings.ERROR_BOT, error_stream, "email mirror error",
"""~~~\n%s\n~~~""" % (error_message,))
def deliver_feedback_by_zulip(message: Mapping[str, Any]) -> None:
subject = "%s" % (message["sender_email"],)
if len(subject) > 60:
subject = subject[:57].rstrip() + "..."
content = ''
sender_email = message['sender_email']
# We generate ticket numbers if it's been more than a few minutes
# since their last message. This avoids some noise when people use
# enter-send.
need_ticket = has_enough_time_expired_since_last_message(sender_email, 180)
if need_ticket:
ticket_number = get_ticket_number()
content += '\n~~~'
content += '\nticket Z%03d (@support please ack)' % (ticket_number,)
content += '\nsender: %s' % (message['sender_full_name'],)
content += '\nemail: %s' % (sender_email,)
if 'sender_realm_str' in message:
content += '\nrealm: %s' % (message['sender_realm_str'],)
content += '\n~~~'
content += '\n\n'
content += message['content']
user_profile = get_system_bot(settings.FEEDBACK_BOT)
internal_send_message(user_profile.realm, settings.FEEDBACK_BOT,
"stream", settings.FEEDBACK_STREAM, subject, content)
def send_initial_pms(user):
# type: (UserProfile) -> None
content = """Welcome to Zulip!
This is a great place to test formatting, sending, and editing messages.
Click anywhere on this message to reply. A compose box will open at the bottom of the screen."""
internal_send_private_message(user.realm, get_system_bot(settings.WELCOME_BOT),
user.email, content)
def test_stream_error_pm_to_bot_owner(self) -> None:
# Note taht this is really just a test for check_send_webhook_message
self.STREAM_NAME = 'nonexistent'
self.url = self.build_webhook_url()
notification_bot = get_system_bot(settings.NOTIFICATION_BOT)
expected_message = "Hi there! We thought you'd like to know that your bot **Zulip Webhook Bot** just tried to send a message to stream `nonexistent`, but that stream does not yet exist. To create it, click the gear in the left-side stream list."
self.send_and_test_private_message('goodbye', expected_message=expected_message,
content_type='application/x-www-form-urlencoded',
sender=notification_bot)
def export_avatars_from_local(realm: Realm, local_dir: Path, output_dir: Path) -> None:
count = 0
records = []
users = list(UserProfile.objects.filter(realm=realm))
users += [
get_system_bot(settings.NOTIFICATION_BOT),
get_system_bot(settings.EMAIL_GATEWAY_BOT),
get_system_bot(settings.WELCOME_BOT),
]
for user in users:
if user.avatar_source == UserProfile.AVATAR_FROM_GRAVATAR:
continue
avatar_path = user_avatar_path_from_ids(user.id, realm.id)
wildcard = os.path.join(local_dir, avatar_path + '.*')
for local_path in glob.glob(wildcard):
logging.info('Copying avatar file for user %s from %s' % (
user.email, local_path))
fn = os.path.relpath(local_path, local_dir)
output_path = os.path.join(output_dir, fn)
os.makedirs(str(os.path.dirname(output_path)), exist_ok=True)
subprocess.check_call(["cp", "-a", str(local_path), str(output_path)])
stat = os.stat(local_path)
record = dict(realm_id=realm.id,
user_profile_id=user.id,
user_profile_email=user.email,
s3_path=fn,
path=fn,
size=stat.st_size,
last_modified=stat.st_mtime,
content_type=None)
records.append(record)
count += 1
if (count % 100 == 0):
logging.info("Finished %s" % (count,))
with open(os.path.join(output_dir, "records.json"), "w") as records_file:
ujson.dump(records, records_file, indent=4)
def zulip_browser_error(report: Dict[str, Any]) -> None:
subject = "JS error: %s" % (report['user_email'],)
user_info = user_info_str(report)
body = "User: %s\n" % (user_info,)
body += ("Message: %(message)s\n"
% (report))
realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(realm, settings.ERROR_BOT,
"stream", "errors", format_subject(subject), body)
def send_initial_pms(user):
# type: (UserProfile) -> None
content = (
"Hello, and welcome to Zulip!\n\nThis is a private message from me, Welcome Bot. "
"Here are some tips to get you started:\n"
"* Download our [Desktop and mobile apps](/apps)\n"
"* Customize your account and notifications on your [Settings page](#settings).\n"
"* Check out our !modal_link(#keyboard-shortcuts, Keyboard shortcuts)\n\n"
"The most important shortcut is `r` or `Enter` to reply.\n\n"
"Practice sending a few messages by replying to this conversation. If you're not into "
"keyboards, that's okay too; clicking anywhere on this message will also do the trick!")
internal_send_private_message(user.realm, get_system_bot(settings.WELCOME_BOT),
user.email, content)
def highlight_html_differences(s1, s2):
# type: (Text, Text) -> Text
differ = diff_match_patch()
ops = differ.diff_main(s1, s2)
differ.diff_cleanupSemantic(ops)
retval = u''
in_tag = False
idx = 0
while idx < len(ops):
op, text = ops[idx]
next_op = None
if idx != len(ops) - 1:
next_op, next_text = ops[idx + 1]
if op == diff_match_patch.DIFF_DELETE and next_op == diff_match_patch.DIFF_INSERT:
# Replace operation
chunks, in_tag = chunkize(next_text, in_tag)
retval += highlight_chunks(chunks, highlight_replaced)
idx += 1
elif op == diff_match_patch.DIFF_INSERT and next_op == diff_match_patch.DIFF_DELETE:
# Replace operation
# I have no idea whether diff_match_patch generates inserts followed
# by deletes, but it doesn't hurt to handle them
chunks, in_tag = chunkize(text, in_tag)
retval += highlight_chunks(chunks, highlight_replaced)
idx += 1
elif op == diff_match_patch.DIFF_DELETE:
retval += highlight_deleted(' ')
elif op == diff_match_patch.DIFF_INSERT:
chunks, in_tag = chunkize(text, in_tag)
retval += highlight_chunks(chunks, highlight_inserted)
elif op == diff_match_patch.DIFF_EQUAL:
chunks, in_tag = chunkize(text, in_tag)
retval += text
idx += 1
if not verify_html(retval):
from zerver.lib.actions import internal_send_message
from zerver.models import get_system_bot
# We probably want more information here
logging.getLogger('').error('HTML diff produced mal-formed HTML')
if settings.ERROR_BOT is not None:
subject = "HTML diff failure on %s" % (platform.node(),)
realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(realm, settings.ERROR_BOT, "stream",
"errors", subject, "HTML diff produced malformed HTML")
return s2
return retval
def send_initial_realm_messages(realm):
# type: (Realm) -> None
welcome_bot = get_system_bot(settings.WELCOME_BOT)
# Make sure each stream created in the realm creation process has at least one message below
# Order corresponds to the ordering of the streams on the left sidebar, to make the initial Home
# view slightly less overwhelming
welcome_messages = [
{'stream': Realm.DEFAULT_NOTIFICATION_STREAM_NAME,
'topic': "welcome",
'content': "This is a message on stream `%s` with the topic `welcome`. We'll use this stream "
"for system-generated notifications." % (Realm.DEFAULT_NOTIFICATION_STREAM_NAME,)},
{'stream': "core team",
'topic': "private streams",
'content': "This is a private stream. Only admins and people you invite "
"to the stream will be able to see that this stream exists."},
{'stream': "general",
'topic': "welcome",
'content': "Welcome to #**general**."},
{'stream': "new members",
'topic': "onboarding",
'content': "A #**new members** stream is great for onboarding new members.\n\nIf you're "
"reading this and aren't the first person here, introduce yourself in a new thread "
"using your name as the topic! Type `c` or click on `New Topic` at the bottom of the "
"screen to start a new topic."},
{'stream': "zulip",
'topic': "topic demonstration",
'content': "Here is a message in one topic. Replies to this message will go to this topic."},
{'stream': "zulip",
'topic': "topic demonstration",
'content': "A second message in this topic. With [turtles](/static/images/cute/turtle.png)!"},
{'stream': "zulip",
'topic': "second topic",
'content': "This is a message in a second topic.\n\nTopics are similar to email subjects, "
"in that each conversation should get its own topic. Keep them short, though; one "
"or two words will do it!"},
] # type: List[Dict[str, Text]]
messages = [internal_prep_stream_message(
realm, welcome_bot,
message['stream'], message['topic'], message['content']) for message in welcome_messages]
message_ids = do_send_messages(messages)
# We find the one of our just-sent messages with turtle.png in it,
# and react to it. This is a bit hacky, but works and is kinda a
# 1-off thing.
turtle_message = Message.objects.get(
id__in=message_ids,
subject='topic demonstration',
content__icontains='cute/turtle.png')
do_add_reaction_legacy(welcome_bot, turtle_message, 'turtle')
def test_cross_realm_file_access(self):
# type: () -> None
def create_user(email, realm_id):
# type: (Text, Text) -> UserProfile
self.register(email, 'test', subdomain=realm_id)
return get_user(email, get_realm(realm_id))
test_subdomain = "uploadtest.example.com"
user1_email = '*****@*****.**'
user2_email = '*****@*****.**'
user3_email = '*****@*****.**'
r1 = Realm.objects.create(string_id=test_subdomain, invite_required=False)
RealmDomain.objects.create(realm=r1, domain=test_subdomain)
create_user(user1_email, test_subdomain)
create_user(user2_email, 'zulip')
create_user(user3_email, test_subdomain)
# Send a message from @zulip.com -> @uploadtest.example.com
self.login(user2_email, 'test')
fp = StringIO("zulip!")
fp.name = "zulip.txt"
result = self.client_post("/json/user_uploads", {'file': fp})
uri = result.json()['uri']
fp_path_id = re.sub('/user_uploads/', '', uri)
body = "First message ...[zulip.txt](http://localhost:9991/user_uploads/" + fp_path_id + ")"
with self.settings(CROSS_REALM_BOT_EMAILS = set((user2_email, user3_email))):
internal_send_private_message(
realm=r1,
sender=get_system_bot(user2_email),
recipient_user=get_user(user1_email, r1),
content=body,
)
self.login(user1_email, 'test')
response = self.client_get(uri, subdomain=test_subdomain)
self.assertEqual(response.status_code, 200)
data = b"".join(response.streaming_content)
self.assertEqual(b"zulip!", data)
self.logout()
# Confirm other cross-realm users can't read it.
self.login(user3_email, 'test')
response = self.client_get(uri, subdomain=test_subdomain)
self.assertEqual(response.status_code, 403)
self.assert_in_response("You are not authorized to view this file.", response)
def send_welcome_bot_response(send_request: SendMessageRequest) -> None:
welcome_bot = get_system_bot(settings.WELCOME_BOT,
send_request.message.sender.realm_id)
human_recipient_id = send_request.message.sender.recipient_id
assert human_recipient_id is not None
if Message.objects.filter(sender=welcome_bot,
recipient_id=human_recipient_id).count() < 2:
content = (
_("Congratulations on your first reply!") + " "
":tada:"
"\n"
"\n" +
_("Feel free to continue using this space to practice your new messaging "
"skills. Or, try clicking on some of the stream names to your left!"
))
internal_send_private_message(welcome_bot, send_request.message.sender,
content)
def test_cross_realm_file_access(self) -> None:
def create_user(email: Text, realm_id: Text) -> UserProfile:
self.register(email, 'test', subdomain=realm_id)
return get_user(email, get_realm(realm_id))
test_subdomain = "uploadtest.example.com"
user1_email = '*****@*****.**'
user2_email = '*****@*****.**'
user3_email = '*****@*****.**'
r1 = Realm.objects.create(string_id=test_subdomain, invite_required=False)
RealmDomain.objects.create(realm=r1, domain=test_subdomain)
create_user(user1_email, test_subdomain)
create_user(user2_email, 'zulip')
create_user(user3_email, test_subdomain)
# Send a message from @zulip.com -> @uploadtest.example.com
self.login(user2_email, 'test')
fp = StringIO("zulip!")
fp.name = "zulip.txt"
result = self.client_post("/json/user_uploads", {'file': fp})
uri = result.json()['uri']
fp_path_id = re.sub('/user_uploads/', '', uri)
body = "First message ...[zulip.txt](http://localhost:9991/user_uploads/" + fp_path_id + ")"
with self.settings(CROSS_REALM_BOT_EMAILS = set((user2_email, user3_email))):
internal_send_private_message(
realm=r1,
sender=get_system_bot(user2_email),
recipient_user=get_user(user1_email, r1),
content=body,
)
self.login(user1_email, 'test', realm=r1)
response = self.client_get(uri, subdomain=test_subdomain)
self.assertEqual(response.status_code, 200)
data = b"".join(response.streaming_content)
self.assertEqual(b"zulip!", data)
self.logout()
# Confirm other cross-realm users can't read it.
self.login(user3_email, 'test', realm=r1)
response = self.client_get(uri, subdomain=test_subdomain)
self.assertEqual(response.status_code, 403)
self.assert_in_response("You are not authorized to view this file.", response)
def highlight_html_differences(s1, s2):
# type: (Text, Text) -> Text
differ = diff_match_patch()
ops = differ.diff_main(s1, s2)
differ.diff_cleanupSemantic(ops)
retval = u''
in_tag = False
idx = 0
while idx < len(ops):
op, text = ops[idx]
text = check_tags(text)
if idx != 0:
prev_op, prev_text = ops[idx - 1]
prev_text = check_tags(prev_text)
# Remove visual offset from editing newlines
if '<p><br>' in text:
text = text.replace('<p><br>', '<p>')
elif prev_text.endswith('<p>') and text.startswith('<br>'):
text = text[4:]
if op == diff_match_patch.DIFF_DELETE:
chunks, in_tag = chunkize(text, in_tag)
retval += highlight_chunks(chunks, highlight_deleted)
elif op == diff_match_patch.DIFF_INSERT:
chunks, in_tag = chunkize(text, in_tag)
retval += highlight_chunks(chunks, highlight_inserted)
elif op == diff_match_patch.DIFF_EQUAL:
chunks, in_tag = chunkize(text, in_tag)
retval += text
idx += 1
if not verify_html(retval):
from zerver.lib.actions import internal_send_message
from zerver.models import get_system_bot
# We probably want more information here
logging.getLogger('').error('HTML diff produced mal-formed HTML')
if settings.ERROR_BOT is not None:
subject = "HTML diff failure on %s" % (platform.node(), )
realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(realm, settings.ERROR_BOT, "stream",
"errors", subject,
"HTML diff produced malformed HTML")
return s2
return retval
def create_internal_realm() -> None:
internal_realm = Realm.objects.create(string_id=settings.SYSTEM_BOT_REALM)
internal_realm_bots = [
(bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN, ))
for bot in settings.INTERNAL_BOTS
]
internal_realm_bots += [
("Zulip Feedback Bot", "*****@*****.**"),
]
create_users(internal_realm,
internal_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
do_change_is_admin(email_gateway_bot, True, permission="api_super_user")
def consume_batch(self, slow_queries: List[Dict[str, Any]]) -> None:
for query in slow_queries:
logging.info("Slow query: %s" % (query))
if settings.ERROR_BOT is None:
return
if len(slow_queries) > 0:
topic = "%s: slow queries" % (settings.EXTERNAL_HOST,)
content = ""
for query in slow_queries:
content += " %s\n" % (query,)
error_bot_realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(error_bot_realm, settings.ERROR_BOT,
"stream", "logs", topic, content)
def test_single_response_to_pm(self) -> None:
user_email = '*****@*****.**'
user = self.example_user('hamlet')
bot = get_system_bot(settings.WELCOME_BOT)
content = 'whatever'
self.login(user_email)
self.send_personal_message(user, bot, content)
user_messages = message_stream_count(user)
expected_response = (
"Congratulations on your first reply! :tada:\n\n"
"Feel free to continue using this space to practice your new messaging "
"skills. Or, try clicking on some of the stream names to your left!"
)
self.assertEqual(most_recent_message(user).content, expected_response)
# Welcome bot shouldn't respond to further PMs.
self.send_personal_message(user, bot, content)
self.assertEqual(message_stream_count(user), user_messages + 1)
def process_stream_message(to: str, message: EmailMessage) -> None:
subject_header = message.get("Subject", "")
subject = strip_from_subject(subject_header) or "(no topic)"
stream, options = decode_stream_email_address(to)
# Don't remove quotations if message is forwarded, unless otherwise specified:
if "include_quotes" not in options:
options["include_quotes"] = is_forwarded(subject_header)
body = construct_zulip_body(message, stream.realm, **options)
send_zulip(get_system_bot(settings.EMAIL_GATEWAY_BOT), stream, subject,
body)
logger.info(
"Successfully processed email to %s (%s)",
stream.name,
stream.realm.string_id,
)
def send_mm_reply_to_stream(user_profile: UserProfile, stream: Stream,
topic: str, body: str) -> None:
try:
check_send_message(
sender=user_profile,
client=get_client("Internal"),
message_type_name="stream",
message_to=[stream.id],
topic_name=topic,
message_content=body,
)
except JsonableError as error:
error_message = "Error sending message to stream {stream} via message notification email reply:\n{error}".format(
stream=stream.name, error=error.msg)
internal_send_private_message(
get_system_bot(settings.NOTIFICATION_BOT), user_profile,
error_message)
def consume(self, event: Mapping[str, Any]) -> None:
if event['type'] == 'mark_stream_messages_as_read':
user_profile = get_user_profile_by_id(event['user_profile_id'])
client = Client.objects.get(id=event['client_id'])
for stream_id in event['stream_ids']:
# Since the user just unsubscribed, we don't require
# an active Subscription object (otherwise, private
# streams would never be accessible)
(stream, recipient,
sub) = access_stream_by_id(user_profile,
stream_id,
require_active=False)
do_mark_stream_messages_as_read(user_profile, client, stream)
elif event['type'] == 'realm_exported':
realm = Realm.objects.get(id=event['realm_id'])
output_dir = tempfile.mkdtemp(prefix="zulip-export-")
public_url = export_realm_wrapper(realm=realm,
output_dir=output_dir,
threads=6,
upload=True,
public_only=True,
delete_after_upload=True)
assert public_url is not None
# TODO: This enables support for delete after access, and needs to be tested.
export_event = RealmAuditLog.objects.get(id=event['id'])
export_event.extra_data = public_url
export_event.save(update_fields=['extra_data'])
# Send a private message notification letting the user who
# triggered the export know the export finished.
user_profile = get_user_profile_by_id(event['user_profile_id'])
content = "Your data export is complete and has been uploaded here:\n\n%s" % (
public_url, )
internal_send_private_message(realm=user_profile.realm,
sender=get_system_bot(
settings.NOTIFICATION_BOT),
recipient_user=user_profile,
content=content)
# For future frontend use, also notify administrator
# clients that the export happened, including sending the
# url.
notify_export_completed(user_profile, public_url)
def approve_sponsorship(realm: Realm) -> None:
from zerver.lib.actions import do_change_plan_type, internal_send_private_message
do_change_plan_type(realm, Realm.STANDARD_FREE)
customer = get_customer_by_realm(realm)
if customer is not None and customer.sponsorship_pending:
customer.sponsorship_pending = False
customer.save(update_fields=["sponsorship_pending"])
notification_bot = get_system_bot(settings.NOTIFICATION_BOT)
for billing_admin in realm.get_human_billing_admin_users():
with override_language(billing_admin.default_language):
# Using variable to make life easier for translators if these details change.
plan_name = "Zulip Cloud Standard"
emoji = ":tada:"
message = _(
f"Your organization's request for sponsored hosting has been approved! {emoji}.\n"
f"You have been upgraded to {plan_name}, free of charge.")
internal_send_private_message(billing_admin.realm, notification_bot, billing_admin, message)
def highlight_html_differences(s1, s2):
# type: (Text, Text) -> Text
differ = diff_match_patch()
ops = differ.diff_main(s1, s2)
differ.diff_cleanupSemantic(ops)
retval = u''
in_tag = False
idx = 0
while idx < len(ops):
op, text = ops[idx]
text = check_tags(text)
if idx != 0:
prev_op, prev_text = ops[idx - 1]
prev_text = check_tags(prev_text)
# Remove visual offset from editing newlines
if '<p><br>' in text:
text = text.replace('<p><br>', '<p>')
elif prev_text.endswith('<p>') and text.startswith('<br>'):
text = text[4:]
if op == diff_match_patch.DIFF_DELETE:
chunks, in_tag = chunkize(text, in_tag)
retval += highlight_chunks(chunks, highlight_deleted)
elif op == diff_match_patch.DIFF_INSERT:
chunks, in_tag = chunkize(text, in_tag)
retval += highlight_chunks(chunks, highlight_inserted)
elif op == diff_match_patch.DIFF_EQUAL:
chunks, in_tag = chunkize(text, in_tag)
retval += text
idx += 1
if not verify_html(retval):
from zerver.lib.actions import internal_send_message
from zerver.models import get_system_bot
# We probably want more information here
logging.getLogger('').error('HTML diff produced mal-formed HTML')
if settings.ERROR_BOT is not None:
subject = "HTML diff failure on %s" % (platform.node(),)
realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(realm, settings.ERROR_BOT, "stream",
"errors", subject, "HTML diff produced malformed HTML")
return s2
return retval
def test_slow_queries_worker(self) -> None:
error_bot = get_system_bot(settings.ERROR_BOT)
fake_client = self.FakeClient()
# TODO: Rewrite this set part of the test by just mocking
# `is_slow_query` to generate the events.
events = [
{
'query': 'test query (data)'
},
{
'query': 'second test query (data)'
},
]
for event in events:
fake_client.queue.append(('slow_queries', event))
worker = SlowQueryWorker()
time_mock = patch(
'zerver.worker.queue_processors.time.sleep',
side_effect=AbortLoop,
)
send_mock = patch(
'zerver.worker.queue_processors.internal_send_message')
with send_mock as sm, time_mock as tm:
with simulated_queue_client(lambda: fake_client):
try:
worker.setup()
worker.start()
except AbortLoop:
pass
self.assertEqual(tm.call_args[0][0], 60) # should sleep 60 seconds
sm.assert_called_once()
args = [c[0] for c in sm.call_args_list][0]
self.assertEqual(args[0], error_bot.realm)
self.assertEqual(args[1], error_bot.email)
self.assertEqual(args[2], "stream")
self.assertEqual(args[3], "errors")
self.assertEqual(args[4], "testserver: slow queries")
self.assertEqual(
args[5], " test query (data)\n second test query (data)\n")
def send_initial_realm_messages(realm: Realm) -> None:
welcome_bot = get_system_bot(settings.WELCOME_BOT)
# Make sure each stream created in the realm creation process has at least one message below
# Order corresponds to the ordering of the streams on the left sidebar, to make the initial Home
# view slightly less overwhelming
welcome_messages = [
{'stream': Realm.DEFAULT_NOTIFICATION_STREAM_NAME,
'topic': "welcome",
'content': "This is a message on stream `%s` with the topic `welcome`. We'll use this stream "
"for system-generated notifications." % (Realm.DEFAULT_NOTIFICATION_STREAM_NAME,)},
{'stream': Realm.INITIAL_PRIVATE_STREAM_NAME,
'topic': "private streams",
'content': "This is a private stream. Only admins and people you invite "
"to the stream will be able to see that this stream exists."},
{'stream': "general",
'topic': "welcome",
'content': "Welcome to #**general**."},
{'stream': "new members",
'topic': "onboarding",
'content': "A #**new members** stream is great for onboarding new members.\n\nIf you're "
"reading this and aren't the first person here, introduce yourself in a new thread "
"using your name as the topic! Type `c` or click on `New Topic` at the bottom of the "
"screen to start a new topic."},
{'stream': "zulip",
'topic': "topic demonstration",
'content': "Here is a message in one topic. Replies to this message will go to this topic."},
{'stream': "zulip",
'topic': "topic demonstration",
'content': "A second message in this topic. With [turtles](/static/images/cute/turtle.png)!"},
{'stream': "zulip",
'topic': "second topic",
'content': "This is a message in a second topic.\n\nTopics are similar to email subjects, "
"in that each conversation should get its own topic. Keep them short, though; one "
"or two words will do it!"},
] # type: List[Dict[str, str]]
messages = [internal_prep_stream_message(
realm, welcome_bot,
message['stream'], message['topic'], message['content']) for message in welcome_messages]
message_ids = do_send_messages(messages)
# We find the one of our just-sent messages with turtle.png in it,
# and react to it. This is a bit hacky, but works and is kinda a
# 1-off thing.
turtle_message = get_turtle_message(message_ids=message_ids)
do_add_reaction_legacy(welcome_bot, turtle_message, 'turtle')
def send_initial_pms(user: UserProfile) -> None:
organization_setup_text = ""
if user.is_realm_admin:
help_url = user.realm.uri + "/help/getting-your-organization-started-with-zulip"
organization_setup_text = (
" " + _("We also have a guide for [Setting up your organization]({help_url}).")
).format(help_url=help_url)
welcome_msg = _("Hello, and welcome to Zulip!")
if user.realm.demo_organization_scheduled_deletion_date is not None:
welcome_msg += " " + _(
"Note that this is a [demo organization]({demo_org_help_url}) and will be automatically deleted in 30 days."
)
content = "".join(
[
welcome_msg + " ",
_("This is a private message from me, Welcome Bot.") + "\n\n",
"* "
+ _(
"If you are new to Zulip, check out our [Getting started guide]({getting_started_url})!"
)
+ "{organization_setup_text}\n",
"* " + _("[Add a profile picture]({profile_url}).") + "\n",
"* " + _("[Browse and subscribe to streams]({streams_url}).") + "\n",
"* " + _("Download our [mobile and desktop apps]({apps_url}).") + " ",
_("Zulip also works great in a browser.") + "\n",
"* " + _("You can type `?` to learn more about Zulip shortcuts.") + "\n\n",
_("Practice sending a few messages by replying to this conversation.") + " ",
_("Click anywhere on this message or press `r` to reply."),
]
)
content = content.format(
getting_started_url="/help/getting-started-with-zulip",
apps_url="/apps",
profile_url="#settings/profile",
streams_url="#streams/all",
organization_setup_text=organization_setup_text,
demo_org_help_url="/help/demo-organizations",
)
internal_send_private_message(
get_system_bot(settings.WELCOME_BOT, user.realm_id), user, content
)
def zulip_server_error(report: Dict[str, Any]) -> None:
email_subject = "{node}: {message}".format(**report)
logger_str = logger_repr(report)
user_info = user_info_str(report)
deployment = deployment_repr(report)
if report["has_request"]:
request_repr = """\
Request info:
~~~~
- path: {path}
- {method}: {data}
""".format(
**report
)
for field in ["REMOTE_ADDR", "QUERY_STRING", "SERVER_NAME"]:
val = report.get(field.lower())
if field == "QUERY_STRING":
val = clean_data_from_query_parameters(str(val))
request_repr += f'- {field}: "{val}"\n'
request_repr += "~~~~"
else:
request_repr = "Request info: none"
message = f"""{logger_str}
Error generated by {user_info}
~~~~ pytb
{report['stack_trace']}
~~~~
{deployment}
{request_repr}"""
error_bot_realm = get_realm(settings.STAFF_SUBDOMAIN)
error_bot = get_system_bot(settings.ERROR_BOT, error_bot_realm.id)
errors_stream = get_stream("errors", error_bot_realm)
internal_send_stream_message(
error_bot,
errors_stream,
format_email_subject(email_subject),
message,
)
def __init__(self, host_url: str, sockjs_url: str, sender_email: str,
run_on_start: Callable[..., None], validate_ssl: bool=True,
**run_kwargs: Any) -> None:
# NOTE: Callable should take a WebsocketClient & kwargs, but this is not standardised
self.validate_ssl = validate_ssl
self.auth_email = sender_email
self.user_profile = get_system_bot(sender_email)
self.request_id_number = 0
self.parsed_host_url = urlparse(host_url)
self.sockjs_url = sockjs_url
self.cookie_dict = self._login()
self.cookie_str = self._get_cookie_header(self.cookie_dict)
self.events_data = self._get_queue_events(self.cookie_str)
self.ioloop_instance = IOLoop.instance()
self.run_on_start = run_on_start
self.run_kwargs = run_kwargs
self.scheme_dict = {'http': 'ws', 'https': 'wss'}
self.ws = None # type: Optional[WebSocketClientConnection]
def zulip_browser_error(report: Dict[str, Any]) -> None:
email_subject = "JS error: {user_email}".format(**report)
user_info = user_info_str(report)
body = f"User: {user_info}\n"
body += "Message: {message}\n".format(**report)
error_bot_realm = get_realm(settings.STAFF_SUBDOMAIN)
error_bot = get_system_bot(settings.ERROR_BOT, error_bot_realm.id)
errors_stream = get_stream("errors", error_bot_realm)
internal_send_stream_message(
error_bot,
errors_stream,
format_email_subject(email_subject),
body,
)
def consume_batch(self, slow_queries):
# type: (List[Dict[str, Any]]) -> None
for query in slow_queries:
logging.info("Slow query: %s" % (query))
if settings.ERROR_BOT is None:
return
if len(slow_queries) > 0:
topic = "%s: slow queries" % (settings.EXTERNAL_HOST,)
content = ""
for query in slow_queries:
content += " %s\n" % (query,)
error_bot_realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(error_bot_realm, settings.ERROR_BOT,
"stream", "logs", topic, content)
def test_slow_query_log(self, mock_logging_info: Mock) -> None:
error_bot = get_system_bot(settings.ERROR_BOT)
create_stream_if_needed(error_bot.realm,
settings.SLOW_QUERY_LOGS_STREAM)
self.log_data['time_started'] = time.time() - self.SLOW_QUERY_TIME
write_log_line(self.log_data,
path='/socket/open',
method='SOCKET',
remote_ip='123.456.789.012',
requestor_for_logs='unknown',
client_name='?')
last_message = self.get_last_message()
self.assertEqual(last_message.sender.email, "*****@*****.**")
self.assertIn("logs", str(last_message.recipient))
self.assertEqual(last_message.topic_name(), "testserver: slow queries")
self.assertRegexpMatches(last_message.content,
r"123\.456\.789\.012 SOCKET 200 10\.\ds .*")
def test_slow_queries_worker(self) -> None:
error_bot = get_system_bot(settings.ERROR_BOT)
fake_client = self.FakeClient()
worker = SlowQueryWorker()
time_mock = patch(
'zerver.worker.queue_processors.time.sleep',
side_effect=AbortLoop,
)
send_mock = patch(
'zerver.worker.queue_processors.internal_send_message')
with send_mock as sm, time_mock as tm:
with simulated_queue_client(lambda: fake_client):
try:
worker.setup()
# `write_log_line` is where we publish slow queries to the queue.
with patch('zerver.middleware.is_slow_query',
return_value=True):
write_log_line(log_data=dict(test='data'),
email='*****@*****.**',
remote_ip='127.0.0.1',
client_name='website',
path='/test/',
method='GET')
worker.start()
except AbortLoop:
pass
self.assertEqual(tm.call_args[0][0], 60) # should sleep 60 seconds
sm.assert_called_once()
args = [c[0] for c in sm.call_args_list][0]
self.assertEqual(args[0], error_bot.realm)
self.assertEqual(args[1], error_bot.email)
self.assertEqual(args[2], "stream")
self.assertEqual(args[3], "errors")
self.assertEqual(args[4], "testserver: slow queries")
# Testing for specific query times can lead to test discrepancies.
logging_info = re.sub(r'\(db: [0-9]+ms/13q\)', '', args[5])
self.assertEqual(
logging_info, ' 127.0.0.1 GET 200 -1000ms '
' /test/ ([email protected] via website) ([email protected])\n')
def send_initial_realm_messages(realm: Realm) -> None:
welcome_bot = get_system_bot(settings.WELCOME_BOT)
# Make sure each stream created in the realm creation process has at least one message below
# Order corresponds to the ordering of the streams on the left sidebar, to make the initial Home
# view slightly less overwhelming
welcome_messages = [
{'stream': Realm.INITIAL_PRIVATE_STREAM_NAME,
'topic': "private streams",
'content': "This is a private stream, as indicated by the "
"lock icon next to the stream name. Private streams are only visible to stream members. "
"\n\nTo manage this stream, go to [Stream settings](#streams/subscribed) and click on "
"`%(initial_private_stream_name)s`."},
{'stream': Realm.DEFAULT_NOTIFICATION_STREAM_NAME,
'topic': "topic demonstration",
'content': "This is a message on stream #**%(default_notification_stream_name)s** with the "
"topic `topic demonstration`."},
{'stream': Realm.DEFAULT_NOTIFICATION_STREAM_NAME,
'topic': "topic demonstration",
'content': "Topics are a lightweight tool to keep conversations organized. "
"You can learn more about topics at [Streams and topics](/help/about-streams-and-topics). "},
{'stream': realm.DEFAULT_NOTIFICATION_STREAM_NAME,
'topic': "swimming turtles",
'content': "This is a message on stream #**%(default_notification_stream_name)s** with the "
"topic `swimming turtles`. "
"\n\n[](/static/images/cute/turtle.png)"
"\n\n[Start a new topic](/help/start-a-new-topic) any time you're not replying to a "
"previous message."},
] # type: List[Dict[str, str]]
messages = [internal_prep_stream_message_by_name(
realm, welcome_bot, message['stream'], message['topic'],
message['content'] % {
'initial_private_stream_name': Realm.INITIAL_PRIVATE_STREAM_NAME,
'default_notification_stream_name': Realm.DEFAULT_NOTIFICATION_STREAM_NAME,
}
) for message in welcome_messages]
message_ids = do_send_messages(messages)
# We find the one of our just-sent messages with turtle.png in it,
# and react to it. This is a bit hacky, but works and is kinda a
# 1-off thing.
turtle_message = Message.objects.get(
id__in=message_ids,
content__icontains='cute/turtle.png')
do_add_reaction_legacy(welcome_bot, turtle_message, 'turtle')
def process_one_batch(self):
# type: () -> None
slow_queries = self.q.drain_queue("slow_queries", json=True)
if settings.ERROR_BOT is None:
return
if len(slow_queries) > 0:
topic = "%s: slow queries" % (settings.EXTERNAL_HOST,)
content = ""
for query in slow_queries:
content += " %s\n" % (query,)
error_bot_realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(error_bot_realm, settings.ERROR_BOT,
"stream", "logs", topic, content)
reset_queries()
def zulip_browser_error(report: Dict[str, Any]) -> None:
email_subject = "JS error: {user_email}".format(**report)
user_info = user_info_str(report)
body = f"User: {user_info}\n"
body += "Message: {message}\n".format(**report)
error_bot = get_system_bot(settings.ERROR_BOT)
realm = error_bot.realm
errors_stream = get_stream('errors', realm)
internal_send_stream_message(
realm,
error_bot,
errors_stream,
format_email_subject(email_subject),
body,
)
def send_change_stream_message_retention_days_notification(
user_profile: UserProfile, stream: Stream, old_value: Optional[int],
new_value: Optional[int]) -> None:
sender = get_system_bot(settings.NOTIFICATION_BOT, user_profile.realm_id)
user_mention = silent_mention_syntax_for_user(user_profile)
# If switching from or to the organization's default retention policy,
# we want to take the realm's default into account.
if old_value is None:
old_value = stream.realm.message_retention_days
if new_value is None:
new_value = stream.realm.message_retention_days
with override_language(stream.realm.default_language):
if old_value == Stream.MESSAGE_RETENTION_SPECIAL_VALUES_MAP[
"unlimited"]:
old_retention_period = _("Forever")
new_retention_period = f"{new_value} days"
summary_line = f"Messages in this stream will now be automatically deleted {new_value} days after they are sent."
elif new_value == Stream.MESSAGE_RETENTION_SPECIAL_VALUES_MAP[
"unlimited"]:
old_retention_period = f"{old_value} days"
new_retention_period = _("Forever")
summary_line = _(
"Messages in this stream will now be retained forever.")
else:
old_retention_period = f"{old_value} days"
new_retention_period = f"{new_value} days"
summary_line = f"Messages in this stream will now be automatically deleted {new_value} days after they are sent."
notification_string = _(
"{user} has changed the [message retention period](/help/message-retention-policy) for this stream:\n"
"* **Old retention period**: {old_retention_period}\n"
"* **New retention period**: {new_retention_period}\n\n"
"{summary_line}")
notification_string = notification_string.format(
user=user_mention,
old_retention_period=old_retention_period,
new_retention_period=new_retention_period,
summary_line=summary_line,
)
internal_send_stream_message(sender, stream,
Realm.STREAM_EVENTS_NOTIFICATION_TOPIC,
notification_string)
def process_one_batch(self):
# type: () -> None
slow_queries = self.q.drain_queue("slow_queries", json=True)
if settings.ERROR_BOT is None:
return
if len(slow_queries) > 0:
topic = "%s: slow queries" % (settings.EXTERNAL_HOST, )
content = ""
for query in slow_queries:
content += " %s\n" % (query, )
error_bot_realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(error_bot_realm, settings.ERROR_BOT,
"stream", "logs", topic, content)
reset_queries()
def send_initial_pms(user: UserProfile) -> None:
organization_setup_text = ""
# We need to override the language in this code path, because it's
# called from account registration, which is a pre-account API
# request and thus may not have the user's language context yet.
with override_language(user.default_language):
if user.is_realm_admin:
help_url = user.realm.uri + "/help/getting-your-organization-started-with-zulip"
organization_setup_text = (" " + _(
"We also have a guide for [Setting up your organization]({help_url})."
)).format(help_url=help_url)
welcome_msg = _("Hello, and welcome to Zulip!") + "👋"
demo_org_warning = ""
if user.realm.demo_organization_scheduled_deletion_date is not None:
demo_org_warning = (_(
"Note that this is a [demo organization]({demo_org_help_url}) and will be "
"**automatically deleted** in 30 days.") + "\n\n")
content = "".join([
welcome_msg + " ",
_("This is a private message from me, Welcome Bot.") + "\n\n",
_("If you are new to Zulip, check out our [Getting started guide]({getting_started_url})!"
),
"{organization_setup_text}" + "\n\n",
"{demo_org_warning}",
_("I can also help you get set up! Just click anywhere on this message or press `r` to reply."
) + "\n\n",
_("Here are a few messages I understand:") + " ",
bot_commands(),
])
content = content.format(
organization_setup_text=organization_setup_text,
demo_org_warning=demo_org_warning,
demo_org_help_url="/help/demo-organizations",
getting_started_url="/help/getting-started-with-zulip",
)
internal_send_private_message(
get_system_bot(settings.WELCOME_BOT, user.realm_id), user, content)
def consume_batch(self, slow_query_events: List[Dict[str, Any]]) -> None:
for event in slow_query_events:
logging.info("Slow query: %s" % (event["query"],))
if settings.SLOW_QUERY_LOGS_STREAM is None:
return
if settings.ERROR_BOT is None:
return
if len(slow_query_events) > 0:
topic = "%s: slow queries" % (settings.EXTERNAL_HOST,)
content = ""
for event in slow_query_events:
content += " %s\n" % (event["query"],)
error_bot_realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(error_bot_realm, settings.ERROR_BOT,
"stream", settings.SLOW_QUERY_LOGS_STREAM, topic, content)
def send_change_stream_post_policy_notification(
stream: Stream, *, old_post_policy: int, new_post_policy: int,
acting_user: UserProfile) -> None:
sender = get_system_bot(settings.NOTIFICATION_BOT, acting_user.realm_id)
user_mention = silent_mention_syntax_for_user(acting_user)
with override_language(stream.realm.default_language):
notification_string = _(
"{user} changed the [posting permissions](/help/stream-sending-policy) "
"for this stream:\n\n"
"* **Old permissions**: {old_policy}.\n"
"* **New permissions**: {new_policy}.\n")
notification_string = notification_string.format(
user=user_mention,
old_policy=Stream.POST_POLICIES[old_post_policy],
new_policy=Stream.POST_POLICIES[new_post_policy],
)
internal_send_stream_message(sender, stream,
Realm.STREAM_EVENTS_NOTIFICATION_TOPIC,
notification_string)
def zulip_server_error(report):
# type: (Dict[str, Any]) -> None
subject = '%(node)s: %(message)s' % (report)
stack_trace = report['stack_trace'] or "No stack trace available"
user_info = user_info_str(report)
request_repr = ("Request info:\n~~~~\n"
"- path: %(path)s\n"
"- %(method)s: %(data)s\n") % (report)
for field in ["REMOTE_ADDR", "QUERY_STRING", "SERVER_NAME"]:
request_repr += "- %s: \"%s\"\n" % (field, report.get(field.lower()))
request_repr += "~~~~"
realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(
realm, settings.ERROR_BOT, "stream", "errors", format_subject(subject),
"Error generated by %s\n\n~~~~ pytb\n%s\n\n~~~~\n%s" %
(user_info, stack_trace, request_repr))
def test_recipient_for_user_ids(self) -> None:
hamlet = self.example_user('hamlet')
othello = self.example_user('othello')
cross_realm_bot = get_system_bot(settings.WELCOME_BOT)
sender = self.example_user('iago')
recipient_user_ids = [hamlet.id, othello.id, cross_realm_bot.id]
result = recipient_for_user_ids(recipient_user_ids, sender)
recipient = get_display_recipient(result)
recipient_ids = [
recipient[0]['id'],
recipient[1]['id'], # type: ignore
recipient[2]['id'],
recipient[3]['id']
] # type: ignore
expected_recipient_ids = [
hamlet.id, othello.id, sender.id, cross_realm_bot.id
]
self.assertEqual(set(recipient_ids), set(expected_recipient_ids))
def consume_batch(self, slow_queries: List[Any]) -> None:
for query in slow_queries:
logging.info("Slow query: %s" % (query))
if settings.SLOW_QUERY_LOGS_STREAM is None:
return
if settings.ERROR_BOT is None:
return
if len(slow_queries) > 0:
topic = "%s: slow queries" % (settings.EXTERNAL_HOST,)
content = ""
for query in slow_queries:
content += " %s\n" % (query,)
error_bot_realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(error_bot_realm, settings.ERROR_BOT,
"stream", settings.SLOW_QUERY_LOGS_STREAM, topic, content)
def extract_and_upload_attachments(message: message.Message, realm: Realm) -> str:
user_profile = get_system_bot(settings.EMAIL_GATEWAY_BOT)
attachment_links = []
for part in message.walk():
content_type = part.get_content_type()
filename = part.get_filename()
if filename:
attachment = part.get_payload(decode=True)
if isinstance(attachment, bytes):
s3_url = upload_message_file(filename, len(attachment), content_type,
attachment,
user_profile,
target_realm=realm)
formatted_link = "[%s](%s)" % (filename, s3_url)
attachment_links.append(formatted_link)
else:
logger.warning("Payload is not bytes (invalid attachment %s in message from %s)." %
(filename, message.get("From")))
return '\n'.join(attachment_links)
def zulip_server_error(report):
# type: (Dict[str, Any]) -> None
subject = '%(node)s: %(message)s' % (report)
stack_trace = report['stack_trace'] or "No stack trace available"
user_info = user_info_str(report)
request_repr = (
"Request info:\n~~~~\n"
"- path: %(path)s\n"
"- %(method)s: %(data)s\n") % (report)
for field in ["REMOTE_ADDR", "QUERY_STRING", "SERVER_NAME"]:
request_repr += "- %s: \"%s\"\n" % (field, report.get(field.lower()))
request_repr += "~~~~"
realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(realm, settings.ERROR_BOT,
"stream", "errors", format_subject(subject),
"Error generated by %s\n\n~~~~ pytb\n%s\n\n~~~~\n%s" % (
user_info, stack_trace, request_repr))
def create_internal_realm() -> None:
realm = Realm.objects.create(string_id=settings.SYSTEM_BOT_REALM)
# Create the "website" and "API" clients:
get_client("website")
get_client("API")
internal_bots = [(bot['name'],
bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN, ))
for bot in settings.INTERNAL_BOTS]
create_users(realm, internal_bots, bot_type=UserProfile.DEFAULT_BOT)
# Set the owners for these bots to the bots themselves
bots = UserProfile.objects.filter(
email__in=[bot_info[1] for bot_info in internal_bots])
for bot in bots:
bot.bot_owner = bot
bot.save()
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
do_change_is_admin(email_gateway_bot, True, permission="api_super_user")
def send_initial_pms(user: UserProfile) -> None:
organization_setup_text = ""
if user.is_realm_admin:
help_url = user.realm.uri + "/help/getting-your-organization-started-with-zulip"
organization_setup_text = ("* " + _(
"[Read the guide]({help_url}) for getting your organization started with Zulip"
) + "\n").format(help_url=help_url)
welcome_msg = _("Hello, and welcome to Zulip!")
if user.realm.demo_organization_scheduled_deletion_date is not None:
welcome_msg += " " + _(
"Note that this is a [demo organization]({demo_org_help_url}) and will be automatically deleted in 30 days."
)
content = "".join([
welcome_msg + "\n\n",
_("This is a private message from me, Welcome Bot.") + " ",
_("Here are some tips to get you started:") + "\n",
"* " + _("Download our [Desktop and mobile apps]({apps_url})") + "\n",
"* " +
_("Customize your account and notifications on your [Settings page]({settings_url})"
) + "\n",
"* " + _("Type `?` to check out Zulip's keyboard shortcuts") +
"\n {organization_setup_text}\n",
_("The most important shortcut is `r` to reply.") + "\n\n",
_("Practice sending a few messages by replying to this conversation.")
+ " ",
_("If you're not into keyboards, that's okay too; "
"clicking anywhere on this message will also do the trick!"),
])
content = content.format(
apps_url="/apps",
settings_url="#settings",
organization_setup_text=organization_setup_text,
demo_org_help_url="/help/demo-organizations",
)
internal_send_private_message(
get_system_bot(settings.WELCOME_BOT, user.realm_id), user, content)
def highlight_html_differences(s1, s2):
# type: (Text, Text) -> Text
differ = diff_match_patch()
ops = differ.diff_main(s1, s2)
differ.diff_cleanupSemantic(ops)
retval = u''
in_tag = False
idx = 0
while idx < len(ops):
op, text = ops[idx]
next_op = None
if idx != len(ops) - 1:
next_op, next_text = ops[idx + 1]
if op == diff_match_patch.DIFF_DELETE:
chunks, in_tag = chunkize(text, in_tag)
retval += highlight_chunks(chunks, highlight_deleted)
elif op == diff_match_patch.DIFF_INSERT:
chunks, in_tag = chunkize(text, in_tag)
retval += highlight_chunks(chunks, highlight_inserted)
elif op == diff_match_patch.DIFF_EQUAL:
chunks, in_tag = chunkize(text, in_tag)
retval += text
idx += 1
if not verify_html(retval):
from zerver.lib.actions import internal_send_message
from zerver.models import get_system_bot
# We probably want more information here
logging.getLogger('').error('HTML diff produced mal-formed HTML')
if settings.ERROR_BOT is not None:
subject = "HTML diff failure on %s" % (platform.node(), )
realm = get_system_bot(settings.ERROR_BOT).realm
internal_send_message(realm, settings.ERROR_BOT, "stream",
"errors", subject,
"HTML diff produced malformed HTML")
return s2
return retval
def test_slow_queries_worker(self) -> None:
error_bot = get_system_bot(settings.ERROR_BOT)
fake_client = self.FakeClient()
events = [
'test query (data)',
'second test query (data)',
]
for event in events:
fake_client.queue.append(('slow_queries', event))
worker = SlowQueryWorker()
time_mock = patch(
'zerver.worker.queue_processors.time.sleep',
side_effect=AbortLoop,
)
send_mock = patch(
'zerver.worker.queue_processors.internal_send_message'
)
with send_mock as sm, time_mock as tm:
with simulated_queue_client(lambda: fake_client):
try:
worker.setup()
worker.start()
except AbortLoop:
pass
self.assertEqual(tm.call_args[0][0], 60) # should sleep 60 seconds
sm.assert_called_once()
args = [c[0] for c in sm.call_args_list][0]
self.assertEqual(args[0], error_bot.realm)
self.assertEqual(args[1], error_bot.email)
self.assertEqual(args[2], "stream")
self.assertEqual(args[3], "errors")
self.assertEqual(args[4], "testserver: slow queries")
self.assertEqual(args[5], " test query (data)\n second test query (data)\n")
def send_initial_pms(user: UserProfile) -> None:
organization_setup_text = ""
if user.is_realm_admin:
help_url = user.realm.uri + "/help/getting-your-organization-started-with-zulip"
organization_setup_text = ("* [Read the guide](%s) for getting your organization "
"started with Zulip\n" % (help_url,))
content = (
"Hello, and welcome to Zulip!\n\nThis is a private message from me, Welcome Bot. "
"Here are some tips to get you started:\n"
"* Download our [Desktop and mobile apps](/apps)\n"
"* Customize your account and notifications on your [Settings page](#settings)\n"
"* Type `?` to check out Zulip's keyboard shortcuts\n"
"%s"
"\n"
"The most important shortcut is `r` to reply.\n\n"
"Practice sending a few messages by replying to this conversation. If you're not into "
"keyboards, that's okay too; clicking anywhere on this message will also do the trick!") \
% (organization_setup_text,)
internal_send_private_message(user.realm, get_system_bot(settings.WELCOME_BOT),
user, content)
def do_import_realm(import_dir: Path, subdomain: str) -> Realm:
logging.info("Importing realm dump %s" % (import_dir,))
if not os.path.exists(import_dir):
raise Exception("Missing import directory!")
realm_data_filename = os.path.join(import_dir, "realm.json")
if not os.path.exists(realm_data_filename):
raise Exception("Missing realm.json file!")
logging.info("Importing realm data from %s" % (realm_data_filename,))
with open(realm_data_filename) as f:
data = ujson.load(f)
bulk_import_client(data, Client, 'zerver_client')
# We don't import the Stream model yet, since it depends on Realm,
# which isn't imported yet. But we need the Stream model IDs for
# notifications_stream.
update_model_ids(Stream, data, 'stream')
re_map_foreign_keys(data, 'zerver_realm', 'notifications_stream', related_table="stream")
re_map_foreign_keys(data, 'zerver_realm', 'signup_notifications_stream', related_table="stream")
fix_datetime_fields(data, 'zerver_realm')
# Fix realm subdomain information
data['zerver_realm'][0]['string_id'] = subdomain
data['zerver_realm'][0]['name'] = subdomain
fix_realm_authentication_bitfield(data, 'zerver_realm', 'authentication_methods')
update_model_ids(Realm, data, 'realm')
realm = Realm(**data['zerver_realm'][0])
if settings.BILLING_ENABLED:
realm.plan_type = Realm.LIMITED
else:
realm.plan_type = Realm.SELF_HOSTED
if realm.notifications_stream_id is not None:
notifications_stream_id = int(realm.notifications_stream_id) # type: Optional[int]
else:
notifications_stream_id = None
realm.notifications_stream_id = None
if realm.signup_notifications_stream_id is not None:
signup_notifications_stream_id = int(realm.signup_notifications_stream_id) # type: Optional[int]
else:
signup_notifications_stream_id = None
realm.signup_notifications_stream_id = None
realm.save()
# Email tokens will automatically be randomly generated when the
# Stream objects are created by Django.
fix_datetime_fields(data, 'zerver_stream')
re_map_foreign_keys(data, 'zerver_stream', 'realm', related_table="realm")
bulk_import_model(data, Stream)
realm.notifications_stream_id = notifications_stream_id
realm.signup_notifications_stream_id = signup_notifications_stream_id
realm.save()
# Remap the user IDs for notification_bot and friends to their
# appropriate IDs on this server
for item in data['zerver_userprofile_crossrealm']:
logging.info("Adding to ID map: %s %s" % (item['id'], get_system_bot(item['email']).id))
new_user_id = get_system_bot(item['email']).id
update_id_map(table='user_profile', old_id=item['id'], new_id=new_user_id)
new_recipient_id = Recipient.objects.get(type=Recipient.PERSONAL, type_id=new_user_id).id
update_id_map(table='recipient', old_id=item['recipient_id'], new_id=new_recipient_id)
# Merge in zerver_userprofile_mirrordummy
data['zerver_userprofile'] = data['zerver_userprofile'] + data['zerver_userprofile_mirrordummy']
del data['zerver_userprofile_mirrordummy']
data['zerver_userprofile'].sort(key=lambda r: r['id'])
# To remap foreign key for UserProfile.last_active_message_id
update_message_foreign_keys(import_dir)
fix_datetime_fields(data, 'zerver_userprofile')
update_model_ids(UserProfile, data, 'user_profile')
re_map_foreign_keys(data, 'zerver_userprofile', 'realm', related_table="realm")
re_map_foreign_keys(data, 'zerver_userprofile', 'bot_owner', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_userprofile', 'default_sending_stream',
related_table="stream")
re_map_foreign_keys(data, 'zerver_userprofile', 'default_events_register_stream',
related_table="stream")
re_map_foreign_keys(data, 'zerver_userprofile', 'last_active_message_id',
related_table="message", id_field=True)
for user_profile_dict in data['zerver_userprofile']:
user_profile_dict['password'] = None
user_profile_dict['api_key'] = generate_api_key()
# Since Zulip doesn't use these permissions, drop them
del user_profile_dict['user_permissions']
del user_profile_dict['groups']
user_profiles = [UserProfile(**item) for item in data['zerver_userprofile']]
for user_profile in user_profiles:
user_profile.set_unusable_password()
UserProfile.objects.bulk_create(user_profiles)
re_map_foreign_keys(data, 'zerver_defaultstream', 'stream', related_table="stream")
re_map_foreign_keys(data, 'zerver_realmemoji', 'author', related_table="user_profile")
for (table, model, related_table) in realm_tables:
re_map_foreign_keys(data, table, 'realm', related_table="realm")
update_model_ids(model, data, related_table)
bulk_import_model(data, model)
if 'zerver_huddle' in data:
update_model_ids(Huddle, data, 'huddle')
# We don't import Huddle yet, since we don't have the data to
# compute huddle hashes until we've imported some of the
# tables below.
# TODO: double-check this.
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="stream",
recipient_field=True, id_field=True)
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="user_profile",
recipient_field=True, id_field=True)
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="huddle",
recipient_field=True, id_field=True)
update_model_ids(Recipient, data, 'recipient')
bulk_import_model(data, Recipient)
re_map_foreign_keys(data, 'zerver_subscription', 'user_profile', related_table="user_profile")
get_huddles_from_subscription(data, 'zerver_subscription')
re_map_foreign_keys(data, 'zerver_subscription', 'recipient', related_table="recipient")
update_model_ids(Subscription, data, 'subscription')
bulk_import_model(data, Subscription)
if 'zerver_realmauditlog' in data:
fix_datetime_fields(data, 'zerver_realmauditlog')
re_map_foreign_keys(data, 'zerver_realmauditlog', 'realm', related_table="realm")
re_map_foreign_keys(data, 'zerver_realmauditlog', 'modified_user',
related_table='user_profile')
re_map_foreign_keys(data, 'zerver_realmauditlog', 'acting_user',
related_table='user_profile')
re_map_foreign_keys(data, 'zerver_realmauditlog', 'modified_stream',
related_table="stream")
update_model_ids(RealmAuditLog, data, related_table="realmauditlog")
bulk_import_model(data, RealmAuditLog)
else:
logging.info('about to call create_subscription_events')
create_subscription_events(
data=data,
realm_id=realm.id,
)
logging.info('done with create_subscription_events')
if 'zerver_huddle' in data:
process_huddle_hash(data, 'zerver_huddle')
bulk_import_model(data, Huddle)
if 'zerver_userhotspot' in data:
fix_datetime_fields(data, 'zerver_userhotspot')
re_map_foreign_keys(data, 'zerver_userhotspot', 'user', related_table='user_profile')
update_model_ids(UserHotspot, data, 'userhotspot')
bulk_import_model(data, UserHotspot)
if 'zerver_mutedtopic' in data:
re_map_foreign_keys(data, 'zerver_mutedtopic', 'user_profile', related_table='user_profile')
re_map_foreign_keys(data, 'zerver_mutedtopic', 'stream', related_table='stream')
re_map_foreign_keys(data, 'zerver_mutedtopic', 'recipient', related_table='recipient')
update_model_ids(MutedTopic, data, 'mutedtopic')
bulk_import_model(data, MutedTopic)
if 'zerver_service' in data:
re_map_foreign_keys(data, 'zerver_service', 'user_profile', related_table='user_profile')
fix_service_tokens(data, 'zerver_service')
update_model_ids(Service, data, 'service')
bulk_import_model(data, Service)
if 'zerver_usergroup' in data:
re_map_foreign_keys(data, 'zerver_usergroup', 'realm', related_table='realm')
re_map_foreign_keys_many_to_many(data, 'zerver_usergroup',
'members', related_table='user_profile')
update_model_ids(UserGroup, data, 'usergroup')
bulk_import_model(data, UserGroup)
re_map_foreign_keys(data, 'zerver_usergroupmembership',
'user_group', related_table='usergroup')
re_map_foreign_keys(data, 'zerver_usergroupmembership',
'user_profile', related_table='user_profile')
update_model_ids(UserGroupMembership, data, 'usergroupmembership')
bulk_import_model(data, UserGroupMembership)
if 'zerver_botstoragedata' in data:
re_map_foreign_keys(data, 'zerver_botstoragedata', 'bot_profile', related_table='user_profile')
update_model_ids(BotStorageData, data, 'botstoragedata')
bulk_import_model(data, BotStorageData)
if 'zerver_botconfigdata' in data:
re_map_foreign_keys(data, 'zerver_botconfigdata', 'bot_profile', related_table='user_profile')
update_model_ids(BotConfigData, data, 'botconfigdata')
bulk_import_model(data, BotConfigData)
fix_datetime_fields(data, 'zerver_userpresence')
re_map_foreign_keys(data, 'zerver_userpresence', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_userpresence', 'client', related_table='client')
update_model_ids(UserPresence, data, 'user_presence')
bulk_import_model(data, UserPresence)
fix_datetime_fields(data, 'zerver_useractivity')
re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client')
update_model_ids(UserActivity, data, 'useractivity')
bulk_import_model(data, UserActivity)
fix_datetime_fields(data, 'zerver_useractivityinterval')
re_map_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', related_table="user_profile")
update_model_ids(UserActivityInterval, data, 'useractivityinterval')
bulk_import_model(data, UserActivityInterval)
re_map_foreign_keys(data, 'zerver_customprofilefield', 'realm', related_table="realm")
update_model_ids(CustomProfileField, data, related_table="customprofilefield")
bulk_import_model(data, CustomProfileField)
re_map_foreign_keys(data, 'zerver_customprofilefieldvalue', 'user_profile',
related_table="user_profile")
re_map_foreign_keys(data, 'zerver_customprofilefieldvalue', 'field',
related_table="customprofilefield")
fix_customprofilefield(data)
update_model_ids(CustomProfileFieldValue, data, related_table="customprofilefieldvalue")
bulk_import_model(data, CustomProfileFieldValue)
# Import uploaded files and avatars
import_uploads(os.path.join(import_dir, "avatars"), processing_avatars=True)
import_uploads(os.path.join(import_dir, "uploads"))
# We need to have this check as the emoji files are only present in the data
# importer from slack
# For Zulip export, this doesn't exist
if os.path.exists(os.path.join(import_dir, "emoji")):
import_uploads(os.path.join(import_dir, "emoji"), processing_emojis=True)
# Import zerver_message and zerver_usermessage
import_message_data(import_dir)
re_map_foreign_keys(data, 'zerver_reaction', 'message', related_table="message")
re_map_foreign_keys(data, 'zerver_reaction', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_reaction', 'emoji_code', related_table="realmemoji", id_field=True,
reaction_field=True)
update_model_ids(Reaction, data, 'reaction')
bulk_import_model(data, Reaction)
# Do attachments AFTER message data is loaded.
# TODO: de-dup how we read these json files.
fn = os.path.join(import_dir, "attachment.json")
if not os.path.exists(fn):
raise Exception("Missing attachment.json file!")
logging.info("Importing attachment data from %s" % (fn,))
with open(fn) as f:
data = ujson.load(f)
import_attachments(data)
return realm
def do_import_realm(import_dir: Path, subdomain: str) -> Realm:
logging.info("Importing realm dump %s" % (import_dir,))
if not os.path.exists(import_dir):
raise Exception("Missing import directory!")
realm_data_filename = os.path.join(import_dir, "realm.json")
if not os.path.exists(realm_data_filename):
raise Exception("Missing realm.json file!")
logging.info("Importing realm data from %s" % (realm_data_filename,))
with open(realm_data_filename) as f:
data = ujson.load(f)
update_model_ids(Stream, data, 'zerver_stream', 'stream')
re_map_foreign_keys(data, 'zerver_realm', 'notifications_stream', related_table="stream")
fix_datetime_fields(data, 'zerver_realm')
# Fix realm subdomain information
data['zerver_realm'][0]['string_id'] = subdomain
data['zerver_realm'][0]['name'] = subdomain
fix_realm_authentication_bitfield(data, 'zerver_realm', 'authentication_methods')
update_model_ids(Realm, data, 'zerver_realm', 'realm')
realm = Realm(**data['zerver_realm'][0])
if realm.notifications_stream_id is not None:
notifications_stream_id = int(realm.notifications_stream_id) # type: Optional[int]
else:
notifications_stream_id = None
realm.notifications_stream_id = None
realm.save()
bulk_import_client(data, Client, 'zerver_client')
# Email tokens will automatically be randomly generated when the
# Stream objects are created by Django.
fix_datetime_fields(data, 'zerver_stream')
re_map_foreign_keys(data, 'zerver_stream', 'realm', related_table="realm")
bulk_import_model(data, Stream, 'zerver_stream')
realm.notifications_stream_id = notifications_stream_id
realm.save()
re_map_foreign_keys(data, 'zerver_defaultstream', 'stream', related_table="stream")
re_map_foreign_keys(data, 'zerver_realmemoji', 'author', related_table="user_profile")
for (table, model, related_table) in realm_tables:
re_map_foreign_keys(data, table, 'realm', related_table="realm")
update_model_ids(model, data, table, related_table)
bulk_import_model(data, model, table)
# Remap the user IDs for notification_bot and friends to their
# appropriate IDs on this server
for item in data['zerver_userprofile_crossrealm']:
logging.info("Adding to ID map: %s %s" % (item['id'], get_system_bot(item['email']).id))
new_user_id = get_system_bot(item['email']).id
update_id_map(table='user_profile', old_id=item['id'], new_id=new_user_id)
# Merge in zerver_userprofile_mirrordummy
data['zerver_userprofile'] = data['zerver_userprofile'] + data['zerver_userprofile_mirrordummy']
del data['zerver_userprofile_mirrordummy']
data['zerver_userprofile'].sort(key=lambda r: r['id'])
# To remap foreign key for UserProfile.last_active_message_id
update_message_foreign_keys(import_dir)
fix_datetime_fields(data, 'zerver_userprofile')
update_model_ids(UserProfile, data, 'zerver_userprofile', 'user_profile')
re_map_foreign_keys(data, 'zerver_userprofile', 'realm', related_table="realm")
re_map_foreign_keys(data, 'zerver_userprofile', 'bot_owner', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_userprofile', 'default_sending_stream',
related_table="stream")
re_map_foreign_keys(data, 'zerver_userprofile', 'default_events_register_stream',
related_table="stream")
re_map_foreign_keys(data, 'zerver_userprofile', 'last_active_message_id',
related_table="message", id_field=True)
for user_profile_dict in data['zerver_userprofile']:
user_profile_dict['password'] = None
user_profile_dict['api_key'] = random_api_key()
# Since Zulip doesn't use these permissions, drop them
del user_profile_dict['user_permissions']
del user_profile_dict['groups']
user_profiles = [UserProfile(**item) for item in data['zerver_userprofile']]
for user_profile in user_profiles:
user_profile.set_unusable_password()
UserProfile.objects.bulk_create(user_profiles)
if 'zerver_huddle' in data:
bulk_import_model(data, Huddle, 'zerver_huddle')
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="stream",
recipient_field=True, id_field=True)
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="user_profile",
recipient_field=True, id_field=True)
update_model_ids(Recipient, data, 'zerver_recipient', 'recipient')
bulk_import_model(data, Recipient, 'zerver_recipient')
re_map_foreign_keys(data, 'zerver_subscription', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_subscription', 'recipient', related_table="recipient")
update_model_ids(Subscription, data, 'zerver_subscription', 'subscription')
bulk_import_model(data, Subscription, 'zerver_subscription')
fix_datetime_fields(data, 'zerver_userpresence')
re_map_foreign_keys(data, 'zerver_userpresence', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_userpresence', 'client', related_table='client')
update_model_ids(UserPresence, data, 'zerver_userpresence', 'user_presence')
bulk_import_model(data, UserPresence, 'zerver_userpresence')
fix_datetime_fields(data, 'zerver_useractivity')
re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client')
update_model_ids(UserActivity, data, 'zerver_useractivity', 'useractivity')
bulk_import_model(data, UserActivity, 'zerver_useractivity')
fix_datetime_fields(data, 'zerver_useractivityinterval')
re_map_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', related_table="user_profile")
update_model_ids(UserActivityInterval, data, 'zerver_useractivityinterval',
'useractivityinterval')
bulk_import_model(data, UserActivityInterval, 'zerver_useractivityinterval')
if 'zerver_customprofilefield' in data:
# As the export of Custom Profile fields is not supported, Zulip exported
# data would not contain this field.
# However this is supported in slack importer script
re_map_foreign_keys(data, 'zerver_customprofilefield', 'realm', related_table="realm")
update_model_ids(CustomProfileField, data, 'zerver_customprofilefield',
related_table="customprofilefield")
bulk_import_model(data, CustomProfileField, 'zerver_customprofilefield')
re_map_foreign_keys(data, 'zerver_customprofilefield_value', 'user_profile',
related_table="user_profile")
re_map_foreign_keys(data, 'zerver_customprofilefield_value', 'field',
related_table="customprofilefield")
update_model_ids(CustomProfileFieldValue, data, 'zerver_customprofilefield_value',
related_table="customprofilefield_value")
bulk_import_model(data, CustomProfileFieldValue, 'zerver_customprofilefield_value')
# Import uploaded files and avatars
import_uploads(os.path.join(import_dir, "avatars"), processing_avatars=True)
import_uploads(os.path.join(import_dir, "uploads"))
# We need to have this check as the emoji files are only present in the data
# importer from slack
# For Zulip export, this doesn't exist
if os.path.exists(os.path.join(import_dir, "emoji")):
import_uploads(os.path.join(import_dir, "emoji"), processing_emojis=True)
# Import zerver_message and zerver_usermessage
import_message_data(import_dir)
# Do attachments AFTER message data is loaded.
# TODO: de-dup how we read these json files.
fn = os.path.join(import_dir, "attachment.json")
if not os.path.exists(fn):
raise Exception("Missing attachment.json file!")
logging.info("Importing attachment data from %s" % (fn,))
with open(fn) as f:
data = ujson.load(f)
import_attachments(data)
return realm
def add_subscriptions_backend(
request: HttpRequest, user_profile: UserProfile,
streams_raw: Iterable[Mapping[str, Text]]=REQ(
"subscriptions", validator=check_list(check_dict([('name', check_string)]))),
invite_only: bool=REQ(validator=check_bool, default=False),
announce: bool=REQ(validator=check_bool, default=False),
principals: List[Text]=REQ(validator=check_list(check_string), default=[]),
authorization_errors_fatal: bool=REQ(validator=check_bool, default=True),
) -> HttpResponse:
stream_dicts = []
for stream_dict in streams_raw:
stream_dict_copy = {} # type: Dict[str, Any]
for field in stream_dict:
stream_dict_copy[field] = stream_dict[field]
# Strip the stream name here.
stream_dict_copy['name'] = stream_dict_copy['name'].strip()
stream_dict_copy["invite_only"] = invite_only
stream_dicts.append(stream_dict_copy)
# Validation of the streams arguments, including enforcement of
# can_create_streams policy and check_stream_name policy is inside
# list_to_streams.
existing_streams, created_streams = \
list_to_streams(stream_dicts, user_profile, autocreate=True)
authorized_streams, unauthorized_streams = \
filter_stream_authorization(user_profile, existing_streams)
if len(unauthorized_streams) > 0 and authorization_errors_fatal:
return json_error(_("Unable to access stream (%s).") % unauthorized_streams[0].name)
# Newly created streams are also authorized for the creator
streams = authorized_streams + created_streams
if len(principals) > 0:
if user_profile.realm.is_zephyr_mirror_realm and not all(stream.invite_only for stream in streams):
return json_error(_("You can only invite other Zephyr mirroring users to invite-only streams."))
subscribers = set(principal_to_user_profile(user_profile, principal) for principal in principals)
else:
subscribers = set([user_profile])
(subscribed, already_subscribed) = bulk_add_subscriptions(streams, subscribers,
acting_user=user_profile)
# We can assume unique emails here for now, but we should eventually
# convert this function to be more id-centric.
email_to_user_profile = dict() # type: Dict[Text, UserProfile]
result = dict(subscribed=defaultdict(list), already_subscribed=defaultdict(list)) # type: Dict[str, Any]
for (subscriber, stream) in subscribed:
result["subscribed"][subscriber.email].append(stream.name)
email_to_user_profile[subscriber.email] = subscriber
for (subscriber, stream) in already_subscribed:
result["already_subscribed"][subscriber.email].append(stream.name)
bots = dict((subscriber.email, subscriber.is_bot) for subscriber in subscribers)
newly_created_stream_names = {s.name for s in created_streams}
private_stream_names = {s.name for s in streams if s.invite_only}
# Inform the user if someone else subscribed them to stuff,
# or if a new stream was created with the "announce" option.
notifications = []
if len(principals) > 0 and result["subscribed"]:
for email, subscribed_stream_names in result["subscribed"].items():
if email == user_profile.email:
# Don't send a Zulip if you invited yourself.
continue
if bots[email]:
# Don't send invitation Zulips to bots
continue
# For each user, we notify them about newly subscribed streams, except for
# streams that were newly created.
notify_stream_names = set(subscribed_stream_names) - newly_created_stream_names
if not notify_stream_names:
continue
msg = you_were_just_subscribed_message(
acting_user=user_profile,
stream_names=notify_stream_names,
private_stream_names=private_stream_names
)
sender = get_system_bot(settings.NOTIFICATION_BOT)
notifications.append(
internal_prep_private_message(
realm=user_profile.realm,
sender=sender,
recipient_user=email_to_user_profile[email],
content=msg))
if announce and len(created_streams) > 0 and settings.NOTIFICATION_BOT is not None:
notifications_stream = user_profile.realm.get_notifications_stream()
if notifications_stream is not None:
if len(created_streams) > 1:
stream_strs = ", ".join('#**%s**' % s.name for s in created_streams)
stream_msg = "the following streams: %s" % (stream_strs,)
else:
stream_msg = "a new stream #**%s**." % created_streams[0].name
msg = ("%s just created %s" % (user_profile.full_name, stream_msg))
sender = get_system_bot(settings.NOTIFICATION_BOT)
stream_name = notifications_stream.name
topic = 'Streams'
notifications.append(
internal_prep_stream_message(
realm=user_profile.realm,
sender=sender,
stream_name=stream_name,
topic=topic,
content=msg))
if not user_profile.realm.is_zephyr_mirror_realm:
for stream in created_streams:
notifications.append(prep_stream_welcome_message(stream))
if len(notifications) > 0:
do_send_messages(notifications)
result["subscribed"] = dict(result["subscribed"])
result["already_subscribed"] = dict(result["already_subscribed"])
if not authorization_errors_fatal:
result["unauthorized"] = [s.name for s in unauthorized_streams]
return json_success(result)
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write("Error! More than 100% of messages allocated.\n")
return
# Get consistent data for backend tests.
if options["test_suite"]:
random.seed(0)
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our two default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
zulip_realm = Realm.objects.create(
string_id="zulip", name="Zulip Dev", emails_restricted_to_domains=True,
description="The Zulip development environment default organization."
" It's great for testing!",
invite_required=False, org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
if options["test_suite"]:
mit_realm = Realm.objects.create(
string_id="zephyr", name="MIT", emails_restricted_to_domains=True,
invite_required=False, org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = Realm.objects.create(
string_id="lear", name="Lear & Co.", emails_restricted_to_domains=False,
invite_required=False, org_type=Realm.CORPORATE)
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
("Polonius", "*****@*****.**"),
]
for i in range(options["extra_users"]):
names.append(('Extra User %d' % (i,), '*****@*****.**' % (i,)))
create_users(zulip_realm, names)
iago = get_user("*****@*****.**", zulip_realm)
do_change_is_admin(iago, True)
iago.is_staff = True
iago.save(update_fields=['is_staff'])
guest_user = get_user("*****@*****.**", zulip_realm)
guest_user.is_guest = True
guest_user.save(update_fields=['is_guest'])
# These bots are directly referenced from code and thus
# are needed for the test suite.
all_realm_bots = [(bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,))
for bot in settings.INTERNAL_BOTS]
zulip_realm_bots = [
("Zulip New User Bot", "*****@*****.**"),
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
("Welcome Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(('Extra Bot %d' % (i,), '*****@*****.**' % (i,)))
zulip_realm_bots.extend(all_realm_bots)
create_users(zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT)
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
zoe = get_user("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(zulip_realm, zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe)
aaron = get_user("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [
("Outgoing Webhook", "*****@*****.**")
]
create_users(zulip_realm, zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, bot_owner=aaron)
outgoing_webhook = get_user("*****@*****.**", zulip_realm)
add_service("outgoing-webhook", user_profile=outgoing_webhook, interface=Service.GENERIC,
base_url="http://127.0.0.1:5002", token=generate_api_key())
# Add the realm internl bots to each realm.
create_if_missing_realm_internal_bots()
# Create public streams.
stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
stream_dict = {
"Verona": {"description": "A city in Italy"},
"Denmark": {"description": "A Scandinavian country"},
"Scotland": {"description": "Located in the United Kingdom"},
"Venice": {"description": "A northeastern Italian city"},
"Rome": {"description": "Yet another Italian city", "is_web_public": True}
} # type: Dict[str, Dict[str, Any]]
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams = [Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list] # type: List[int]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users). For the test suite, we have a fixed list of
# subscriptions to make sure test data is consistent
# across platforms.
subscriptions_list = [] # type: List[Tuple[UserProfile, Recipient]]
profiles = UserProfile.objects.select_related().filter(
is_bot=False).order_by("email") # type: Sequence[UserProfile]
if options["test_suite"]:
subscriptions_map = {
'*****@*****.**': ['Verona'],
'*****@*****.**': ['Verona'],
'*****@*****.**': ['Verona', 'Denmark'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice', 'Rome'],
'*****@*****.**': ['Verona'],
}
for profile in profiles:
if profile.email not in subscriptions_map:
raise Exception('Subscriptions not listed for user %s' % (profile.email,))
for stream_name in subscriptions_map[profile.email]:
stream = Stream.objects.get(name=stream_name)
r = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id)
subscriptions_list.append((profile, r))
else:
for i, profile in enumerate(profiles):
# Subscribe to some streams.
for type_id in recipient_streams[:int(len(recipient_streams) *
float(i)/len(profiles)) + 1]:
r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id)
subscriptions_list.append((profile, r))
subscriptions_to_add = [] # type: List[Subscription]
event_time = timezone_now()
all_subscription_logs = [] # type: (List[RealmAuditLog])
i = 0
for profile, recipient in subscriptions_list:
i += 1
color = STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)]
s = Subscription(
recipient=recipient,
user_profile=profile,
color=color)
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream_id=recipient.type_id,
event_last_message_id=0,
event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(zulip_realm, "Phone number",
CustomProfileField.SHORT_TEXT,
hint='')
biography = try_add_realm_custom_profile_field(zulip_realm, "Biography",
CustomProfileField.LONG_TEXT,
hint='What are you known for?')
favorite_food = try_add_realm_custom_profile_field(zulip_realm, "Favorite food",
CustomProfileField.SHORT_TEXT,
hint="Or drink, if you'd prefer")
field_data = {
'vim': {'text': 'Vim', 'order': '1'},
'emacs': {'text': 'Emacs', 'order': '2'},
}
favorite_editor = try_add_realm_custom_profile_field(zulip_realm,
"Favorite editor",
CustomProfileField.CHOICE,
field_data=field_data)
birthday = try_add_realm_custom_profile_field(zulip_realm, "Birthday",
CustomProfileField.DATE)
favorite_website = try_add_realm_custom_profile_field(zulip_realm, "GitHub profile",
CustomProfileField.URL,
hint="Or your personal blog's URL")
mentor = try_add_realm_custom_profile_field(zulip_realm, "Mentor",
CustomProfileField.USER)
# Fill in values for Iago and Hamlet
hamlet = get_user("*****@*****.**", zulip_realm)
do_update_user_custom_profile_data(iago, [
{"id": phone_number.id, "value": "+1-234-567-8901"},
{"id": biography.id, "value": "Betrayer of Othello."},
{"id": favorite_food.id, "value": "Apples"},
{"id": favorite_editor.id, "value": "emacs"},
{"id": birthday.id, "value": "2000-1-1"},
{"id": favorite_website.id, "value": "https://github.com/zulip/zulip"},
{"id": mentor.id, "value": [hamlet.id]},
])
do_update_user_custom_profile_data(hamlet, [
{"id": phone_number.id, "value": "+0-11-23-456-7890"},
{"id": biography.id, "value": "Prince of Denmark, and other things!"},
{"id": favorite_food.id, "value": "Dark chocolate"},
{"id": favorite_editor.id, "value": "vim"},
{"id": birthday.id, "value": "1900-1-1"},
{"id": favorite_website.id, "value": "https://blog.zulig.org"},
{"id": mentor.id, "value": [iago.id]},
])
else:
zulip_realm = get_realm("zulip")
recipient_streams = [klass.type_id for klass in
Recipient.objects.filter(type=Recipient.STREAM)]
# Extract a list of all users
user_profiles = list(UserProfile.objects.filter(is_bot=False)) # type: List[UserProfile]
# Create a test realm emoji.
IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images', 'test-images', 'checkbox.png')
with open(IMAGE_FILE_PATH, 'rb') as fp:
check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status = UserPresence.ACTIVE # type: int
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= 'H':
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(user_profile=user,
client=client,
timestamp=date,
status=status)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])]
# Generate a new set of test data.
create_test_data()
# prepopulate the URL preview/embed data for the links present
# in the config.generate_data.json data set. This makes it
# possible for populate_db to run happily without Internet
# access.
with open("zerver/tests/fixtures/docs_url_preview_data.json", "r") as f:
urls_with_preview_data = ujson.load(f)
for url in urls_with_preview_data:
cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)
threads = options["threads"]
jobs = [] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write, random.randint(0, 10**10)))
for job in jobs:
send_messages(job)
if options["delete"]:
# Create the "website" and "API" clients; if we don't, the
# default values in zerver/decorators.py will not work
# with the Django test suite.
get_client("website")
get_client("API")
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)", "*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm, testsuite_mit_users)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
]
create_users(lear_realm, testsuite_lear_users)
if not options["test_suite"]:
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
zulip_stream_dict = {
"devel": {"description": "For developing"},
"all": {"description": "For everything"},
"announce": {"description": "For announcements", 'is_announcement_only': True},
"design": {"description": "For design"},
"support": {"description": "For support"},
"social": {"description": "For socializing"},
"test": {"description": "For testing"},
"errors": {"description": "For errors"},
"sales": {"description": "For sales discussion"}
} # type: Dict[str, Dict[str, Any]]
# Calculate the maximum number of digits in any extra stream's
# number, since a stream with name "Extra Stream 3" could show
# up after "Extra Stream 29". (Used later to pad numbers with
# 0s).
maximum_digits = len(str(options['extra_streams'] - 1))
for i in range(options['extra_streams']):
# Pad the number with 0s based on `maximum_digits`.
number_str = str(i).zfill(maximum_digits)
extra_stream_name = 'Extra Stream ' + number_str
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream("announce", zulip_realm)
zulip_realm.save(update_fields=['notifications_stream'])
# Add a few default streams
for default_stream_name in ["design", "devel", "social", "support"]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(default_stream_name, zulip_realm))
# Now subscribe everyone to these streams
subscribe_users_to_streams(zulip_realm, zulip_stream_dict)
# These bots are not needed by the test suite
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm, internal_zulip_users_nosubs, bot_type=UserProfile.DEFAULT_BOT)
zulip_cross_realm_bots = [
("Zulip Feedback Bot", "*****@*****.**"),
]
create_users(zulip_realm, zulip_cross_realm_bots, bot_type=UserProfile.DEFAULT_BOT)
# Mark all messages as read
UserMessage.objects.all().update(flags=UserMessage.flags.read)
if not options["test_suite"]:
# Update pointer of each user to point to the last message in their
# UserMessage rows with sender_id=user_profile_id.
users = list(UserMessage.objects.filter(
message__sender_id=F('user_profile_id')).values(
'user_profile_id').annotate(pointer=Max('message_id')))
for user in users:
UserProfile.objects.filter(id=user['user_profile_id']).update(
pointer=user['pointer'])
create_user_groups()
if not options["test_suite"]:
# We populate the analytics database here for
# development purpose only
call_command('populate_analytics_db')
self.stdout.write("Successfully populated test database.\n")
def export_files_from_s3(realm: Realm, bucket_name: str, output_dir: Path,
processing_avatars: bool=False) -> None:
conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
bucket = conn.get_bucket(bucket_name, validate=True)
records = []
logging.info("Downloading uploaded files from %s" % (bucket_name))
avatar_hash_values = set()
user_ids = set()
if processing_avatars:
bucket_list = bucket.list()
for user_profile in UserProfile.objects.filter(realm=realm):
avatar_path = user_avatar_path_from_ids(user_profile.id, realm.id)
avatar_hash_values.add(avatar_path)
avatar_hash_values.add(avatar_path + ".original")
user_ids.add(user_profile.id)
else:
bucket_list = bucket.list(prefix="%s/" % (realm.id,))
if settings.EMAIL_GATEWAY_BOT is not None:
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT) # type: Optional[UserProfile]
else:
email_gateway_bot = None
count = 0
for bkey in bucket_list:
if processing_avatars and bkey.name not in avatar_hash_values:
continue
key = bucket.get_key(bkey.name)
# This can happen if an email address has moved realms
if 'realm_id' in key.metadata and key.metadata['realm_id'] != str(realm.id):
if email_gateway_bot is None or key.metadata['user_profile_id'] != str(email_gateway_bot.id):
raise AssertionError("Key metadata problem: %s %s / %s" % (key.name, key.metadata, realm.id))
# Email gateway bot sends messages, potentially including attachments, cross-realm.
print("File uploaded by email gateway bot: %s / %s" % (key.name, key.metadata))
elif processing_avatars:
if 'user_profile_id' not in key.metadata:
raise AssertionError("Missing user_profile_id in key metadata: %s" % (key.metadata,))
if int(key.metadata['user_profile_id']) not in user_ids:
raise AssertionError("Wrong user_profile_id in key metadata: %s" % (key.metadata,))
elif 'realm_id' not in key.metadata:
raise AssertionError("Missing realm_id in key metadata: %s" % (key.metadata,))
record = dict(s3_path=key.name, bucket=bucket_name,
size=key.size, last_modified=key.last_modified,
content_type=key.content_type, md5=key.md5)
record.update(key.metadata)
# A few early avatars don't have 'realm_id' on the object; fix their metadata
user_profile = get_user_profile_by_id(record['user_profile_id'])
if 'realm_id' not in record:
record['realm_id'] = user_profile.realm_id
record['user_profile_email'] = user_profile.email
if processing_avatars:
dirname = output_dir
filename = os.path.join(dirname, key.name)
record['path'] = key.name
else:
fields = key.name.split('/')
if len(fields) != 3:
raise AssertionError("Suspicious key with invalid format %s" % (key.name))
dirname = os.path.join(output_dir, fields[1])
filename = os.path.join(dirname, fields[2])
record['path'] = os.path.join(fields[1], fields[2])
if not os.path.exists(dirname):
os.makedirs(dirname)
key.get_contents_to_filename(filename)
records.append(record)
count += 1
if (count % 100 == 0):
logging.info("Finished %s" % (count,))
with open(os.path.join(output_dir, "records.json"), "w") as records_file:
ujson.dump(records, records_file, indent=4)
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write("Error! More than 100% of messages allocated.\n")
return
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our two default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
zulip_realm = Realm.objects.create(
string_id="zulip", name="Zulip Dev", restricted_to_domain=True,
description="The Zulip development environment default organization."
" It's great for testing!",
invite_required=False, org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
if options["test_suite"]:
mit_realm = Realm.objects.create(
string_id="zephyr", name="MIT", restricted_to_domain=True,
invite_required=False, org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = Realm.objects.create(
string_id="lear", name="Lear & Co.", restricted_to_domain=False,
invite_required=False, org_type=Realm.CORPORATE)
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
]
for i in range(options["extra_users"]):
names.append(('Extra User %d' % (i,), '*****@*****.**' % (i,)))
create_users(zulip_realm, names)
iago = get_user("*****@*****.**", zulip_realm)
do_change_is_admin(iago, True)
iago.is_staff = True
iago.save(update_fields=['is_staff'])
# These bots are directly referenced from code and thus
# are needed for the test suite.
all_realm_bots = [(bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,))
for bot in settings.INTERNAL_BOTS]
zulip_realm_bots = [
("Zulip New User Bot", "*****@*****.**"),
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
("Welcome Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(('Extra Bot %d' % (i,), '*****@*****.**' % (i,)))
zulip_realm_bots.extend(all_realm_bots)
create_users(zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT)
zoe = get_user("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(zulip_realm, zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe)
aaron = get_user("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [
("Outgoing Webhook", "*****@*****.**")
]
create_users(zulip_realm, zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, bot_owner=aaron)
# TODO: Clean up this initial bot creation code
Service.objects.create(
name="test",
user_profile=get_user("*****@*****.**", zulip_realm),
base_url="http://127.0.0.1:5002/bots/followup",
token="abcd1234",
interface=1)
# Create public streams.
stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
stream_dict = {
"Verona": {"description": "A city in Italy", "invite_only": False},
"Denmark": {"description": "A Scandinavian country", "invite_only": False},
"Scotland": {"description": "Located in the United Kingdom", "invite_only": False},
"Venice": {"description": "A northeastern Italian city", "invite_only": False},
"Rome": {"description": "Yet another Italian city", "invite_only": False}
} # type: Dict[Text, Dict[Text, Any]]
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams = [Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list] # type: List[int]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users).
subscriptions_to_add = [] # type: List[Subscription]
event_time = timezone_now()
all_subscription_logs = [] # type: (List[RealmAuditLog])
profiles = UserProfile.objects.select_related().filter(
is_bot=False).order_by("email") # type: Sequence[UserProfile]
for i, profile in enumerate(profiles):
# Subscribe to some streams.
for type_id in recipient_streams[:int(len(recipient_streams) *
float(i)/len(profiles)) + 1]:
r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id)
s = Subscription(
recipient=r,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream_id=type_id,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
if not options['test_suite']:
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(zulip_realm, "Phone number",
CustomProfileField.SHORT_TEXT)
biography = try_add_realm_custom_profile_field(zulip_realm, "Biography",
CustomProfileField.LONG_TEXT)
favorite_integer = try_add_realm_custom_profile_field(zulip_realm, "Favorite integer",
CustomProfileField.INTEGER)
# Fill in values for Iago and Hamlet
hamlet = get_user("*****@*****.**", zulip_realm)
do_update_user_custom_profile_data(iago, [
{"id": phone_number.id, "value": "+1-234-567-8901"},
{"id": biography.id, "value": "Betrayer of Othello."},
{"id": favorite_integer.id, "value": 17},
])
do_update_user_custom_profile_data(hamlet, [
{"id": phone_number.id, "value": "+0-11-23-456-7890"},
{"id": biography.id, "value": "Prince of Denmark, and other things!"},
{"id": favorite_integer.id, "value": 12},
])
else:
zulip_realm = get_realm("zulip")
recipient_streams = [klass.type_id for klass in
Recipient.objects.filter(type=Recipient.STREAM)]
# Extract a list of all users
user_profiles = list(UserProfile.objects.filter(is_bot=False)) # type: List[UserProfile]
# Create a test realm emoji.
IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images', 'test-images', 'checkbox.png')
UPLOADED_EMOJI_FILE_NAME = 'green_tick.png'
with open(IMAGE_FILE_PATH, 'rb') as fp:
upload_backend.upload_emoji_image(fp, UPLOADED_EMOJI_FILE_NAME, iago)
RealmEmoji.objects.create(
name='green_tick',
author=iago,
realm=zulip_realm,
deactivated=False,
file_name=UPLOADED_EMOJI_FILE_NAME,
)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status = UserPresence.ACTIVE # type: int
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= 'H':
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(user_profile=user,
client=client,
timestamp=date,
status=status)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])]
# Generate a new set of test data.
create_test_data()
threads = options["threads"]
jobs = [] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write, random.randint(0, 10**10)))
for job in jobs:
send_messages(job)
if options["delete"]:
# Create the "website" and "API" clients; if we don't, the
# default values in zerver/decorators.py will not work
# with the Django test suite.
get_client("website")
get_client("API")
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)", "*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm, testsuite_mit_users)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
]
create_users(lear_realm, testsuite_lear_users)
if not options["test_suite"]:
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
zulip_stream_dict = {
"devel": {"description": "For developing", "invite_only": False},
"all": {"description": "For everything", "invite_only": False},
"announce": {"description": "For announcements", "invite_only": False},
"design": {"description": "For design", "invite_only": False},
"support": {"description": "For support", "invite_only": False},
"social": {"description": "For socializing", "invite_only": False},
"test": {"description": "For testing", "invite_only": False},
"errors": {"description": "For errors", "invite_only": False},
"sales": {"description": "For sales discussion", "invite_only": False}
} # type: Dict[Text, Dict[Text, Any]]
# Calculate the maximum number of digits in any extra stream's
# number, since a stream with name "Extra Stream 3" could show
# up after "Extra Stream 29". (Used later to pad numbers with
# 0s).
maximum_digits = len(str(options['extra_streams'] - 1))
for i in range(options['extra_streams']):
# Pad the number with 0s based on `maximum_digits`.
number_str = str(i).zfill(maximum_digits)
extra_stream_name = 'Extra Stream ' + number_str
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
"invite_only": False,
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream("announce", zulip_realm)
zulip_realm.save(update_fields=['notifications_stream'])
# Add a few default streams
for default_stream_name in ["design", "devel", "social", "support"]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(default_stream_name, zulip_realm))
# Now subscribe everyone to these streams
subscriptions_to_add = []
event_time = timezone_now()
all_subscription_logs = []
profiles = UserProfile.objects.select_related().filter(realm=zulip_realm)
for i, stream_name in enumerate(zulip_stream_dict):
stream = Stream.objects.get(name=stream_name, realm=zulip_realm)
recipient = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id)
for profile in profiles:
# Subscribe to some streams.
s = Subscription(
recipient=recipient,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream=stream,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# These bots are not needed by the test suite
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm, internal_zulip_users_nosubs, bot_type=UserProfile.DEFAULT_BOT)
zulip_cross_realm_bots = [
("Zulip Feedback Bot", "*****@*****.**"),
]
create_users(zulip_realm, zulip_cross_realm_bots, bot_type=UserProfile.DEFAULT_BOT)
# Mark all messages as read
UserMessage.objects.all().update(flags=UserMessage.flags.read)
if not options["test_suite"]:
# Update pointer of each user to point to the last message in their
# UserMessage rows with sender_id=user_profile_id.
users = list(UserMessage.objects.filter(
message__sender_id=F('user_profile_id')).values(
'user_profile_id').annotate(pointer=Max('message_id')))
for user in users:
UserProfile.objects.filter(id=user['user_profile_id']).update(
pointer=user['pointer'])
create_user_groups()
self.stdout.write("Successfully populated test database.\n")
