def test_UserActivityWorker(self) -> None:
fake_client = self.FakeClient()
user = self.example_user('hamlet')
UserActivity.objects.filter(
user_profile = user.id,
client = get_client('ios')
).delete()
data = dict(
user_profile_id = user.id,
client = 'ios',
time = time.time(),
query = 'send_message'
)
fake_client.queue.append(('user_activity', data))
with simulated_queue_client(lambda: fake_client):
worker = queue_processors.UserActivityWorker()
worker.setup()
worker.start()
activity_records = UserActivity.objects.filter(
user_profile = user.id,
client = get_client('ios')
)
self.assertTrue(len(activity_records), 1)
self.assertTrue(activity_records[0].count, 1)
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
Realm.objects.create(string_id=settings.INTERNAL_BOT_DOMAIN.split('.')[0],
domain=settings.INTERNAL_BOT_DOMAIN)
names = [(settings.FEEDBACK_BOT_NAME, settings.FEEDBACK_BOT)]
create_users(names, bot_type=UserProfile.DEFAULT_BOT)
get_client("website")
get_client("API")
internal_bots = [(bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,))
for bot in settings.INTERNAL_BOTS]
create_users(internal_bots, bot_type=UserProfile.DEFAULT_BOT)
# Set the owners for these bots to the bots themselves
bots = UserProfile.objects.filter(email__in=[bot_info[1] for bot_info in internal_bots])
for bot in bots:
bot.bot_owner = bot
bot.save()
# Initialize the email gateway bot as an API Super User
email_gateway_bot = UserProfile.objects.get(email__iexact=settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
self.stdout.write("Successfully populated database with initial data.\n")
self.stdout.write("Please run ./manage.py generate_realm_creation_link to generate link for creating organization")
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
Realm.objects.create(domain="zulip.com")
names = [(settings.FEEDBACK_BOT_NAME, settings.FEEDBACK_BOT)]
create_users(names, bot_type=UserProfile.DEFAULT_BOT)
get_client("website")
get_client("API")
internal_bots = [
(bot["name"], bot["email_template"] % (settings.INTERNAL_BOT_DOMAIN,)) for bot in settings.INTERNAL_BOTS
]
create_users(internal_bots, bot_type=UserProfile.DEFAULT_BOT)
# Set the owners for these bots to the bots themselves
bots = UserProfile.objects.filter(email__in=[bot_info[1] for bot_info in internal_bots])
for bot in bots:
bot.bot_owner = bot
bot.save()
# Initialize the email gateway bot as an API Super User
email_gateway_bot = UserProfile.objects.get(email__iexact=settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
(admin_realm, _) = do_create_realm(settings.ADMIN_DOMAIN, settings.ADMIN_DOMAIN, True)
set_default_streams(admin_realm, settings.DEFAULT_NEW_REALM_STREAMS)
self.stdout.write("Successfully populated database with initial data.\n")
def api_teamcity_webhook(request, user_profile, payload=REQ(argument_type='body'),
stream=REQ(default='teamcity')):
message = payload['build']
build_name = message['buildFullName']
build_url = message['buildStatusUrl']
changes_url = build_url + '&tab=buildChangesDiv'
build_number = message['buildNumber']
build_result = message['buildResult']
build_result_delta = message['buildResultDelta']
build_status = message['buildStatus']
if build_result == 'success':
if build_result_delta == 'fixed':
status = 'has been fixed! :thumbsup:'
else:
status = 'was successful! :thumbsup:'
elif build_result == 'failure':
if build_result_delta == 'broken':
status = 'is broken with status %s! :thumbsdown:' % (build_status)
else:
status = 'is still broken with status %s! :thumbsdown:' % (build_status)
elif build_result == 'running':
status = 'has started.'
else:
status = '(has no message specified for status %s)' % (build_status)
template = (
u'%s build %s %s\n'
u'Details: [changes](%s), [build log](%s)')
body = template % (build_name, build_number, status, changes_url, build_url)
topic = build_name
# Check if this is a personal build, and if so try to private message the user who triggered it.
if get_teamcity_property_value(message['teamcityProperties'], 'env.BUILD_IS_PERSONAL') == 'true':
# The triggeredBy field gives us the teamcity user full name, and the "teamcity.build.triggeredBy.username"
# property gives us the teamcity username. Let's try finding the user email from both.
teamcity_fullname = message['triggeredBy'].split(';')[0]
teamcity_user = guess_zulip_user_from_teamcity(teamcity_fullname, user_profile.realm)
if teamcity_user is None:
teamcity_shortname = get_teamcity_property_value(message['teamcityProperties'], 'teamcity.build.triggeredBy.username')
if teamcity_shortname is not None:
teamcity_user = guess_zulip_user_from_teamcity(teamcity_shortname, user_profile.realm)
if teamcity_user is None:
# We can't figure out who started this build - there's nothing we can do here.
logging.info("Teamcity webhook couldn't find a matching Zulip user for Teamcity user '%s' or '%s'" % (teamcity_fullname, teamcity_shortname))
return json_success()
body = "Your personal build of " + body
check_send_message(user_profile, get_client('ZulipTeamcityWebhook'), 'private', [teamcity_user.email], topic, body)
return json_success()
check_send_message(user_profile, get_client('ZulipTeamcityWebhook'), 'stream', [stream], topic, body)
return json_success()
def test_alert_words(self):
user_profile = get_user_profile_by_email("*****@*****.**")
do_set_alert_words(user_profile, ["ALERTWORD", "scaryword"])
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "We have an ALERTWORD day today!"
self.assertEqual(msg.render_markdown(content), "<p>We have an ALERTWORD day today!</p>")
self.assertEqual(msg.user_ids_with_alert_words, set([user_profile.id]))
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "We have a NOTHINGWORD day today!"
self.assertEqual(msg.render_markdown(content), "<p>We have a NOTHINGWORD day today!</p>")
self.assertEqual(msg.user_ids_with_alert_words, set())
def send_response_message(bot_id: str, message: Dict[str, Any], response_message_content: Text) -> None:
recipient_type_name = message['type']
bot_user = get_user_profile_by_id(bot_id)
realm = bot_user.realm
if recipient_type_name == 'stream':
recipients = [message['display_recipient']]
check_send_message(bot_user, get_client("OutgoingWebhookResponse"), recipient_type_name, recipients,
message['subject'], response_message_content, realm)
elif recipient_type_name == 'private':
recipients = [recipient['email'] for recipient in message['display_recipient']]
check_send_message(bot_user, get_client("OutgoingWebhookResponse"), recipient_type_name, recipients,
None, response_message_content, realm)
else:
raise JsonableError(_("Invalid message type"))
def consume(self, event):
logging.info("Received event: %s" % (event),)
user_profile = get_user_profile_by_id(event["user_profile_id"])
client = get_client(event["client"])
log_time = timestamp_to_datetime(event["time"])
status = event["status"]
do_update_user_presence(user_profile, client, log_time, status)
def api_yo_app_webhook(request, user_profile, email=REQ(default=None),
username=REQ(default='Yo Bot'), topic=REQ(default='None'),
user_ip=REQ(default='None')):
body = ('Yo from %s') % (username,)
check_send_message(user_profile, get_client('ZulipYoWebhook'), 'private', [email], topic, body)
return json_success()
def api_freshdesk_webhook(request: HttpRequest, user_profile: UserProfile,
payload: Dict[str, Any]=REQ(argument_type='body'),
stream: Text=REQ(default='freshdesk')) -> HttpResponse:
ticket_data = payload["freshdesk_webhook"]
required_keys = [
"triggered_event", "ticket_id", "ticket_url", "ticket_type",
"ticket_subject", "ticket_description", "ticket_status",
"ticket_priority", "requester_name", "requester_email",
]
for key in required_keys:
if ticket_data.get(key) is None:
logging.warning("Freshdesk webhook error. Payload was:")
logging.warning(request.body)
return json_error(_("Missing key %s in JSON") % (key,))
ticket = TicketDict(ticket_data)
subject = "#%s: %s" % (ticket.id, ticket.subject)
event_info = parse_freshdesk_event(ticket.triggered_event)
if event_info[1] == "created":
content = format_freshdesk_ticket_creation_message(ticket)
elif event_info[0] == "note_type":
content = format_freshdesk_note_message(ticket, event_info)
elif event_info[0] in ("status", "priority"):
content = format_freshdesk_property_change_message(ticket, event_info)
else:
# Not an event we know handle; do nothing.
return json_success()
check_send_stream_message(user_profile, get_client("ZulipFreshdeskWebhook"),
stream, subject, content)
return json_success()
def api_wordpress_webhook(request, user_profile,
stream=REQ(default="wordpress"),
topic=REQ(default="WordPress Notification"),
hook=REQ(default="WordPress Action"),
post_title=REQ(default="New WordPress Post"),
post_type=REQ(default="post"),
post_url=REQ(default="WordPress Post URL"),
display_name=REQ(default="New User Name"),
user_email=REQ(default="New User Email"),
user_login=REQ(default="Logged in User")):
# type: (HttpRequest, UserProfile, text_type, text_type, text_type, text_type, text_type, text_type, text_type, text_type, text_type) -> HttpResponse
# remove trailing whitespace (issue for some test fixtures)
hook = hook.rstrip()
if hook == 'publish_post' or hook == 'publish_page':
data = PUBLISH_POST_OR_PAGE_TEMPLATE.format(type=post_type, title=post_title, url=post_url)
elif hook == 'user_register':
data = USER_REGISTER_TEMPLATE.format(name=display_name, email=user_email)
elif hook == 'wp_login':
data = WP_LOGIN_TEMPLATE.format(name=user_login)
else:
return json_error(_("Unknown WordPress webhook action: " + hook))
check_send_message(user_profile, get_client("ZulipWordPressWebhook"), "stream",
[stream], topic, data)
return json_success()
def api_bitbucket_webhook(request, user_profile, payload=REQ(validator=check_dict([])),
stream=REQ(default='commits')):
# type: (HttpRequest, UserProfile, Mapping[Text, Any], Text) -> HttpResponse
repository = payload['repository']
commits = [
{
'sha': commit.get('raw_node'),
'message': commit.get('message'),
'url': u'{}{}commits/{}'.format(
payload.get('canon_url'),
repository.get('absolute_url'),
commit.get('raw_node'))
}
for commit in payload.get('commits')
]
if len(commits) == 0:
# Bitbucket doesn't give us enough information to really give
# a useful message :/
subject = repository['name']
content = (u"%s [force pushed](%s)"
% (payload['user'],
payload['canon_url'] + repository['absolute_url']))
else:
branch = payload['commits'][-1]['branch']
content = get_push_commits_event_message(payload.get('user'), None, branch, commits)
subject = SUBJECT_WITH_BRANCH_TEMPLATE.format(repo=repository['name'], branch=branch)
check_send_message(user_profile, get_client("ZulipBitBucketWebhook"), "stream",
[stream], subject, content)
return json_success()
def api_beanstalk_webhook(request, user_profile,
payload=REQ(validator=check_dict([]))):
# Beanstalk supports both SVN and git repositories
# We distinguish between the two by checking for a
# 'uri' key that is only present for git repos
git_repo = 'uri' in payload
if git_repo:
# To get a linkable url,
subject, content = build_message_from_gitlog(user_profile, payload['repository']['name'],
payload['ref'], payload['commits'],
payload['before'], payload['after'],
payload['repository']['url'],
payload['pusher_name'])
else:
author = payload.get('author_full_name')
url = payload.get('changeset_url')
revision = payload.get('revision')
(short_commit_msg, _, _) = payload.get('message').partition("\n")
subject = "svn r%s" % (revision,)
content = "%s pushed [revision %s](%s):\n\n> %s" % (author, revision, url, short_commit_msg)
check_send_message(user_profile, get_client("ZulipBeanstalkWebhook"), "stream",
["commits"], subject, content)
return json_success()
def api_deskdotcom_webhook(request, user_profile, data=REQ(),
topic=REQ(default="Desk.com notification"),
stream=REQ(default="desk.com")):
# type: (HttpRequest, UserProfile, text_type, text_type, text_type) -> HttpResponse
check_send_message(user_profile, get_client("ZulipDeskWebhook"), "stream",
[stream], topic, data)
return json_success()
def api_travis_webhook(request, user_profile, stream=REQ(default='travis'), topic=REQ(default=None)):
message = ujson.loads(request.POST['payload'])
author = message['author_name']
message_type = message['status_message']
changes = message['compare_url']
good_status = ['Passed', 'Fixed']
bad_status = ['Failed', 'Broken', 'Still Failing']
emoji = ''
if message_type in good_status:
emoji = ':thumbsup:'
elif message_type in bad_status:
emoji = ':thumbsdown:'
else:
emoji = "(No emoji specified for status '%s'.)" % (message_type,)
build_url = message['build_url']
template = (
u'Author: %s\n'
u'Build status: %s %s\n'
u'Details: [changes](%s), [build log](%s)')
body = template % (author, message_type, emoji, changes, build_url)
check_send_message(user_profile, get_client('ZulipTravisWebhook'), 'stream', [stream], topic, body)
return json_success()
def fake_message_sender(event):
# type: (Dict[str, Any]) -> None
"""This function is used only for Casper and backend tests, where
rabbitmq is disabled"""
log_data = dict() # type: Dict[str, Any]
record_request_start_data(log_data)
req = event['request']
try:
sender = get_user_profile_by_id(event['server_meta']['user_id'])
client = get_client("website")
msg_id = check_send_message(sender, client, req['type'],
extract_recipients(req['to']),
req['subject'], req['content'],
local_id=req.get('local_id', None),
sender_queue_id=req.get('queue_id', None))
resp = {"result": "success", "msg": "", "id": msg_id}
except JsonableError as e:
resp = {"result": "error", "msg": str(e)}
server_meta = event['server_meta']
server_meta.update({'worker_log_data': log_data,
'time_request_finished': time.time()})
result = {'response': resp, 'req_id': event['req_id'],
'server_meta': server_meta}
respond_send_message(result)
def send_response_message(bot_id, message, response_message_content):
# type: (str, Dict[str, Any], Text) -> None
recipient_type_name = message['type']
bot_user = get_user_profile_by_id(bot_id)
realm = get_realm_by_email_domain(message['sender_email'])
if recipient_type_name == 'stream':
recipients = [message['display_recipient']]
check_send_message(bot_user, get_client("OutgoingWebhookResponse"), recipient_type_name, recipients,
message['subject'], response_message_content, realm, forwarder_user_profile=bot_user)
else:
# Private message; only send if the bot is there in the recipients
recipients = [recipient['email'] for recipient in message['display_recipient']]
if bot_user.email in recipients:
check_send_message(bot_user, get_client("OutgoingWebhookResponse"), recipient_type_name, recipients,
message['subject'], response_message_content, realm, forwarder_user_profile=bot_user)
def api_circleci_webhook(request, user_profile, payload=REQ(argument_type='body'), stream=REQ(default='circleci')):
payload = payload['payload']
subject = get_subject(payload)
body = get_body(payload)
check_send_message(user_profile, get_client('ZulipCircleCIWebhook'), 'stream', [stream], subject, body)
return json_success()
def api_beanstalk_webhook(request, user_profile,
payload=REQ(validator=check_dict([])),
branches=REQ(default=None)):
# type: (HttpRequest, UserProfile, Dict[str, Any], Optional[Text]) -> HttpResponse
# Beanstalk supports both SVN and git repositories
# We distinguish between the two by checking for a
# 'uri' key that is only present for git repos
git_repo = 'uri' in payload
if git_repo:
if branches is not None and branches.find(payload['branch']) == -1:
return json_success()
# To get a linkable url,
for commit in payload['commits']:
commit['author'] = {'username': commit['author']['name']}
subject, content = build_message_from_gitlog(user_profile, payload['repository']['name'],
payload['ref'], payload['commits'],
payload['before'], payload['after'],
payload['repository']['url'],
payload['pusher_name'])
else:
author = payload.get('author_full_name')
url = payload.get('changeset_url')
revision = payload.get('revision')
(short_commit_msg, _, _) = payload.get('message').partition("\n")
subject = "svn r%s" % (revision,)
content = "%s pushed [revision %s](%s):\n\n> %s" % (author, revision, url, short_commit_msg)
check_send_message(user_profile, get_client("ZulipBeanstalkWebhook"), "stream",
["commits"], subject, content)
return json_success()
def api_bitbucket_webhook(request, user_profile, payload=REQ(validator=check_dict([])),
stream=REQ(default='commits')):
# type: (HttpRequest, UserProfile, Dict[str, Any], str) -> None
repository = payload['repository']
commits = [{'id': commit['raw_node'], 'message': commit['message'],
'url': '%s%scommits/%s' % (payload['canon_url'],
repository['absolute_url'],
commit['raw_node'])}
for commit in payload['commits']]
subject = repository['name']
if len(commits) == 0:
# Bitbucket doesn't give us enough information to really give
# a useful message :/
content = ("%s [force pushed](%s)"
% (payload['user'],
payload['canon_url'] + repository['absolute_url']))
else:
branch = payload['commits'][-1]['branch']
content = build_commit_list_content(commits, branch, None, payload['user'])
subject += '/%s' % (branch,)
check_send_message(user_profile, get_client("ZulipBitBucketWebhook"), "stream",
[stream], subject, content)
return json_success()
def consume(self, event):
# type: (Mapping[str, Any]) -> None
user_profile = get_user_profile_by_id(event["user_profile_id"])
client = get_client(event["client"])
log_time = timestamp_to_datetime(event["time"])
query = event["query"]
do_update_user_activity(user_profile, client, query, log_time)
def api_newrelic_webhook(request, user_profile, alert=REQ(validator=check_dict([]), default=None),
deployment=REQ(validator=check_dict([]), default=None)):
try:
stream = request.GET['stream']
except (AttributeError, KeyError):
return json_error("Missing stream parameter.")
if alert:
# Use the message as the subject because it stays the same for
# "opened", "acknowledged", and "closed" messages that should be
# grouped.
subject = alert['message']
content = "%(long_description)s\n[View alert](%(alert_url)s)" % (alert)
elif deployment:
subject = "%s deploy" % (deployment['application_name'])
content = """`%(revision)s` deployed by **%(deployed_by)s**
%(description)s
%(changelog)s""" % (deployment)
else:
return json_error("Unknown webhook request")
check_send_message(user_profile, get_client("ZulipNewRelicWebhook"), "stream",
[stream], subject, content)
return json_success()
def api_stash_webhook(request, user_profile, payload=REQ(argument_type='body'),
stream=REQ(default='commits')):
# type: (HttpRequest, UserProfile, Dict[str, Any], text_type) -> HttpResponse
# We don't get who did the push, or we'd try to report that.
try:
repo_name = payload["repository"]["name"]
project_name = payload["repository"]["project"]["name"]
branch_name = payload["refChanges"][0]["refId"].split("/")[-1]
commit_entries = payload["changesets"]["values"]
commits = [(entry["toCommit"]["displayId"],
entry["toCommit"]["message"].split("\n")[0]) for \
entry in commit_entries]
head_ref = commit_entries[-1]["toCommit"]["displayId"]
except KeyError as e:
return json_error(_("Missing key %s in JSON") % (e.message,))
subject = "%s/%s: %s" % (project_name, repo_name, branch_name)
content = "`%s` was pushed to **%s** in **%s/%s** with:\n\n" % (
head_ref, branch_name, project_name, repo_name)
content += "\n".join("* `%s`: %s" % (
commit[0], commit[1]) for commit in commits)
check_send_message(user_profile, get_client("ZulipStashWebhook"), "stream",
[stream], subject, content)
return json_success()
def api_stash_webhook(request, user_profile, stream=REQ(default='')):
try:
payload = ujson.loads(request.body)
except ValueError:
return json_error("Malformed JSON input")
# We don't get who did the push, or we'd try to report that.
try:
repo_name = payload["repository"]["name"]
project_name = payload["repository"]["project"]["name"]
branch_name = payload["refChanges"][0]["refId"].split("/")[-1]
commit_entries = payload["changesets"]["values"]
commits = [(entry["toCommit"]["displayId"],
entry["toCommit"]["message"].split("\n")[0]) for \
entry in commit_entries]
head_ref = commit_entries[-1]["toCommit"]["displayId"]
except KeyError as e:
return json_error("Missing key %s in JSON" % (e.message,))
try:
stream = request.GET['stream']
except (AttributeError, KeyError):
stream = 'commits'
subject = "%s/%s: %s" % (project_name, repo_name, branch_name)
content = "`%s` was pushed to **%s** in **%s/%s** with:\n\n" % (
head_ref, branch_name, project_name, repo_name)
content += "\n".join("* `%s`: %s" % (
commit[0], commit[1]) for commit in commits)
check_send_message(user_profile, get_client("ZulipStashWebhook"), "stream",
[stream], subject, content)
return json_success()
def test_messages_sent_by_client(self):
# type: () -> None
stat = COUNT_STATS['messages_sent:client:day']
self.current_property = stat.property
user1 = self.create_user(is_bot=True)
user2 = self.create_user()
recipient_user2 = Recipient.objects.create(type_id=user2.id, type=Recipient.PERSONAL)
recipient_stream = self.create_stream_with_recipient()[1]
recipient_huddle = self.create_huddle_with_recipient()[1]
client2 = Client.objects.create(name='client2')
self.create_message(user1, recipient_user2, sending_client=client2)
self.create_message(user1, recipient_stream)
self.create_message(user1, recipient_huddle)
self.create_message(user2, recipient_user2, sending_client=client2)
self.create_message(user2, recipient_user2, sending_client=client2)
do_fill_count_stat_at_hour(stat, self.TIME_ZERO)
client2_id = str(client2.id)
website_client_id = str(get_client('website').id) # default for self.create_message
self.assertTableState(UserCount, ['value', 'subgroup', 'user'],
[[2, website_client_id, user1],
[1, client2_id, user1], [2, client2_id, user2],
[1, website_client_id, self.hourly_user],
[1, website_client_id, self.daily_user]])
self.assertTableState(RealmCount, ['value', 'subgroup', 'realm'],
[[2, website_client_id], [3, client2_id],
[2, website_client_id, self.second_realm]])
self.assertTableState(InstallationCount, ['value', 'subgroup'],
[[4, website_client_id], [3, client2_id]])
self.assertTableState(StreamCount, [], [])
def consume(self, event: Mapping[str, Any]) -> None:
logging.debug("Received presence event: %s" % (event),)
user_profile = get_user_profile_by_id(event["user_profile_id"])
client = get_client(event["client"])
log_time = timestamp_to_datetime(event["time"])
status = event["status"]
do_update_user_presence(user_profile, client, log_time, status)
def from_dict(cls, d):
ret = cls(d['user_profile_id'], d['realm_id'],
EventQueue.from_dict(d['event_queue']), d['event_types'],
get_client(d['client_type']), d['apply_markdown'], d['all_public_streams'],
d['queue_timeout'], d.get('narrow', []))
ret.last_connection_time = d['last_connection_time']
return ret
def test_mention_everyone(self):
user_profile = get_user_profile_by_email("*****@*****.**")
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "@everyone test"
self.assertEqual(msg.render_markdown(content),
'<p><span class="user-mention" data-user-email="*">@everyone</span> test</p>')
self.assertTrue(msg.mentions_wildcard)
def test_mention_invalid(self):
sender_user_profile = get_user_profile_by_email("*****@*****.**")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "Hey @**Nonexistent User**"
self.assertEqual(msg.render_markdown(content),
'<p>Hey @<strong>Nonexistent User</strong></p>')
self.assertEqual(msg.mentions_user_ids, set())
def process_client(request, user_profile, is_browser_view=False, client_name=None,
remote_server_request=False):
# type: (HttpRequest, UserProfile, bool, Optional[Text], bool) -> None
if client_name is None:
client_name = get_client_name(request, is_browser_view)
request.client = get_client(client_name)
if not remote_server_request:
update_user_activity(request, user_profile)
def send_raw_pagerduty_json(user_profile, stream, message, topic):
subject = topic or 'pagerduty'
body = (
u'Unknown pagerduty message\n'
u'``` py\n'
u'%s\n'
u'```') % (pprint.pformat(message),)
check_send_message(user_profile, get_client('ZulipPagerDutyWebhook'), 'stream',
[stream], subject, body)
def consume_batch(self, user_activity_events: List[Dict[str, Any]]) -> None:
uncommitted_events: Dict[Tuple[int, int, str], Tuple[int, float]] = {}
# First, we drain the queue of all user_activity events and
# deduplicate them for insertion into the database.
for event in user_activity_events:
user_profile_id = event["user_profile_id"]
if "client_id" not in event:
# This is for compatibility with older events still stuck in the queue,
# that used the client name in event["client"] instead of having
# event["client_id"] directly.
# TODO: This can be deleted for release >= 4.0.
if event["client"] not in self.client_id_map:
client = get_client(event["client"])
self.client_id_map[event["client"]] = client.id
client_id = self.client_id_map[event["client"]]
else:
client_id = event["client_id"]
key_tuple = (user_profile_id, client_id, event["query"])
if key_tuple not in uncommitted_events:
uncommitted_events[key_tuple] = (1, event['time'])
else:
count, time = uncommitted_events[key_tuple]
uncommitted_events[key_tuple] = (count + 1, max(time, event['time']))
# Then we insert the updates into the database.
#
# TODO: Doing these updates in sequence individually is likely
# inefficient; the idealized version would do some sort of
# bulk insert_or_update query.
for key_tuple in uncommitted_events:
(user_profile_id, client_id, query) = key_tuple
count, time = uncommitted_events[key_tuple]
log_time = timestamp_to_datetime(time)
do_update_user_activity(user_profile_id, client_id, query, count, log_time)
def test_messages_sent_by_client(self):
# type: () -> None
stat = COUNT_STATS['messages_sent:client:day']
self.current_property = stat.property
user1 = self.create_user(is_bot=True)
user2 = self.create_user()
recipient_user2 = Recipient.objects.create(type_id=user2.id,
type=Recipient.PERSONAL)
recipient_stream = self.create_stream_with_recipient()[1]
recipient_huddle = self.create_huddle_with_recipient()[1]
client2 = Client.objects.create(name='client2')
self.create_message(user1, recipient_user2, sending_client=client2)
self.create_message(user1, recipient_stream)
self.create_message(user1, recipient_huddle)
self.create_message(user2, recipient_user2, sending_client=client2)
self.create_message(user2, recipient_user2, sending_client=client2)
do_fill_count_stat_at_hour(stat, self.TIME_ZERO)
client2_id = str(client2.id)
website_client_id = str(
get_client('website').id) # default for self.create_message
self.assertTableState(
UserCount, ['value', 'subgroup', 'user'],
[[2, website_client_id, user1], [1, client2_id, user1],
[2, client2_id, user2], [1, website_client_id, self.hourly_user],
[1, website_client_id, self.daily_user]])
self.assertTableState(RealmCount, ['value', 'subgroup', 'realm'],
[[2, website_client_id], [3, client2_id],
[2, website_client_id, self.second_realm]])
self.assertTableState(InstallationCount, ['value', 'subgroup'],
[[4, website_client_id], [3, client2_id]])
self.assertTableState(StreamCount, [], [])
def test_inactive_users_queued_for_digest(self, mock_django_timezone,
mock_queue_digest_recipient):
# type: (mock.MagicMock, mock.MagicMock) -> None
cutoff = timezone_now()
# Test Tuesday
mock_django_timezone.return_value = datetime.datetime(year=2016,
month=1,
day=5)
# Mock user activity for each user
realm = get_realm("zulip")
for realm in Realm.objects.filter(deactivated=False,
show_digest_email=True):
for user_profile in UserProfile.objects.filter(realm=realm):
UserActivity.objects.create(last_visit=cutoff -
datetime.timedelta(days=1),
user_profile=user_profile,
count=0,
client=get_client('test_client'))
# Check that inactive users are enqueued
enqueue_emails(cutoff)
self.assertEqual(mock_queue_digest_recipient.call_count, 13)
def api_pivotal_webhook(request, user_profile):
try:
stream = request.GET['stream']
except (AttributeError, KeyError):
return json_error("Missing stream parameter.")
subject = content = None
try:
subject, content = api_pivotal_webhook_v3(request, user_profile, stream)
except AttributeError:
return json_error("Failed to extract data from Pivotal XML response")
except:
# Attempt to parse v5 JSON payload
try:
subject, content = api_pivotal_webhook_v5(request, user_profile, stream)
except AttributeError:
return json_error("Failed to extract data from Pivotal V5 JSON response")
if subject is None or content is None:
return json_error("Unable to handle Pivotal payload")
check_send_message(user_profile, get_client("ZulipPivotalWebhook"), "stream",
[stream], subject, content)
return json_success()
def test_active_users_not_enqueued(
self, mock_django_timezone: mock.MagicMock,
mock_enough_traffic: mock.MagicMock) -> None:
cutoff = timezone_now()
# A Tuesday
mock_django_timezone.return_value = datetime.datetime(year=2016,
month=1,
day=5)
realms = Realm.objects.filter(deactivated=False,
show_digest_email=True)
for realm in realms:
user_profiles = UserProfile.objects.filter(realm=realm)
for counter, user_profile in enumerate(user_profiles, 1):
UserActivity.objects.create(last_visit=cutoff +
datetime.timedelta(days=1),
user_profile=user_profile,
count=0,
client=get_client('test_client'))
# When there are only two users left who have not been set to active,
# check the full enqueue-digest flow for them.
if realm == realms.last(
) and user_profiles.count() - counter == 2:
with mock.patch(
'zerver.models.ScheduledEmail.objects.create'
) as mock_email_create:
enqueue_emails(cutoff)
self.assert_length(mock_email_create.call_args_list, 2)
for call_args in mock_email_create.call_args_list:
email_data = ujson.loads(call_args[1]['data'])
self.assertEqual(email_data['template_prefix'],
'zerver/emails/digest')
# Check that an active user is not enqueued
with mock.patch('zerver.lib.digest.queue_digest_recipient'
) as mock_queue_digest_recipient:
enqueue_emails(cutoff)
self.assertEqual(mock_queue_digest_recipient.call_count, 0)
def test_subscriptions(self) -> None:
now = timezone_now()
user = self.example_user("hamlet")
stream = self.make_stream("test_stream")
acting_user = self.example_user("iago")
bulk_add_subscriptions(user.realm, [stream], [user],
acting_user=acting_user)
subscription_creation_logs = RealmAuditLog.objects.filter(
event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
event_time__gte=now,
acting_user=acting_user,
modified_user=user,
modified_stream=stream,
)
modified_stream = subscription_creation_logs[0].modified_stream
assert modified_stream is not None
self.assertEqual(subscription_creation_logs.count(), 1)
self.assertEqual(modified_stream.id, stream.id)
self.assertEqual(subscription_creation_logs[0].modified_user, user)
bulk_remove_subscriptions([user], [stream],
get_client("website"),
acting_user=acting_user)
subscription_deactivation_logs = RealmAuditLog.objects.filter(
event_type=RealmAuditLog.SUBSCRIPTION_DEACTIVATED,
event_time__gte=now,
acting_user=acting_user,
modified_user=user,
modified_stream=stream,
)
modified_stream = subscription_deactivation_logs[0].modified_stream
assert modified_stream is not None
self.assertEqual(subscription_deactivation_logs.count(), 1)
self.assertEqual(modified_stream.id, stream.id)
self.assertEqual(subscription_deactivation_logs[0].modified_user, user)
def api_bitbucket_webhook(request, user_profile, payload=REQ(validator=check_dict([])),
stream=REQ(default='commits')):
repository = payload['repository']
commits = [{'id': commit['raw_node'], 'message': commit['message'],
'url': '%s%scommits/%s' % (payload['canon_url'],
repository['absolute_url'],
commit['raw_node'])}
for commit in payload['commits']]
subject = repository['name']
if len(commits) == 0:
# Bitbucket doesn't give us enough information to really give
# a useful message :/
content = ("%s [force pushed](%s)"
% (payload['user'],
payload['canon_url'] + repository['absolute_url']))
else:
branch = payload['commits'][-1]['branch']
content = build_commit_list_content(commits, branch, None, payload['user'])
subject += '/%s' % (branch,)
check_send_message(user_profile, get_client("ZulipBitBucketWebhook"), "stream",
[stream], subject, content)
return json_success()
def send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open("var/test_messages.json", "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
] # type: List[int]
recipient_huddles = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100. / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + str(random.randint(1, 3))
saved_data['subject'] = message.subject
# Spoofing time not supported with threading
if options['threads'] != 1:
message.pub_date = timezone_now()
else:
# Distrubutes 80% of messages starting from 5 days ago, over a period
# of 3 days. Then, distributes remaining messages over past 24 hours.
spoofed_date = timezone_now() - timezone_timedelta(days=5)
if (num_messages < tot_messages * 0.8):
# Maximum of 3 days ahead, convert to minutes
time_ahead = 3 * 24 * 60
time_ahead //= int(tot_messages * 0.8)
else:
time_ahead = 24 * 60
time_ahead //= int(tot_messages * 0.2)
spoofed_minute = random.randint(time_ahead * num_messages,
time_ahead * (num_messages + 1))
spoofed_date += timezone_timedelta(minutes=spoofed_minute)
message.pub_date = spoofed_date
# We disable USING_RABBITMQ here, so that deferred work is
# executed in do_send_message_messages, rather than being
# queued. This is important, because otherwise, if run-dev.py
# wasn't running when populate_db was run, a developer can end
# up with queued events that reference objects from a previous
# life of the database, which naturally throws exceptions.
settings.USING_RABBITMQ = False
do_send_messages([{'message': message}])
settings.USING_RABBITMQ = True
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
return tot_messages
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write(
"Error! More than 100% of messages allocated.\n")
return
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our two default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
zulip_realm = Realm.objects.create(
string_id="zulip",
name="Zulip Dev",
restricted_to_domain=True,
description=
"The Zulip development environment default organization."
" It's great for testing!",
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
if options["test_suite"]:
mit_realm = Realm.objects.create(string_id="zephyr",
name="MIT",
restricted_to_domain=True,
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = Realm.objects.create(string_id="lear",
name="Lear & Co.",
restricted_to_domain=False,
invite_required=False,
org_type=Realm.CORPORATE)
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
("Polonius", "*****@*****.**"),
]
for i in range(options["extra_users"]):
names.append(
('Extra User %d' % (i, ), '*****@*****.**' % (i, )))
create_users(zulip_realm, names)
iago = get_user("*****@*****.**", zulip_realm)
do_change_is_admin(iago, True)
iago.is_staff = True
iago.save(update_fields=['is_staff'])
guest_user = get_user("*****@*****.**", zulip_realm)
guest_user.is_guest = True
guest_user.save(update_fields=['is_guest'])
# These bots are directly referenced from code and thus
# are needed for the test suite.
all_realm_bots = [
(bot['name'],
bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN, ))
for bot in settings.INTERNAL_BOTS
]
zulip_realm_bots = [
("Zulip New User Bot", "*****@*****.**"),
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
("Welcome Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(
('Extra Bot %d' % (i, ), '*****@*****.**' % (i, )))
zulip_realm_bots.extend(all_realm_bots)
create_users(zulip_realm,
zulip_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
zoe = get_user("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(zulip_realm,
zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
bot_owner=zoe)
aaron = get_user("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [("Outgoing Webhook",
"*****@*****.**")]
create_users(zulip_realm,
zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
bot_owner=aaron)
# TODO: Clean up this initial bot creation code
Service.objects.create(
name="test",
user_profile=get_user("*****@*****.**",
zulip_realm),
base_url="http://127.0.0.1:5002/bots/followup",
token="abcd1234",
interface=1)
# Create public streams.
stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
stream_dict = {
"Verona": {
"description": "A city in Italy",
"invite_only": False,
"is_web_public": False
},
"Denmark": {
"description": "A Scandinavian country",
"invite_only": False,
"is_web_public": False
},
"Scotland": {
"description": "Located in the United Kingdom",
"invite_only": False,
"is_web_public": False
},
"Venice": {
"description": "A northeastern Italian city",
"invite_only": False,
"is_web_public": False
},
"Rome": {
"description": "Yet another Italian city",
"invite_only": False,
"is_web_public": True
}
} # type: Dict[str, Dict[str, Any]]
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams = [
Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list
] # type: List[int]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users).
subscriptions_to_add = [] # type: List[Subscription]
event_time = timezone_now()
all_subscription_logs = [] # type: (List[RealmAuditLog])
profiles = UserProfile.objects.select_related().filter(
is_bot=False, is_guest=False).order_by(
"email") # type: Sequence[UserProfile]
for i, profile in enumerate(profiles):
# Subscribe to some streams.
for type_id in recipient_streams[:int(
len(recipient_streams) * float(i) / len(profiles)) +
1]:
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=type_id)
s = Subscription(recipient=r,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[
i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream_id=type_id,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(
zulip_realm,
"Phone number",
CustomProfileField.SHORT_TEXT,
hint='')
biography = try_add_realm_custom_profile_field(
zulip_realm,
"Biography",
CustomProfileField.LONG_TEXT,
hint='What are you known for?')
favorite_food = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite food",
CustomProfileField.SHORT_TEXT,
hint="Or drink, if you'd prefer")
field_data = {
'vim': {
'text': 'Vim',
'order': '1'
},
'emacs': {
'text': 'Emacs',
'order': '2'
},
}
favorite_editor = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite editor",
CustomProfileField.CHOICE,
field_data=field_data)
birthday = try_add_realm_custom_profile_field(
zulip_realm, "Birthday", CustomProfileField.DATE)
favorite_website = try_add_realm_custom_profile_field(
zulip_realm,
"GitHub profile",
CustomProfileField.URL,
hint="Or your personal blog's URL")
# Fill in values for Iago and Hamlet
hamlet = get_user("*****@*****.**", zulip_realm)
do_update_user_custom_profile_data(iago, [
{
"id": phone_number.id,
"value": "+1-234-567-8901"
},
{
"id": biography.id,
"value": "Betrayer of Othello."
},
{
"id": favorite_food.id,
"value": "Apples"
},
{
"id": favorite_editor.id,
"value": "emacs"
},
{
"id": birthday.id,
"value": "2000-1-1"
},
{
"id": favorite_website.id,
"value": "https://github.com/zulip/zulip"
},
])
do_update_user_custom_profile_data(hamlet, [
{
"id": phone_number.id,
"value": "+0-11-23-456-7890"
},
{
"id": biography.id,
"value": "Prince of Denmark, and other things!"
},
{
"id": favorite_food.id,
"value": "Dark chocolate"
},
{
"id": favorite_editor.id,
"value": "vim"
},
{
"id": birthday.id,
"value": "1900-1-1"
},
{
"id": favorite_website.id,
"value": "https://blog.zulig.org"
},
])
else:
zulip_realm = get_realm("zulip")
recipient_streams = [
klass.type_id
for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
# Extract a list of all users
user_profiles = list(UserProfile.objects.filter(
is_bot=False)) # type: List[UserProfile]
# Create a test realm emoji.
IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images',
'test-images', 'checkbox.png')
with open(IMAGE_FILE_PATH, 'rb') as fp:
check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status = UserPresence.ACTIVE # type: int
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= 'H':
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(user_profile=user,
client=client,
timestamp=date,
status=status)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [
random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])
]
# Generate a new set of test data.
create_test_data()
# prepopulate the URL preview/embed data for the links present
# in the config.generate_data.json data set. This makes it
# possible for populate_db to run happily without Internet
# access.
with open("zerver/tests/fixtures/docs_url_preview_data.json",
"r") as f:
urls_with_preview_data = ujson.load(f)
for url in urls_with_preview_data:
cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)
threads = options["threads"]
jobs = [
] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write,
random.randint(0, 10**10)))
for job in jobs:
send_messages(job)
if options["delete"]:
# Create the "website" and "API" clients; if we don't, the
# default values in zerver/decorators.py will not work
# with the Django test suite.
get_client("website")
get_client("API")
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)",
"*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm, testsuite_mit_users)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
]
create_users(lear_realm, testsuite_lear_users)
if not options["test_suite"]:
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
zulip_stream_dict = {
"devel": {
"description": "For developing",
"invite_only": False,
"is_web_public": False
},
"all": {
"description": "For everything",
"invite_only": False,
"is_web_public": False
},
"announce": {
"description": "For announcements",
"invite_only": False,
"is_web_public": False
},
"design": {
"description": "For design",
"invite_only": False,
"is_web_public": False
},
"support": {
"description": "For support",
"invite_only": False,
"is_web_public": False
},
"social": {
"description": "For socializing",
"invite_only": False,
"is_web_public": False
},
"test": {
"description": "For testing",
"invite_only": False,
"is_web_public": False
},
"errors": {
"description": "For errors",
"invite_only": False,
"is_web_public": False
},
"sales": {
"description": "For sales discussion",
"invite_only": False,
"is_web_public": False
}
} # type: Dict[str, Dict[str, Any]]
# Calculate the maximum number of digits in any extra stream's
# number, since a stream with name "Extra Stream 3" could show
# up after "Extra Stream 29". (Used later to pad numbers with
# 0s).
maximum_digits = len(str(options['extra_streams'] - 1))
for i in range(options['extra_streams']):
# Pad the number with 0s based on `maximum_digits`.
number_str = str(i).zfill(maximum_digits)
extra_stream_name = 'Extra Stream ' + number_str
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
"invite_only": False,
"is_web_public": False,
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream(
"announce", zulip_realm)
zulip_realm.save(update_fields=['notifications_stream'])
# Add a few default streams
for default_stream_name in [
"design", "devel", "social", "support"
]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(
default_stream_name,
zulip_realm))
# Now subscribe everyone to these streams
subscriptions_to_add = []
event_time = timezone_now()
all_subscription_logs = []
profiles = UserProfile.objects.select_related().filter(
realm=zulip_realm)
for i, stream_name in enumerate(zulip_stream_dict):
stream = Stream.objects.get(name=stream_name,
realm=zulip_realm)
recipient = Recipient.objects.get(type=Recipient.STREAM,
type_id=stream.id)
for profile in profiles:
# Subscribe to some streams.
s = Subscription(
recipient=recipient,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[
i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream=stream,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# These bots are not needed by the test suite
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm,
internal_zulip_users_nosubs,
bot_type=UserProfile.DEFAULT_BOT)
zulip_cross_realm_bots = [
("Zulip Feedback Bot", "*****@*****.**"),
]
create_users(zulip_realm,
zulip_cross_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
# Mark all messages as read
UserMessage.objects.all().update(flags=UserMessage.flags.read)
if not options["test_suite"]:
# Update pointer of each user to point to the last message in their
# UserMessage rows with sender_id=user_profile_id.
users = list(
UserMessage.objects.filter(message__sender_id=F(
'user_profile_id')).values('user_profile_id').annotate(
pointer=Max('message_id')))
for user in users:
UserProfile.objects.filter(
id=user['user_profile_id']).update(
pointer=user['pointer'])
create_user_groups()
self.stdout.write("Successfully populated test database.\n")
def handle(self, **options):
# type: (**Any) -> None
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write(
"Error! More than 100% of messages allocated.\n")
return
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our two default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
zulip_realm = Realm.objects.create(
string_id="zulip",
name="Zulip Dev",
restricted_to_domain=True,
description=
"The Zulip development environment default organization."
" It's great for testing!",
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
if options["test_suite"]:
mit_realm = Realm.objects.create(string_id="zephyr",
name="MIT",
restricted_to_domain=True,
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
]
for i in range(options["extra_users"]):
names.append(
('Extra User %d' % (i, ), '*****@*****.**' % (i, )))
create_users(zulip_realm, names)
iago = get_user("*****@*****.**", zulip_realm)
do_change_is_admin(iago, True)
iago.is_staff = True
iago.save(update_fields=['is_staff'])
# These bots are directly referenced from code and thus
# are needed for the test suite.
all_realm_bots = [
(bot['name'],
bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN, ))
for bot in settings.INTERNAL_BOTS
]
zulip_realm_bots = [
("Zulip New User Bot", "*****@*****.**"),
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
("Welcome Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(
('Extra Bot %d' % (i, ), '*****@*****.**' % (i, )))
zulip_realm_bots.extend(all_realm_bots)
create_users(zulip_realm,
zulip_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
create_users(zulip_realm,
zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT)
zulip_outgoing_bots = [("Outgoing Webhook",
"*****@*****.**")]
aaron = get_user("*****@*****.**", zulip_realm)
create_users(zulip_realm,
zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
bot_owner=aaron)
# TODO: Clean up this initial bot creation code
Service.objects.create(
name="test",
user_profile=get_user("*****@*****.**",
zulip_realm),
base_url="http://127.0.0.1:5002/bots/followup",
token="abcd1234",
interface=1)
# Create public streams.
stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
stream_dict = {
"Verona": {
"description": "A city in Italy",
"invite_only": False
},
"Denmark": {
"description": "A Scandinavian country",
"invite_only": False
},
"Scotland": {
"description": "Located in the United Kingdom",
"invite_only": False
},
"Venice": {
"description": "A northeastern Italian city",
"invite_only": False
},
"Rome": {
"description": "Yet another Italian city",
"invite_only": False
}
} # type: Dict[Text, Dict[Text, Any]]
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams = [
Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list
] # type: List[int]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users).
subscriptions_to_add = [] # type: List[Subscription]
event_time = timezone_now()
all_subscription_logs = [] # type: (List[RealmAuditLog])
profiles = UserProfile.objects.select_related().filter(
is_bot=False).order_by("email") # type: Sequence[UserProfile]
for i, profile in enumerate(profiles):
# Subscribe to some streams.
for type_id in recipient_streams[:int(
len(recipient_streams) * float(i) / len(profiles)) +
1]:
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=type_id)
s = Subscription(recipient=r,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[
i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream_id=type_id,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
else:
zulip_realm = get_realm("zulip")
recipient_streams = [
klass.type_id
for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
# Extract a list of all users
user_profiles = list(UserProfile.objects.filter(
is_bot=False)) # type: List[UserProfile]
# Create a test realm emoji.
IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images',
'test-images', 'checkbox.png')
UPLOADED_EMOJI_FILE_NAME = 'green_tick.png'
with open(IMAGE_FILE_PATH, 'rb') as fp:
upload_backend.upload_emoji_image(fp, UPLOADED_EMOJI_FILE_NAME,
iago)
RealmEmoji.objects.create(
name='green_tick',
author=iago,
realm=zulip_realm,
deactivated=False,
file_name=UPLOADED_EMOJI_FILE_NAME,
)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status = UserPresence.ACTIVE # type: int
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= 'H':
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(user_profile=user,
client=client,
timestamp=date,
status=status)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [
random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])
]
# Generate a new set of test data.
create_test_data()
threads = options["threads"]
jobs = [
] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write,
random.randint(0, 10**10)))
for job in jobs:
send_messages(job)
if options["delete"]:
# Create the "website" and "API" clients; if we don't, the
# default values in zerver/decorators.py will not work
# with the Django test suite.
get_client("website")
get_client("API")
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)",
"*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm, testsuite_mit_users)
if not options["test_suite"]:
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
zulip_stream_dict = {
"devel": {
"description": "For developing",
"invite_only": False
},
"all": {
"description": "For everything",
"invite_only": False
},
"announce": {
"description": "For announcements",
"invite_only": False
},
"design": {
"description": "For design",
"invite_only": False
},
"support": {
"description": "For support",
"invite_only": False
},
"social": {
"description": "For socializing",
"invite_only": False
},
"test": {
"description": "For testing",
"invite_only": False
},
"errors": {
"description": "For errors",
"invite_only": False
},
"sales": {
"description": "For sales discussion",
"invite_only": False
}
} # type: Dict[Text, Dict[Text, Any]]
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream(
"announce", zulip_realm)
zulip_realm.save(update_fields=['notifications_stream'])
# Add a few default streams
for default_stream_name in [
"design", "devel", "social", "support"
]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(
default_stream_name,
zulip_realm))
# Now subscribe everyone to these streams
subscriptions_to_add = []
event_time = timezone_now()
all_subscription_logs = []
profiles = UserProfile.objects.select_related().filter(
realm=zulip_realm)
for i, stream_name in enumerate(zulip_stream_dict):
stream = Stream.objects.get(name=stream_name,
realm=zulip_realm)
recipient = Recipient.objects.get(type=Recipient.STREAM,
type_id=stream.id)
for profile in profiles:
# Subscribe to some streams.
s = Subscription(
recipient=recipient,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[
i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream=stream,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# These bots are not needed by the test suite
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm,
internal_zulip_users_nosubs,
bot_type=UserProfile.DEFAULT_BOT)
zulip_cross_realm_bots = [
("Zulip Feedback Bot", "*****@*****.**"),
]
create_users(zulip_realm,
zulip_cross_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
# Mark all messages as read
UserMessage.objects.all().update(flags=UserMessage.flags.read)
if not options["test_suite"]:
# Update pointer of each user to point to the last message in their
# UserMessage rows with sender_id=user_profile_id.
users = list(
UserMessage.objects.filter(message__sender_id=F(
'user_profile_id')).values('user_profile_id').annotate(
pointer=Max('message_id')))
for user in users:
UserProfile.objects.filter(
id=user['user_profile_id']).update(
pointer=user['pointer'])
create_user_groups()
self.stdout.write("Successfully populated test database.\n")
def api_deskdotcom_webhook(request, user_profile, data=REQ(),
topic=REQ(default="Desk.com notification"),
stream=REQ(default="desk.com")):
check_send_message(user_profile, get_client("ZulipDeskWebhook"), "stream",
[stream], topic, data)
return json_success()
def unsubscribe(self, user_profile: UserProfile, stream_name: str) -> None:
client = get_client("website")
stream = get_stream(stream_name, user_profile.realm)
bulk_remove_subscriptions([user_profile], [stream], client)
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write(
"Error! More than 100% of messages allocated.\n")
return
# Get consistent data for backend tests.
if options["test_suite"]:
random.seed(0)
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our three default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
create_internal_realm()
zulip_realm = Realm.objects.create(
string_id="zulip",
name="Zulip Dev",
emails_restricted_to_domains=True,
description=
"The Zulip development environment default organization."
" It's great for testing!",
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
if options["test_suite"]:
mit_realm = Realm.objects.create(
string_id="zephyr",
name="MIT",
emails_restricted_to_domains=True,
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = Realm.objects.create(
string_id="lear",
name="Lear & Co.",
emails_restricted_to_domains=False,
invite_required=False,
org_type=Realm.CORPORATE)
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
("Polonius", "*****@*****.**"),
]
# For testing really large batches:
# Create extra users with semi realistic names to make search
# functions somewhat realistic. We'll still create 1000 users
# like Extra222 User for some predicability.
num_names = options['extra_users']
num_boring_names = 1000
for i in range(min(num_names, num_boring_names)):
full_name = 'Extra%03d User' % (i, )
names.append((full_name, '*****@*****.**' % (i, )))
if num_names > num_boring_names:
fnames = [
'Amber', 'Arpita', 'Bob', 'Cindy', 'Daniela', 'Dan',
'Dinesh', 'Faye', 'François', 'George', 'Hank', 'Irene',
'James', 'Janice', 'Jenny', 'Jill', 'John', 'Kate',
'Katelyn', 'Kobe', 'Lexi', 'Manish', 'Mark', 'Matt',
'Mayna', 'Michael', 'Pete', 'Peter', 'Phil', 'Phillipa',
'Preston', 'Sally', 'Scott', 'Sandra', 'Steve',
'Stephanie', 'Vera'
]
mnames = ['de', 'van', 'von', 'Shaw', 'T.']
lnames = [
'Adams', 'Agarwal', 'Beal', 'Benson', 'Bonita', 'Davis',
'George', 'Harden', 'James', 'Jones', 'Johnson', 'Jordan',
'Lee', 'Leonard', 'Singh', 'Smith', 'Patel', 'Towns',
'Wall'
]
for i in range(num_boring_names, num_names):
fname = random.choice(fnames) + str(i)
full_name = fname
if random.random() < 0.7:
if random.random() < 0.5:
full_name += ' ' + random.choice(mnames)
full_name += ' ' + random.choice(lnames)
email = fname.lower() + '@zulip.com'
names.append((full_name, email))
create_users(zulip_realm, names, tos_version=settings.TOS_VERSION)
iago = get_user("*****@*****.**", zulip_realm)
do_change_is_admin(iago, True)
iago.is_staff = True
iago.save(update_fields=['is_staff'])
guest_user = get_user("*****@*****.**", zulip_realm)
guest_user.role = UserProfile.ROLE_GUEST
guest_user.save(update_fields=['role'])
# These bots are directly referenced from code and thus
# are needed for the test suite.
zulip_realm_bots = [
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(
('Extra Bot %d' % (i, ), '*****@*****.**' % (i, )))
create_users(zulip_realm,
zulip_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
zoe = get_user("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(zulip_realm,
zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
bot_owner=zoe)
aaron = get_user("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [("Outgoing Webhook",
"*****@*****.**")]
create_users(zulip_realm,
zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
bot_owner=aaron)
outgoing_webhook = get_user("*****@*****.**",
zulip_realm)
add_service("outgoing-webhook",
user_profile=outgoing_webhook,
interface=Service.GENERIC,
base_url="http://127.0.0.1:5002",
token=generate_api_key())
# Add the realm internl bots to each realm.
create_if_missing_realm_internal_bots()
# Create public streams.
stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
stream_dict = {
"Verona": {
"description": "A city in Italy"
},
"Denmark": {
"description": "A Scandinavian country"
},
"Scotland": {
"description": "Located in the United Kingdom"
},
"Venice": {
"description": "A northeastern Italian city"
},
"Rome": {
"description": "Yet another Italian city",
"is_web_public": True
}
} # type: Dict[str, Dict[str, Any]]
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams = [
Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list
] # type: List[int]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users). For the test suite, we have a fixed list of
# subscriptions to make sure test data is consistent
# across platforms.
subscriptions_list = [
] # type: List[Tuple[UserProfile, Recipient]]
profiles = UserProfile.objects.select_related().filter(
is_bot=False).order_by("email") # type: Sequence[UserProfile]
if options["test_suite"]:
subscriptions_map = {
'*****@*****.**': ['Verona'],
'*****@*****.**': ['Verona'],
'*****@*****.**': ['Verona', 'Denmark'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
'*****@*****.**':
['Verona', 'Denmark', 'Scotland', 'Venice'],
'*****@*****.**':
['Verona', 'Denmark', 'Scotland', 'Venice', 'Rome'],
'*****@*****.**': ['Verona'],
}
for profile in profiles:
if profile.email not in subscriptions_map:
raise Exception(
'Subscriptions not listed for user %s' %
(profile.email, ))
for stream_name in subscriptions_map[profile.email]:
stream = Stream.objects.get(name=stream_name)
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=stream.id)
subscriptions_list.append((profile, r))
else:
num_streams = len(recipient_streams)
num_users = len(profiles)
for i, profile in enumerate(profiles):
# Subscribe to some streams.
fraction = float(i) / num_users
num_recips = int(num_streams * fraction) + 1
for type_id in recipient_streams[:num_recips]:
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=type_id)
subscriptions_list.append((profile, r))
subscriptions_to_add = [] # type: List[Subscription]
event_time = timezone_now()
all_subscription_logs = [] # type: (List[RealmAuditLog])
i = 0
for profile, recipient in subscriptions_list:
i += 1
color = STREAM_ASSIGNMENT_COLORS[i %
len(STREAM_ASSIGNMENT_COLORS)]
s = Subscription(recipient=recipient,
user_profile=profile,
color=color)
subscriptions_to_add.append(s)
log = RealmAuditLog(
realm=profile.realm,
modified_user=profile,
modified_stream_id=recipient.type_id,
event_last_message_id=0,
event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(
zulip_realm,
"Phone number",
CustomProfileField.SHORT_TEXT,
hint='')
biography = try_add_realm_custom_profile_field(
zulip_realm,
"Biography",
CustomProfileField.LONG_TEXT,
hint='What are you known for?')
favorite_food = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite food",
CustomProfileField.SHORT_TEXT,
hint="Or drink, if you'd prefer")
field_data = {
'vim': {
'text': 'Vim',
'order': '1'
},
'emacs': {
'text': 'Emacs',
'order': '2'
},
} # type: ProfileFieldData
favorite_editor = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite editor",
CustomProfileField.CHOICE,
field_data=field_data)
birthday = try_add_realm_custom_profile_field(
zulip_realm, "Birthday", CustomProfileField.DATE)
favorite_website = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite website",
CustomProfileField.URL,
hint="Or your personal blog's URL")
mentor = try_add_realm_custom_profile_field(
zulip_realm, "Mentor", CustomProfileField.USER)
github_profile = try_add_realm_default_custom_profile_field(
zulip_realm, "github")
# Fill in values for Iago and Hamlet
hamlet = get_user("*****@*****.**", zulip_realm)
do_update_user_custom_profile_data_if_changed(
iago, [
{
"id": phone_number.id,
"value": "+1-234-567-8901"
},
{
"id": biography.id,
"value": "Betrayer of Othello."
},
{
"id": favorite_food.id,
"value": "Apples"
},
{
"id": favorite_editor.id,
"value": "emacs"
},
{
"id": birthday.id,
"value": "2000-1-1"
},
{
"id": favorite_website.id,
"value": "https://zulip.readthedocs.io/en/latest/"
},
{
"id": mentor.id,
"value": [hamlet.id]
},
{
"id": github_profile.id,
"value": 'zulip'
},
])
do_update_user_custom_profile_data_if_changed(
hamlet, [
{
"id": phone_number.id,
"value": "+0-11-23-456-7890"
},
{
"id":
biography.id,
"value":
"I am:\n* The prince of Denmark\n* Nephew to the usurping Claudius",
},
{
"id": favorite_food.id,
"value": "Dark chocolate"
},
{
"id": favorite_editor.id,
"value": "vim"
},
{
"id": birthday.id,
"value": "1900-1-1"
},
{
"id": favorite_website.id,
"value": "https://blog.zulig.org"
},
{
"id": mentor.id,
"value": [iago.id]
},
{
"id": github_profile.id,
"value": 'zulipbot'
},
])
else:
zulip_realm = get_realm("zulip")
recipient_streams = [
klass.type_id
for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
# Extract a list of all users
user_profiles = list(UserProfile.objects.filter(
is_bot=False)) # type: List[UserProfile]
# Create a test realm emoji.
IMAGE_FILE_PATH = static_path('images/test-images/checkbox.png')
with open(IMAGE_FILE_PATH, 'rb') as fp:
check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status = UserPresence.ACTIVE # type: int
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= 'H':
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(user_profile=user,
client=client,
timestamp=date,
status=status)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [
random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])
]
# Generate a new set of test data.
create_test_data()
# prepopulate the URL preview/embed data for the links present
# in the config.generate_data.json data set. This makes it
# possible for populate_db to run happily without Internet
# access.
with open("zerver/tests/fixtures/docs_url_preview_data.json",
"r") as f:
urls_with_preview_data = ujson.load(f)
for url in urls_with_preview_data:
cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)
threads = options["threads"]
jobs = [
] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write,
random.randint(0, 10**10)))
for job in jobs:
generate_and_send_messages(job)
if options["delete"]:
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)",
"*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm,
testsuite_mit_users,
tos_version=settings.TOS_VERSION)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
]
create_users(lear_realm,
testsuite_lear_users,
tos_version=settings.TOS_VERSION)
if not options["test_suite"]:
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
zulip_stream_dict = {
"devel": {
"description": "For developing"
},
"all": {
"description": "For **everything**"
},
"announce": {
"description": "For announcements",
'stream_post_policy': Stream.STREAM_POST_POLICY_ADMINS
},
"design": {
"description": "For design"
},
"support": {
"description": "For support"
},
"social": {
"description": "For socializing"
},
"test": {
"description": "For testing `code`"
},
"errors": {
"description": "For errors"
},
"sales": {
"description": "For sales discussion"
}
} # type: Dict[str, Dict[str, Any]]
# Calculate the maximum number of digits in any extra stream's
# number, since a stream with name "Extra Stream 3" could show
# up after "Extra Stream 29". (Used later to pad numbers with
# 0s).
maximum_digits = len(str(options['extra_streams'] - 1))
for i in range(options['extra_streams']):
# Pad the number with 0s based on `maximum_digits`.
number_str = str(i).zfill(maximum_digits)
extra_stream_name = 'Extra Stream ' + number_str
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream(
"announce", zulip_realm)
zulip_realm.save(update_fields=['notifications_stream'])
# Add a few default streams
for default_stream_name in [
"design", "devel", "social", "support"
]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(
default_stream_name,
zulip_realm))
# Now subscribe everyone to these streams
subscribe_users_to_streams(zulip_realm, zulip_stream_dict)
# These bots are not needed by the test suite
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm,
internal_zulip_users_nosubs,
bot_type=UserProfile.DEFAULT_BOT)
# Mark all messages as read
UserMessage.objects.all().update(flags=UserMessage.flags.read)
if not options["test_suite"]:
# Update pointer of each user to point to the last message in their
# UserMessage rows with sender_id=user_profile_id.
users = list(
UserMessage.objects.filter(message__sender_id=F(
'user_profile_id')).values('user_profile_id').annotate(
pointer=Max('message_id')))
for user in users:
UserProfile.objects.filter(
id=user['user_profile_id']).update(
pointer=user['pointer'])
create_user_groups()
if not options["test_suite"]:
# We populate the analytics database here for
# development purpose only
call_command('populate_analytics_db')
self.stdout.write("Successfully populated test database.\n")
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write(
"Error! More than 100% of messages allocated.\n")
return
# Get consistent data for backend tests.
if options["test_suite"]:
random.seed(0)
with connection.cursor() as cursor:
# Sometimes bugs relating to confusing recipient.id for recipient.type_id
# or <object>.id for <object>.recipient_id remain undiscovered by the test suite
# due to these numbers happening to coincide in such a way that it makes tests
# accidentally pass. By bumping the Recipient.id sequence by a large enough number,
# we can have those ids in a completely different range of values than object ids,
# eliminatng the possibility of such coincidences.
cursor.execute("SELECT setval('zerver_recipient_id_seq', 100)")
# If max_topics is not set, we set it proportional to the
# number of messages.
if options["max_topics"] is None:
options["max_topics"] = 1 + options["num_messages"] // 100
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our three default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
create_internal_realm()
zulip_realm = do_create_realm(
string_id="zulip",
name="Zulip Dev",
emails_restricted_to_domains=False,
email_address_visibility=Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS,
description=
"The Zulip development environment default organization."
" It's great for testing!",
invite_required=False,
plan_type=Realm.SELF_HOSTED,
org_type=Realm.ORG_TYPES["business"]["id"],
)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
assert zulip_realm.notifications_stream is not None
zulip_realm.notifications_stream.name = "Verona"
zulip_realm.notifications_stream.description = "A city in Italy"
zulip_realm.notifications_stream.save(
update_fields=["name", "description"])
if options["test_suite"]:
mit_realm = do_create_realm(
string_id="zephyr",
name="MIT",
emails_restricted_to_domains=True,
invite_required=False,
plan_type=Realm.SELF_HOSTED,
org_type=Realm.ORG_TYPES["business"]["id"],
)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = do_create_realm(
string_id="lear",
name="Lear & Co.",
emails_restricted_to_domains=False,
invite_required=False,
plan_type=Realm.SELF_HOSTED,
org_type=Realm.ORG_TYPES["business"]["id"],
)
# Default to allowing all members to send mentions in
# large streams for the test suite to keep
# mention-related tests simple.
zulip_realm.wildcard_mention_policy = Realm.WILDCARD_MENTION_POLICY_MEMBERS
zulip_realm.save(update_fields=["wildcard_mention_policy"])
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia, Lear's daughter", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
("Polonius", "*****@*****.**"),
("Desdemona", "*****@*****.**"),
("शिव", "*****@*****.**"),
]
# For testing really large batches:
# Create extra users with semi realistic names to make search
# functions somewhat realistic. We'll still create 1000 users
# like Extra222 User for some predicability.
num_names = options["extra_users"]
num_boring_names = 300
for i in range(min(num_names, num_boring_names)):
full_name = f"Extra{i:03} User"
names.append((full_name, f"extrauser{i}@zulip.com"))
if num_names > num_boring_names:
fnames = [
"Amber",
"Arpita",
"Bob",
"Cindy",
"Daniela",
"Dan",
"Dinesh",
"Faye",
"François",
"George",
"Hank",
"Irene",
"James",
"Janice",
"Jenny",
"Jill",
"John",
"Kate",
"Katelyn",
"Kobe",
"Lexi",
"Manish",
"Mark",
"Matt",
"Mayna",
"Michael",
"Pete",
"Peter",
"Phil",
"Phillipa",
"Preston",
"Sally",
"Scott",
"Sandra",
"Steve",
"Stephanie",
"Vera",
]
mnames = ["de", "van", "von", "Shaw", "T."]
lnames = [
"Adams",
"Agarwal",
"Beal",
"Benson",
"Bonita",
"Davis",
"George",
"Harden",
"James",
"Jones",
"Johnson",
"Jordan",
"Lee",
"Leonard",
"Singh",
"Smith",
"Patel",
"Towns",
"Wall",
]
non_ascii_names = [
"Günter",
"أحمد",
"Magnús",
"आशी",
"イツキ",
"语嫣",
"அருண்",
"Александр",
"José",
]
# to imitate emoji insertions in usernames
raw_emojis = ["😎", "😂", "🐱👤"]
for i in range(num_boring_names, num_names):
fname = random.choice(fnames) + str(i)
full_name = fname
if random.random() < 0.7:
if random.random() < 0.3:
full_name += " " + random.choice(non_ascii_names)
else:
full_name += " " + random.choice(mnames)
if random.random() < 0.1:
full_name += " {} ".format(random.choice(raw_emojis))
else:
full_name += " " + random.choice(lnames)
email = fname.lower() + "@zulip.com"
names.append((full_name, email))
create_users(zulip_realm, names, tos_version=settings.TOS_VERSION)
iago = get_user_by_delivery_email("*****@*****.**", zulip_realm)
do_change_user_role(iago,
UserProfile.ROLE_REALM_ADMINISTRATOR,
acting_user=None)
iago.is_staff = True
iago.save(update_fields=["is_staff"])
desdemona = get_user_by_delivery_email("*****@*****.**",
zulip_realm)
do_change_user_role(desdemona,
UserProfile.ROLE_REALM_OWNER,
acting_user=None)
shiva = get_user_by_delivery_email("*****@*****.**", zulip_realm)
do_change_user_role(shiva,
UserProfile.ROLE_MODERATOR,
acting_user=None)
guest_user = get_user_by_delivery_email("*****@*****.**",
zulip_realm)
guest_user.role = UserProfile.ROLE_GUEST
guest_user.save(update_fields=["role"])
# These bots are directly referenced from code and thus
# are needed for the test suite.
zulip_realm_bots = [
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(
(f"Extra Bot {i}", f"extrabot{i}@zulip.com"))
create_users(zulip_realm,
zulip_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
zoe = get_user_by_delivery_email("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(
zulip_realm,
zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
bot_owner=zoe,
)
aaron = get_user_by_delivery_email("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [
("Outgoing Webhook", "*****@*****.**"),
]
create_users(
zulip_realm,
zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
bot_owner=aaron,
)
outgoing_webhook = get_user("*****@*****.**",
zulip_realm)
add_service(
"outgoing-webhook",
user_profile=outgoing_webhook,
interface=Service.GENERIC,
base_url="http://127.0.0.1:5002",
token=generate_api_key(),
)
# Add the realm internal bots to each realm.
create_if_missing_realm_internal_bots()
# Create public streams.
signups_stream = Realm.INITIAL_PRIVATE_STREAM_NAME
stream_list = [
"Verona",
"Denmark",
"Scotland",
"Venice",
"Rome",
signups_stream,
]
stream_dict: Dict[str, Dict[str, Any]] = {
"Denmark": {
"description": "A Scandinavian country"
},
"Scotland": {
"description": "Located in the United Kingdom"
},
"Venice": {
"description": "A northeastern Italian city"
},
"Rome": {
"description": "Yet another Italian city",
"is_web_public": True
},
}
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams: List[int] = [
Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list
]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users). For the test suite, we have a fixed list of
# subscriptions to make sure test data is consistent
# across platforms.
subscriptions_list: List[Tuple[UserProfile, Recipient]] = []
profiles: Sequence[UserProfile] = (
UserProfile.objects.select_related().filter(
is_bot=False).order_by("email"))
if options["test_suite"]:
subscriptions_map = {
"*****@*****.**": ["Verona"],
"*****@*****.**": ["Verona"],
"*****@*****.**": ["Verona", "Denmark", signups_stream],
"*****@*****.**": [
"Verona",
"Denmark",
"Scotland",
signups_stream,
],
"*****@*****.**": ["Verona", "Denmark", "Scotland"],
"*****@*****.**":
["Verona", "Denmark", "Scotland", "Venice"],
"*****@*****.**":
["Verona", "Denmark", "Scotland", "Venice", "Rome"],
"*****@*****.**": ["Verona"],
"*****@*****.**": [
"Verona",
"Denmark",
"Venice",
signups_stream,
],
"*****@*****.**": ["Verona", "Denmark", "Scotland"],
}
for profile in profiles:
email = profile.delivery_email
if email not in subscriptions_map:
raise Exception(
f"Subscriptions not listed for user {email}")
for stream_name in subscriptions_map[email]:
stream = Stream.objects.get(name=stream_name,
realm=zulip_realm)
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=stream.id)
subscriptions_list.append((profile, r))
else:
num_streams = len(recipient_streams)
num_users = len(profiles)
for i, profile in enumerate(profiles):
# Subscribe to some streams.
fraction = float(i) / num_users
num_recips = int(num_streams * fraction) + 1
for type_id in recipient_streams[:num_recips]:
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=type_id)
subscriptions_list.append((profile, r))
subscriptions_to_add: List[Subscription] = []
event_time = timezone_now()
all_subscription_logs: (List[RealmAuditLog]) = []
i = 0
for profile, recipient in subscriptions_list:
i += 1
color = STREAM_ASSIGNMENT_COLORS[i %
len(STREAM_ASSIGNMENT_COLORS)]
s = Subscription(
recipient=recipient,
user_profile=profile,
is_user_active=profile.is_active,
color=color,
)
subscriptions_to_add.append(s)
log = RealmAuditLog(
realm=profile.realm,
modified_user=profile,
modified_stream_id=recipient.type_id,
event_last_message_id=0,
event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
event_time=event_time,
)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(
zulip_realm,
"Phone number",
CustomProfileField.SHORT_TEXT,
hint="")
biography = try_add_realm_custom_profile_field(
zulip_realm,
"Biography",
CustomProfileField.LONG_TEXT,
hint="What are you known for?",
)
favorite_food = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite food",
CustomProfileField.SHORT_TEXT,
hint="Or drink, if you'd prefer",
)
field_data: ProfileFieldData = {
"vim": {
"text": "Vim",
"order": "1"
},
"emacs": {
"text": "Emacs",
"order": "2"
},
}
favorite_editor = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite editor",
CustomProfileField.SELECT,
field_data=field_data)
birthday = try_add_realm_custom_profile_field(
zulip_realm, "Birthday", CustomProfileField.DATE)
favorite_website = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite website",
CustomProfileField.URL,
hint="Or your personal blog's URL",
)
mentor = try_add_realm_custom_profile_field(
zulip_realm, "Mentor", CustomProfileField.USER)
github_profile = try_add_realm_default_custom_profile_field(
zulip_realm, "github")
# Fill in values for Iago and Hamlet
hamlet = get_user_by_delivery_email("*****@*****.**",
zulip_realm)
do_update_user_custom_profile_data_if_changed(
iago,
[
{
"id": phone_number.id,
"value": "+1-234-567-8901"
},
{
"id": biography.id,
"value": "Betrayer of Othello."
},
{
"id": favorite_food.id,
"value": "Apples"
},
{
"id": favorite_editor.id,
"value": "emacs"
},
{
"id": birthday.id,
"value": "2000-01-01"
},
{
"id": favorite_website.id,
"value": "https://zulip.readthedocs.io/en/latest/"
},
{
"id": mentor.id,
"value": [hamlet.id]
},
{
"id": github_profile.id,
"value": "zulip"
},
],
)
do_update_user_custom_profile_data_if_changed(
hamlet,
[
{
"id": phone_number.id,
"value": "+0-11-23-456-7890"
},
{
"id":
biography.id,
"value":
"I am:\n* The prince of Denmark\n* Nephew to the usurping Claudius",
},
{
"id": favorite_food.id,
"value": "Dark chocolate"
},
{
"id": favorite_editor.id,
"value": "vim"
},
{
"id": birthday.id,
"value": "1900-01-01"
},
{
"id": favorite_website.id,
"value": "https://blog.zulig.org"
},
{
"id": mentor.id,
"value": [iago.id]
},
{
"id": github_profile.id,
"value": "zulipbot"
},
],
)
else:
zulip_realm = get_realm("zulip")
recipient_streams = [
klass.type_id
for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
# Extract a list of all users
user_profiles: List[UserProfile] = list(
UserProfile.objects.filter(is_bot=False))
# Create a test realm emoji.
IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
with open(IMAGE_FILE_PATH, "rb") as fp:
check_add_realm_emoji(zulip_realm, "green_tick", iago, fp)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status: int = UserPresence.ACTIVE
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= "H":
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(
user_profile=user,
realm_id=user.realm_id,
client=client,
timestamp=date,
status=status,
)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [
random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])
]
create_alert_words(zulip_realm.id)
# Generate a new set of test data.
create_test_data()
# prepopulate the URL preview/embed data for the links present
# in the config.generate_data.json data set. This makes it
# possible for populate_db to run happily without Internet
# access.
with open("zerver/tests/fixtures/docs_url_preview_data.json",
"rb") as f:
urls_with_preview_data = orjson.loads(f.read())
for url in urls_with_preview_data:
cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)
if options["delete"]:
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)",
"*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm,
testsuite_mit_users,
tos_version=settings.TOS_VERSION)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia, Lear's daughter", "*****@*****.**"),
]
create_users(lear_realm,
testsuite_lear_users,
tos_version=settings.TOS_VERSION)
if not options["test_suite"]:
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
# to imitate emoji insertions in stream names
raw_emojis = ["😎", "😂", "🐱👤"]
zulip_stream_dict: Dict[str, Dict[str, Any]] = {
"devel": {
"description": "For developing"
},
# ビデオゲーム - VideoGames (japanese)
"ビデオゲーム": {
"description":
"Share your favorite video games! {}".format(
raw_emojis[2])
},
"announce": {
"description": "For announcements",
"stream_post_policy": Stream.STREAM_POST_POLICY_ADMINS,
},
"design": {
"description": "For design"
},
"support": {
"description": "For support"
},
"social": {
"description": "For socializing"
},
"test": {
"description": "For testing `code`"
},
"errors": {
"description": "For errors"
},
# 조리법 - Recipes (Korean) , Пельмени - Dumplings (Russian)
"조리법 " + raw_emojis[0]: {
"description":
"Everything cooking, from pasta to Пельмени"
},
}
extra_stream_names = [
"802.11a",
"Ad Hoc Network",
"Augmented Reality",
"Cycling",
"DPI",
"FAQ",
"FiFo",
"commits",
"Control panel",
"desktop",
"компьютеры",
"Data security",
"desktop",
"काम",
"discussions",
"Cloud storage",
"GCI",
"Vaporware",
"Recent Trends",
"issues",
"live",
"Health",
"mobile",
"空間",
"provision",
"hidrógeno",
"HR",
"アニメ",
]
# Add stream names and stream descriptions
for i in range(options["extra_streams"]):
extra_stream_name = random.choice(
extra_stream_names) + " " + str(i)
# to imitate emoji insertions in stream names
if random.random() <= 0.15:
extra_stream_name += random.choice(raw_emojis)
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream(
"announce", zulip_realm)
zulip_realm.save(update_fields=["notifications_stream"])
# Add a few default streams
for default_stream_name in [
"design", "devel", "social", "support"
]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(
default_stream_name,
zulip_realm))
# Now subscribe everyone to these streams
subscribe_users_to_streams(zulip_realm, zulip_stream_dict)
create_user_groups()
if not options["test_suite"]:
# We populate the analytics database here for
# development purpose only
call_command("populate_analytics_db")
threads = options["threads"]
jobs: List[Tuple[int, List[List[int]], Dict[str, Any],
Callable[[str], int], int]] = []
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write,
random.randint(0, 10**10)))
for job in jobs:
generate_and_send_messages(job)
if options["delete"]:
if not options["test_suite"]:
# These bots are not needed by the test suite
# Also, we don't want interacting with each other
# in dev setup.
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm,
internal_zulip_users_nosubs,
bot_type=UserProfile.DEFAULT_BOT)
mark_all_messages_as_read()
self.stdout.write("Successfully populated test database.\n")
def generate_and_send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open(
os.path.join(get_or_create_dev_uuid_var_path("test-backend"),
"test_messages.json"), "rb") as infile:
dialog = orjson.loads(infile.read())
random.shuffle(dialog)
texts = itertools.cycle(dialog)
# We need to filter out streams from the analytics realm as we don't want to generate
# messages to its streams - and they might also have no subscribers, which would break
# our message generation mechanism below.
stream_ids = Stream.objects.filter(realm=get_realm("zulip")).values_list(
"id", flat=True)
recipient_streams: List[int] = [
recipient.id
for recipient in Recipient.objects.filter(type=Recipient.STREAM,
type_id__in=stream_ids)
]
recipient_huddles: List[int] = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
]
huddle_members: Dict[int, List[int]] = {}
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
# Generate different topics for each stream
possible_topics = {}
for stream_id in recipient_streams:
possible_topics[stream_id] = generate_topics(options["max_topics"])
message_batch_size = options["batch_size"]
num_messages = 0
random_max = 1000000
recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {}
messages: List[Message] = []
while num_messages < tot_messages:
saved_data: Dict[str, Any] = {}
message = Message()
message.sending_client = get_client("populate_db")
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100.0 / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data["personals_pair"]
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data["subject"]
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif randkey <= random_max * options["percent_huddles"] / 100.0:
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.0):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif randkey <= random_max * 1.0:
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data["personals_pair"] = personals_pair
elif message_type == Recipient.STREAM:
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = random.choice(
possible_topics[message.recipient.id])
saved_data["subject"] = message.subject
message.date_sent = choose_date_sent(num_messages, tot_messages,
options["threads"])
messages.append(message)
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
if (num_messages % message_batch_size) == 0:
# Send the batch and empty the list:
send_messages(messages)
messages = []
if len(messages) > 0:
# If there are unsent messages after exiting the loop, send them:
send_messages(messages)
return tot_messages
def api_github_landing(
request: HttpRequest,
user_profile: UserProfile,
event: str = REQ(),
payload: Mapping[str, Any] = REQ(validator=check_dict([])),
branches: str = REQ(default=''),
stream: str = REQ(default=''),
version: int = REQ(converter=to_non_negative_int, default=1),
commit_stream: str = REQ(default=''),
issue_stream: str = REQ(default=''),
exclude_pull_requests: bool = REQ(converter=flexible_boolean,
default=False),
exclude_issues: bool = REQ(converter=flexible_boolean, default=False),
exclude_commits: bool = REQ(converter=flexible_boolean, default=False),
emphasize_branch_in_topic: bool = REQ(converter=flexible_boolean,
default=False),
) -> HttpResponse:
repository = payload['repository']
# Special hook for capturing event data. If we see our special test repo, log the payload from github.
try:
if is_test_repository(repository) and settings.PRODUCTION:
with open('/var/log/zulip/github-payloads', 'a') as f:
f.write(
ujson.dumps({
'event':
event,
'payload':
payload,
'branches':
branches,
'stream':
stream,
'version':
version,
'commit_stream':
commit_stream,
'issue_stream':
issue_stream,
'exclude_pull_requests':
exclude_pull_requests,
'exclude_issues':
exclude_issues,
'exclude_commits':
exclude_commits,
'emphasize_branch_in_topic':
emphasize_branch_in_topic,
}))
f.write('\n')
except Exception:
logging.exception('Error while capturing Github event')
if not stream:
stream = 'commits'
short_ref = re.sub(r'^refs/heads/', '', payload.get('ref', ''))
kwargs = dict()
if emphasize_branch_in_topic and short_ref:
kwargs['topic_focus'] = short_ref
allowed_events = set()
if not exclude_pull_requests:
allowed_events.add('pull_request')
if not exclude_issues:
allowed_events.add('issues')
allowed_events.add('issue_comment')
if not exclude_commits:
allowed_events.add('push')
allowed_events.add('commit_comment')
if event not in allowed_events:
return json_success()
# We filter issue_comment events for issue creation events
if event == 'issue_comment' and payload['action'] != 'created':
return json_success()
if event == 'push':
# If we are given a whitelist of branches, then we silently ignore
# any push notification on a branch that is not in our whitelist.
if branches and short_ref not in re.split(r'[\s,;|]+', branches):
return json_success()
# Map payload to the handler with the right version
if version == 2:
target_stream, subject, content = api_github_v2(
user_profile, event, payload, branches, stream, commit_stream,
issue_stream, **kwargs)
else:
target_stream, subject, content = api_github_v1(
user_profile, event, payload, branches, stream, **kwargs)
request.client = get_client('ZulipGitHubLegacyWebhook')
return send_message_backend(request,
user_profile,
message_type_name='stream',
message_to=[target_stream],
forged=False,
topic_name=subject,
message_content=content)
def api_teamcity_webhook(request, user_profile,
stream=REQ(default='teamcity')):
try:
payload = ujson.loads(request.body)
except ValueError:
return json_error("Malformed JSON input")
message = payload['build']
build_name = message['buildFullName']
build_url = message['buildStatusUrl']
changes_url = build_url + '&tab=buildChangesDiv'
build_number = message['buildNumber']
build_result = message['buildResult']
build_result_delta = message['buildResultDelta']
build_status = message['buildStatus']
if build_result == 'success':
if build_result_delta == 'fixed':
status = 'has been fixed! :thumbsup:'
else:
status = 'was successful! :thumbsup:'
elif build_result == 'failure':
if build_result_delta == 'broken':
status = 'is broken with status %s! :thumbsdown:' % (build_status)
else:
status = 'is still broken with status %s! :thumbsdown:' % (
build_status)
elif build_result == 'running':
status = 'has started.'
else:
status = '(has no message specified for status %s)' % (build_status)
template = (u'%s build %s %s\n' u'Details: [changes](%s), [build log](%s)')
body = template % (build_name, build_number, status, changes_url,
build_url)
topic = build_name
# Check if this is a personal build, and if so try to private message the user who triggered it.
if get_teamcity_property_value(message['teamcityProperties'],
'env.BUILD_IS_PERSONAL') == 'true':
# The triggeredBy field gives us the teamcity user full name, and the "teamcity.build.triggeredBy.username"
# property gives us the teamcity username. Let's try finding the user email from both.
teamcity_fullname = message['triggeredBy'].split(';')[0]
teamcity_user = guess_zulip_user_from_teamcity(teamcity_fullname,
user_profile.realm)
if teamcity_user is None:
teamcity_shortname = get_teamcity_property_value(
message['teamcityProperties'],
'teamcity.build.triggeredBy.username')
if teamcity_shortname is not None:
teamcity_user = guess_zulip_user_from_teamcity(
teamcity_shortname, user_profile.realm)
if teamcity_user is None:
# We can't figure out who started this build - there's nothing we can do here.
logging.info(
"Teamcity webhook couldn't find a matching Zulip user for Teamcity user '%s' or '%s'"
% (teamcity_fullname, teamcity_shortname))
return json_success()
body = "Your personal build of " + body
check_send_message(user_profile, get_client('ZulipTeamcityWebhook'),
'private', [teamcity_user.email], topic, body)
return json_success()
check_send_message(user_profile, get_client('ZulipTeamcityWebhook'),
'stream', [stream], topic, body)
return json_success()
def test_basics(self) -> None:
cordelia = self.example_user('cordelia')
hamlet = self.example_user('hamlet')
king_lear = self.lear_user('king')
realm_id = hamlet.realm_id
away_user_ids = get_away_user_ids(realm_id=realm_id)
self.assertEqual(away_user_ids, set())
client1 = get_client('web')
client2 = get_client('ZT')
set_away_status(
user_profile_id=hamlet.id,
client_id=client1.id,
)
away_user_ids = get_away_user_ids(realm_id=realm_id)
self.assertEqual(away_user_ids, {hamlet.id})
# Test that second client just updates
# the record. We only store one record
# per user. The user's status transcends
# clients; we only store the client for
# reference and to maybe reconcile timeout
# situations.
set_away_status(
user_profile_id=hamlet.id,
client_id=client2.id,
)
away_user_ids = get_away_user_ids(realm_id=realm_id)
self.assertEqual(away_user_ids, {hamlet.id})
rec_count = UserStatus.objects.filter(
user_profile_id=hamlet.id).count()
self.assertEqual(rec_count, 1)
revoke_away_status(user_profile_id=hamlet.id, )
away_user_ids = get_away_user_ids(realm_id=realm_id)
self.assertEqual(away_user_ids, set())
# Now set away status for three different users across
# two realms.
set_away_status(
user_profile_id=hamlet.id,
client_id=client1.id,
)
set_away_status(
user_profile_id=cordelia.id,
client_id=client2.id,
)
set_away_status(
user_profile_id=king_lear.id,
client_id=client2.id,
)
away_user_ids = get_away_user_ids(realm_id=realm_id)
self.assertEqual(away_user_ids, {cordelia.id, hamlet.id})
away_user_ids = get_away_user_ids(realm_id=king_lear.realm.id)
self.assertEqual(away_user_ids, {king_lear.id})
# Revoke Hamlet again.
revoke_away_status(user_profile_id=hamlet.id, )
away_user_ids = get_away_user_ids(realm_id=realm_id)
self.assertEqual(away_user_ids, {cordelia.id})
def test_get_events(self) -> None:
user_profile = self.example_user("hamlet")
email = user_profile.email
recipient_user_profile = self.example_user("othello")
recipient_email = recipient_user_profile.email
self.login_user(user_profile)
result = self.tornado_call(
get_events,
user_profile,
{
"apply_markdown": orjson.dumps(True).decode(),
"client_gravatar": orjson.dumps(True).decode(),
"event_types": orjson.dumps(["message"]).decode(),
"user_client": "website",
"dont_block": orjson.dumps(True).decode(),
},
)
self.assert_json_success(result)
queue_id = orjson.loads(result.content)["queue_id"]
recipient_result = self.tornado_call(
get_events,
recipient_user_profile,
{
"apply_markdown": orjson.dumps(True).decode(),
"client_gravatar": orjson.dumps(True).decode(),
"event_types": orjson.dumps(["message"]).decode(),
"user_client": "website",
"dont_block": orjson.dumps(True).decode(),
},
)
self.assert_json_success(recipient_result)
recipient_queue_id = orjson.loads(recipient_result.content)["queue_id"]
result = self.tornado_call(
get_events,
user_profile,
{
"queue_id": queue_id,
"user_client": "website",
"last_event_id": -1,
"dont_block": orjson.dumps(True).decode(),
},
)
events = orjson.loads(result.content)["events"]
self.assert_json_success(result)
self.assert_length(events, 0)
local_id = "10.01"
check_send_message(
sender=user_profile,
client=get_client("whatever"),
message_type_name="private",
message_to=[recipient_email],
topic_name=None,
message_content="hello",
local_id=local_id,
sender_queue_id=queue_id,
)
result = self.tornado_call(
get_events,
user_profile,
{
"queue_id": queue_id,
"user_client": "website",
"last_event_id": -1,
"dont_block": orjson.dumps(True).decode(),
},
)
events = orjson.loads(result.content)["events"]
self.assert_json_success(result)
self.assert_length(events, 1)
self.assertEqual(events[0]["type"], "message")
self.assertEqual(events[0]["message"]["sender_email"], email)
self.assertEqual(events[0]["local_message_id"], local_id)
self.assertEqual(
events[0]["message"]["display_recipient"][0]["is_mirror_dummy"],
False)
self.assertEqual(
events[0]["message"]["display_recipient"][1]["is_mirror_dummy"],
False)
last_event_id = events[0]["id"]
local_id = "10.02"
check_send_message(
sender=user_profile,
client=get_client("whatever"),
message_type_name="private",
message_to=[recipient_email],
topic_name=None,
message_content="hello",
local_id=local_id,
sender_queue_id=queue_id,
)
result = self.tornado_call(
get_events,
user_profile,
{
"queue_id": queue_id,
"user_client": "website",
"last_event_id": last_event_id,
"dont_block": orjson.dumps(True).decode(),
},
)
events = orjson.loads(result.content)["events"]
self.assert_json_success(result)
self.assert_length(events, 1)
self.assertEqual(events[0]["type"], "message")
self.assertEqual(events[0]["message"]["sender_email"], email)
self.assertEqual(events[0]["local_message_id"], local_id)
# Test that the received message in the receiver's event queue
# exists and does not contain a local id
recipient_result = self.tornado_call(
get_events,
recipient_user_profile,
{
"queue_id": recipient_queue_id,
"user_client": "website",
"last_event_id": -1,
"dont_block": orjson.dumps(True).decode(),
},
)
recipient_events = orjson.loads(recipient_result.content)["events"]
self.assert_json_success(recipient_result)
self.assert_length(recipient_events, 2)
self.assertEqual(recipient_events[0]["type"], "message")
self.assertEqual(recipient_events[0]["message"]["sender_email"], email)
self.assertTrue("local_message_id" not in recipient_events[0])
self.assertEqual(recipient_events[1]["type"], "message")
self.assertEqual(recipient_events[1]["message"]["sender_email"], email)
self.assertTrue("local_message_id" not in recipient_events[1])
def get_events_backend(
request: HttpRequest,
user_profile: UserProfile,
# user_client is intended only for internal Django=>Tornado requests
# and thus shouldn't be documented for external use.
user_client: Optional[Client] = REQ(
converter=lambda var_name, s: get_client(s), default=None, intentionally_undocumented=True
),
last_event_id: Optional[int] = REQ(json_validator=check_int, default=None),
queue_id: Optional[str] = REQ(default=None),
# apply_markdown, client_gravatar, all_public_streams, and various
# other parameters are only used when registering a new queue via this
# endpoint. This is a feature used primarily by get_events_internal
# and not expected to be used by third-party clients.
apply_markdown: bool = REQ(
default=False, json_validator=check_bool, intentionally_undocumented=True
),
client_gravatar: bool = REQ(
default=False, json_validator=check_bool, intentionally_undocumented=True
),
slim_presence: bool = REQ(
default=False, json_validator=check_bool, intentionally_undocumented=True
),
all_public_streams: bool = REQ(
default=False, json_validator=check_bool, intentionally_undocumented=True
),
event_types: Optional[Sequence[str]] = REQ(
default=None, json_validator=check_list(check_string), intentionally_undocumented=True
),
dont_block: bool = REQ(default=False, json_validator=check_bool),
narrow: Sequence[Sequence[str]] = REQ(
default=[],
json_validator=check_list(check_list(check_string)),
intentionally_undocumented=True,
),
lifespan_secs: int = REQ(
default=0, converter=to_non_negative_int, intentionally_undocumented=True
),
bulk_message_deletion: bool = REQ(
default=False, json_validator=check_bool, intentionally_undocumented=True
),
stream_typing_notifications: bool = REQ(
default=False, json_validator=check_bool, intentionally_undocumented=True
),
user_settings_object: bool = REQ(
default=False, json_validator=check_bool, intentionally_undocumented=True
),
) -> HttpResponse:
if all_public_streams and not user_profile.can_access_public_streams():
raise JsonableError(_("User not authorized for this query"))
# Extract the Tornado handler from the request
tornado_handler = RequestNotes.get_notes(request).tornado_handler
assert tornado_handler is not None
handler = tornado_handler()
assert handler is not None
if user_client is None:
valid_user_client = RequestNotes.get_notes(request).client
assert valid_user_client is not None
else:
valid_user_client = user_client
events_query = dict(
user_profile_id=user_profile.id,
queue_id=queue_id,
last_event_id=last_event_id,
event_types=event_types,
client_type_name=valid_user_client.name,
all_public_streams=all_public_streams,
lifespan_secs=lifespan_secs,
narrow=narrow,
dont_block=dont_block,
handler_id=handler.handler_id,
)
if queue_id is None:
events_query["new_queue_data"] = dict(
user_profile_id=user_profile.id,
realm_id=user_profile.realm_id,
event_types=event_types,
client_type_name=valid_user_client.name,
apply_markdown=apply_markdown,
client_gravatar=client_gravatar,
slim_presence=slim_presence,
all_public_streams=all_public_streams,
queue_timeout=lifespan_secs,
last_connection_time=time.time(),
narrow=narrow,
bulk_message_deletion=bulk_message_deletion,
stream_typing_notifications=stream_typing_notifications,
user_settings_object=user_settings_object,
)
result = in_tornado_thread(lambda: fetch_events(events_query))
if "extra_log_data" in result:
log_data = RequestNotes.get_notes(request).log_data
assert log_data is not None
log_data["extra"] = result["extra_log_data"]
if result["type"] == "async":
# Mark this response with .asynchronous; this will result in
# Tornado discarding the response and instead long-polling the
# request. See zulip_finish for more design details.
handler._request = request
response = json_success(request)
response.asynchronous = True
return response
if result["type"] == "error":
raise result["exception"]
return json_success(request, data=result["response"])
def generate_and_send_messages(data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open(os.path.join(get_or_create_dev_uuid_var_path('test-backend'),
"test_messages.json")) as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams: List[int] = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
recipient_huddles: List[int] = [h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)]
huddle_members: Dict[int, List[int]] = {}
for h in recipient_huddles:
huddle_members[h] = [s.user_profile.id for s in
Subscription.objects.filter(recipient_id=h)]
# Generate different topics for each stream
possible_topics = dict()
for stream_id in recipient_streams:
possible_topics[stream_id] = generate_topics(options["max_topics"])
message_batch_size = options['batch_size']
num_messages = 0
random_max = 1000000
recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {}
messages = []
messages_add_reaction = []
while num_messages < tot_messages:
saved_data: Dict[str, Any] = {}
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0 and
random.randint(1, random_max) * 100. / random_max < options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages - 1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(random.choice(recipient_huddles))
elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(type=Recipient.PERSONAL,
type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
# Pick a random subscriber to the stream
message.sender = random.choice(Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = random.choice(possible_topics[message.recipient.id])
saved_data['subject'] = message.subject
message.date_sent = choose_date_sent(num_messages, tot_messages, options['threads'])
messages.append(message)
messages_add_reaction.append(message)
recipients[num_messages] = (message_type, message.recipient.id, saved_data)
num_messages += 1
if (num_messages % message_batch_size) == 0:
# Send the batch and empty the list:
send_messages(messages)
messages = []
if len(messages) > 0:
# If there are unsent messages after exiting the loop, send them:
send_messages(messages)
reactions_message = []
add_emojis = ["1f44d", "1f642", "1f60a"]
for rmessage in messages_add_reaction:
if random.random() > 0.9:
reactedmessage = Reaction(user_profile=rmessage.sender,
message=rmessage,
emoji_name="+1",
emoji_code=add_emojis[random.randint(0, 2)],
reaction_type="unicode_emoji")
reactions_message.append(reactedmessage)
Reaction.objects.bulk_create(reactions_message)
return tot_messages
def generate_and_send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open(
os.path.join(get_or_create_dev_uuid_var_path('test-backend'),
"test_messages.json"), "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
] # type: List[int]
recipient_huddles = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
message_batch_size = options['batch_size']
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
messages = []
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100. / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + str(random.randint(1, 3))
saved_data['subject'] = message.subject
message.date_sent = choose_date_sent(num_messages, tot_messages,
options['threads'])
messages.append(message)
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
if (num_messages % message_batch_size) == 0:
# Send the batch and empty the list:
send_messages(messages)
messages = []
if len(messages) > 0:
# If there are unsent messages after exiting the loop, send them:
send_messages(messages)
return tot_messages
def consume(self, event: Mapping[str, Any]) -> None:
user_profile = get_user_profile_by_id(event["user_profile_id"])
client = get_client(event["client"])
log_time = timestamp_to_datetime(event["time"])
query = event["query"]
do_update_user_activity(user_profile, client, query, log_time)
def get_client(self) -> Client:
"""Returns a Zulip Client object to be used for things done in management commands"""
return get_client("ZulipServer")
def send_messages(data):
# type: (Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int]) -> int
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open("var/test_messages.json", "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
] # type: List[int]
recipient_huddles = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100. / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + Text(random.randint(1, 3))
saved_data['subject'] = message.subject
message.pub_date = timezone_now()
do_send_messages([{'message': message}])
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
return tot_messages
def test_UserActivityWorker(self) -> None:
fake_client = self.FakeClient()
user = self.example_user('hamlet')
UserActivity.objects.filter(
user_profile=user.id,
client=get_client('ios'),
).delete()
data = dict(
user_profile_id=user.id,
client_id=get_client('ios').id,
time=time.time(),
query='send_message',
)
fake_client.queue.append(('user_activity', data))
# The block below adds an event using the old format,
# having the client name instead of id, to test the queue
# worker handles it correctly. That compatibility code can
# be deleted in a later release, and this test should then be cleaned up.
data_old_format = dict(
user_profile_id=user.id,
client='ios',
time=time.time(),
query='send_message',
)
fake_client.queue.append(('user_activity', data_old_format))
with loopworker_sleep_mock:
with simulated_queue_client(lambda: fake_client):
worker = queue_processors.UserActivityWorker()
worker.setup()
try:
worker.start()
except AbortLoop:
pass
activity_records = UserActivity.objects.filter(
user_profile=user.id,
client=get_client('ios'),
)
self.assertEqual(len(activity_records), 1)
self.assertEqual(activity_records[0].count, 2)
# Now process the event a second time and confirm count goes
# up. Ideally, we'd use an event with a slightly newer
# time, but it's not really important.
fake_client.queue.append(('user_activity', data))
with loopworker_sleep_mock:
with simulated_queue_client(lambda: fake_client):
worker = queue_processors.UserActivityWorker()
worker.setup()
try:
worker.start()
except AbortLoop:
pass
activity_records = UserActivity.objects.filter(
user_profile=user.id,
client=get_client('ios'),
)
self.assertEqual(len(activity_records), 1)
self.assertEqual(activity_records[0].count, 3)
def api_jira_webhook(request, user_profile):
try:
payload = ujson.loads(request.body)
except ValueError:
return json_error("Malformed JSON input")
try:
stream = request.GET['stream']
except (AttributeError, KeyError):
stream = 'jira'
def get_in(payload, keys, default=''):
try:
for key in keys:
payload = payload[key]
except (AttributeError, KeyError, TypeError):
return default
return payload
event = payload.get('webhookEvent')
author = get_in(payload, ['user', 'displayName'])
issueId = get_in(payload, ['issue', 'key'])
# Guess the URL as it is not specified in the payload
# We assume that there is a /browse/BUG-### page
# from the REST url of the issue itself
baseUrl = re.match("(.*)\/rest\/api/.*", get_in(payload,
['issue', 'self']))
if baseUrl and len(baseUrl.groups()):
issue = "[%s](%s/browse/%s)" % (issueId, baseUrl.group(1), issueId)
else:
issue = issueId
title = get_in(payload, ['issue', 'fields', 'summary'])
priority = get_in(payload, ['issue', 'fields', 'priority', 'name'])
assignee = get_in(payload, ['issue', 'fields', 'assignee', 'displayName'],
'no one')
assignee_email = get_in(payload,
['issue', 'fields', 'assignee', 'emailAddress'],
'')
assignee_mention = ''
if assignee_email != '':
try:
assignee_profile = get_user_profile_by_email(assignee_email)
assignee_mention = "@**%s**" % (assignee_profile.full_name, )
except UserProfile.DoesNotExist:
assignee_mention = "**%s**" % (assignee_email, )
subject = "%s: %s" % (issueId, title)
if event == 'jira:issue_created':
content = "%s **created** %s priority %s, assigned to @**%s**:\n\n> %s" % \
(author, issue, priority, assignee, title)
elif event == 'jira:issue_deleted':
content = "%s **deleted** %s!" % \
(author, issue)
elif event == 'jira:issue_updated':
# Reassigned, commented, reopened, and resolved events are all bundled
# into this one 'updated' event type, so we try to extract the meaningful
# event that happened
if assignee_mention != '':
assignee_blurb = " (assigned to %s)" % (assignee_mention, )
else:
assignee_blurb = ''
content = "%s **updated** %s%s:\n\n" % (author, issue, assignee_blurb)
changelog = get_in(payload, [
'changelog',
])
comment = get_in(payload, ['comment', 'body'])
if changelog != '':
# Use the changelog to display the changes, whitelist types we accept
items = changelog.get('items')
for item in items:
field = item.get('field')
# Convert a user's target to a @-mention if possible
targetFieldString = "**%s**" % (item.get('toString'), )
if field == 'assignee' and assignee_mention != '':
targetFieldString = assignee_mention
fromFieldString = item.get('fromString')
if targetFieldString or fromFieldString:
content += "* Changed %s from **%s** to %s\n" % (
field, fromFieldString, targetFieldString)
if comment != '':
comment = convert_jira_markup(comment, user_profile.realm)
content += "\n%s\n" % (comment, )
elif event in ['jira:worklog_updated']:
# We ignore these event types
return json_success()
elif 'transition' in payload:
from_status = get_in(payload, ['transition', 'from_status'])
to_status = get_in(payload, ['transition', 'to_status'])
content = "%s **transitioned** %s from %s to %s" % (
author, issue, from_status, to_status)
else:
# Unknown event type
if not settings.TEST_SUITE:
if event is None:
logging.warning("Got JIRA event with None event type: %s" %
(payload, ))
else:
logging.warning("Got JIRA event type we don't understand: %s" %
(event, ))
return json_error("Unknown JIRA event type")
check_send_message(user_profile, get_client("ZulipJIRAWebhook"), "stream",
[stream], subject, content)
return json_success()
def home_real(request):
# type: (HttpRequest) -> HttpResponse
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
user_profile = request.user
request._email = request.user.email
request.client = get_client("website")
# If a user hasn't signed the current Terms of Service, send them there
if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \
int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version():
return accounts_accept_terms(request)
narrow = [] # type: List[List[text_type]]
narrow_stream = None
narrow_topic = request.GET.get("topic")
if request.GET.get("stream"):
try:
narrow_stream = get_stream(request.GET.get("stream"),
user_profile.realm)
assert (narrow_stream is not None)
assert (narrow_stream.is_public())
narrow = [["stream", narrow_stream.name]]
except Exception:
logging.exception("Narrow parsing")
if narrow_topic is not None:
narrow.append(["topic", narrow_topic])
register_ret = do_events_register(user_profile,
request.client,
apply_markdown=True,
narrow=narrow)
user_has_messages = (register_ret['max_message_id'] != -1)
# Reset our don't-spam-users-with-email counter since the
# user has since logged in
if not user_profile.last_reminder is None:
user_profile.last_reminder = None
user_profile.save(update_fields=["last_reminder"])
# Brand new users get the tutorial
needs_tutorial = settings.TUTORIAL_ENABLED and \
user_profile.tutorial_status != UserProfile.TUTORIAL_FINISHED
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = first_in_realm and \
not PreregistrationUser.objects.filter(referred_by=user_profile).count()
if user_profile.pointer == -1 and user_has_messages:
# Put the new user's pointer at the bottom
#
# This improves performance, because we limit backfilling of messages
# before the pointer. It's also likely that someone joining an
# organization is interested in recent messages more than the very
# first messages on the system.
register_ret['pointer'] = register_ret['max_message_id']
user_profile.last_pointer_updater = request.session.session_key
if user_profile.pointer == -1:
latest_read = None
else:
try:
latest_read = UserMessage.objects.get(
user_profile=user_profile, message__id=user_profile.pointer)
except UserMessage.DoesNotExist:
# Don't completely fail if your saved pointer ID is invalid
logging.warning("%s has invalid pointer %s" %
(user_profile.email, user_profile.pointer))
latest_read = None
desktop_notifications_enabled = user_profile.enable_desktop_notifications
if narrow_stream is not None:
desktop_notifications_enabled = False
if user_profile.realm.notifications_stream:
notifications_stream = user_profile.realm.notifications_stream.name
else:
notifications_stream = ""
# Set default language and make it persist
default_language = register_ret['default_language']
url_lang = '/{}'.format(request.LANGUAGE_CODE)
if not request.path.startswith(url_lang):
translation.activate(default_language)
request.session[translation.LANGUAGE_SESSION_KEY] = default_language
# Pass parameters to the client-side JavaScript code.
# These end up in a global JavaScript Object named 'page_params'.
page_params = dict(
zulip_version=ZULIP_VERSION,
share_the_love=settings.SHARE_THE_LOVE,
development_environment=settings.DEVELOPMENT,
debug_mode=settings.DEBUG,
test_suite=settings.TEST_SUITE,
poll_timeout=settings.POLL_TIMEOUT,
login_page=settings.HOME_NOT_LOGGED_IN,
server_uri=settings.SERVER_URI,
realm_uri=user_profile.realm.uri,
maxfilesize=settings.MAX_FILE_UPLOAD_SIZE,
server_generation=settings.SERVER_GENERATION,
password_auth_enabled=password_auth_enabled(user_profile.realm),
have_initial_messages=user_has_messages,
subbed_info=register_ret['subscriptions'],
unsubbed_info=register_ret['unsubscribed'],
neversubbed_info=register_ret['never_subscribed'],
people_list=register_ret['realm_users'],
bot_list=register_ret['realm_bots'],
initial_pointer=register_ret['pointer'],
initial_presences=register_ret['presences'],
initial_servertime=time.time(
), # Used for calculating relative presence age
fullname=user_profile.full_name,
email=user_profile.email,
domain=user_profile.realm.domain,
domains=list_of_domains_for_realm(user_profile.realm),
realm_name=register_ret['realm_name'],
realm_invite_required=register_ret['realm_invite_required'],
realm_invite_by_admins_only=register_ret[
'realm_invite_by_admins_only'],
realm_authentication_methods=register_ret[
'realm_authentication_methods'],
realm_create_stream_by_admins_only=register_ret[
'realm_create_stream_by_admins_only'],
realm_allow_message_editing=register_ret[
'realm_allow_message_editing'],
realm_message_content_edit_limit_seconds=register_ret[
'realm_message_content_edit_limit_seconds'],
realm_restricted_to_domain=register_ret['realm_restricted_to_domain'],
realm_default_language=register_ret['realm_default_language'],
enter_sends=user_profile.enter_sends,
user_id=user_profile.id,
left_side_userlist=register_ret['left_side_userlist'],
default_language=register_ret['default_language'],
default_language_name=get_language_name(
register_ret['default_language']),
language_list_dbl_col=get_language_list_for_templates(
register_ret['default_language']),
language_list=get_language_list(),
referrals=register_ret['referrals'],
realm_emoji=register_ret['realm_emoji'],
needs_tutorial=needs_tutorial,
first_in_realm=first_in_realm,
prompt_for_invites=prompt_for_invites,
notifications_stream=notifications_stream,
cross_realm_bots=list(get_cross_realm_dicts()),
use_websockets=settings.USE_WEBSOCKETS,
# Stream message notification settings:
stream_desktop_notifications_enabled=user_profile.
enable_stream_desktop_notifications,
stream_sounds_enabled=user_profile.enable_stream_sounds,
# Private message and @-mention notification settings:
desktop_notifications_enabled=desktop_notifications_enabled,
sounds_enabled=user_profile.enable_sounds,
enable_offline_email_notifications=user_profile.
enable_offline_email_notifications,
enable_offline_push_notifications=user_profile.
enable_offline_push_notifications,
enable_online_push_notifications=user_profile.
enable_online_push_notifications,
twenty_four_hour_time=register_ret['twenty_four_hour_time'],
enable_digest_emails=user_profile.enable_digest_emails,
event_queue_id=register_ret['queue_id'],
last_event_id=register_ret['last_event_id'],
max_message_id=register_ret['max_message_id'],
unread_count=approximate_unread_count(user_profile),
furthest_read_time=sent_time_in_epoch_seconds(latest_read),
save_stacktraces=settings.SAVE_FRONTEND_STACKTRACES,
alert_words=register_ret['alert_words'],
muted_topics=register_ret['muted_topics'],
realm_filters=register_ret['realm_filters'],
realm_default_streams=register_ret['realm_default_streams'],
is_admin=user_profile.is_realm_admin,
can_create_streams=user_profile.can_create_streams(),
name_changes_disabled=name_changes_disabled(user_profile.realm),
has_mobile_devices=num_push_devices_for_user(user_profile) > 0,
autoscroll_forever=user_profile.autoscroll_forever,
default_desktop_notifications=user_profile.
default_desktop_notifications,
avatar_url=avatar_url(user_profile),
avatar_url_medium=avatar_url(user_profile, medium=True),
mandatory_topics=user_profile.realm.mandatory_topics,
show_digest_email=user_profile.realm.show_digest_email,
presence_disabled=user_profile.realm.presence_disabled,
is_zephyr_mirror_realm=user_profile.realm.is_zephyr_mirror_realm,
)
if narrow_stream is not None:
# In narrow_stream context, initial pointer is just latest message
recipient = get_recipient(Recipient.STREAM, narrow_stream.id)
try:
initial_pointer = Message.objects.filter(
recipient=recipient).order_by('id').reverse()[0].id
except IndexError:
initial_pointer = -1
page_params["narrow_stream"] = narrow_stream.name
if narrow_topic is not None:
page_params["narrow_topic"] = narrow_topic
page_params["narrow"] = [
dict(operator=term[0], operand=term[1]) for term in narrow
]
page_params["max_message_id"] = initial_pointer
page_params["initial_pointer"] = initial_pointer
page_params["have_initial_messages"] = (initial_pointer != -1)
statsd.incr('views.home')
show_invites = True
# Some realms only allow admins to invite users
if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin:
show_invites = False
product_name = "Zulip"
page_params['product_name'] = product_name
request._log_data['extra'] = "[%s]" % (register_ret["queue_id"], )
response = render_to_response('zerver/index.html', {
'user_profile':
user_profile,
'page_params':
simplejson.encoder.JSONEncoderForHTML().encode(page_params),
'nofontface':
is_buggy_ua(request.META.get("HTTP_USER_AGENT", "Unspecified")),
'avatar_url':
avatar_url(user_profile),
'show_debug':
settings.DEBUG and ('show_debug' in request.GET),
'pipeline':
settings.PIPELINE_ENABLED,
'show_invites':
show_invites,
'is_admin':
user_profile.is_realm_admin,
'show_webathena':
user_profile.realm.webathena_enabled,
'enable_feedback':
settings.ENABLE_FEEDBACK,
'embedded':
narrow_stream is not None,
'product_name':
product_name
},
request=request)
patch_cache_control(response,
no_cache=True,
no_store=True,
must_revalidate=True)
return response
def test_basics(self) -> None:
cordelia = self.example_user('cordelia')
hamlet = self.example_user('hamlet')
king_lear = self.lear_user('king')
realm_id = hamlet.realm_id
away_user_ids = get_away_user_ids(realm_id=realm_id)
self.assertEqual(away_user_ids, set())
client1 = get_client('web')
client2 = get_client('ZT')
update_user_status(
user_profile_id=hamlet.id,
status=UserStatus.AWAY,
status_text=None,
client_id=client1.id,
)
away_user_ids = get_away_user_ids(realm_id=realm_id)
self.assertEqual(away_user_ids, {hamlet.id})
# Test that second client just updates
# the record. We only store one record
# per user. The user's status transcends
# clients; we only store the client for
# reference and to maybe reconcile timeout
# situations.
update_user_status(
user_profile_id=hamlet.id,
status=UserStatus.AWAY,
status_text='out to lunch',
client_id=client2.id,
)
self.assertEqual(
user_info(hamlet),
dict(away=True, status_text='out to lunch'),
)
away_user_ids = get_away_user_ids(realm_id=realm_id)
self.assertEqual(away_user_ids, {hamlet.id})
rec_count = UserStatus.objects.filter(
user_profile_id=hamlet.id).count()
self.assertEqual(rec_count, 1)
# Setting status_text to None causes it be ignored.
update_user_status(
user_profile_id=hamlet.id,
status=UserStatus.NORMAL,
status_text=None,
client_id=client2.id,
)
self.assertEqual(
user_info(hamlet),
dict(status_text='out to lunch'),
)
# Clear the status_text now.
update_user_status(
user_profile_id=hamlet.id,
status=None,
status_text='',
client_id=client2.id,
)
self.assertEqual(
user_info(hamlet),
dict(),
)
away_user_ids = get_away_user_ids(realm_id=realm_id)
self.assertEqual(away_user_ids, set())
# Now set away status for three different users across
# two realms.
update_user_status(
user_profile_id=hamlet.id,
status=UserStatus.AWAY,
status_text=None,
client_id=client1.id,
)
update_user_status(
user_profile_id=cordelia.id,
status=UserStatus.AWAY,
status_text=None,
client_id=client2.id,
)
update_user_status(
user_profile_id=king_lear.id,
status=UserStatus.AWAY,
status_text=None,
client_id=client2.id,
)
away_user_ids = get_away_user_ids(realm_id=realm_id)
self.assertEqual(away_user_ids, {cordelia.id, hamlet.id})
away_user_ids = get_away_user_ids(realm_id=king_lear.realm.id)
self.assertEqual(away_user_ids, {king_lear.id})
# Set Hamlet to NORMAL but in a meeting.
update_user_status(
user_profile_id=hamlet.id,
status=UserStatus.NORMAL,
status_text='in a meeting',
client_id=client2.id,
)
self.assertEqual(
user_info(hamlet),
dict(status_text='in a meeting'),
)
away_user_ids = get_away_user_ids(realm_id=realm_id)
self.assertEqual(away_user_ids, {cordelia.id})