def update_user_custom_profile_data(
request: HttpRequest,
user_profile: UserProfile,
data: List[Dict[str, Union[int, str]]]=REQ(validator=check_list(
check_dict([('id', check_int)])))) -> HttpResponse:
for item in data:
field_id = item['id']
try:
field = CustomProfileField.objects.get(id=field_id)
except CustomProfileField.DoesNotExist:
return json_error(_('Field id {id} not found.').format(id=field_id))
validators = CustomProfileField.FIELD_VALIDATORS
extended_validators = CustomProfileField.EXTENDED_FIELD_VALIDATORS
field_type = field.field_type
value = item['value']
if field_type in validators:
validator = validators[field_type]
var_name = 'value[{}]'.format(field_id)
result = validator(var_name, value)
else:
# Check extended validators.
extended_validator = extended_validators[field_type]
field_data = field.field_data
var_name = 'value[{}]'.format(field_id)
result = extended_validator(var_name, field_data, value)
if result is not None:
return json_error(result)
do_update_user_custom_profile_data(user_profile, data)
# We need to call this explicitly otherwise constraints are not check
return json_success()
def test_delete_field_value(self) -> None:
iago = self.example_user("iago")
self.login(iago.email)
realm = get_realm("zulip")
invalid_field_id = 1234
result = self.client_delete("/json/users/me/profile_data",
{'data': ujson.dumps([invalid_field_id])})
self.assert_json_error(
result, u'Field id %d not found.' % (invalid_field_id, ))
field = CustomProfileField.objects.get(name="Mentor", realm=realm)
data = [{
'id': field.id,
'value': [self.example_user("aaron").id]
}] # type: List[Dict[str, Union[int, str, List[int]]]]
do_update_user_custom_profile_data(iago, data)
iago_value = CustomProfileFieldValue.objects.get(user_profile=iago,
field=field)
converter = field.FIELD_CONVERTERS[field.field_type]
self.assertEqual([self.example_user("aaron").id],
converter(iago_value.value))
result = self.client_delete("/json/users/me/profile_data",
{'data': ujson.dumps([field.id])})
self.assert_json_success(result)
# Don't throw an exception here
result = self.client_delete("/json/users/me/profile_data",
{'data': ujson.dumps([field.id])})
self.assert_json_success(result)
def update_user_custom_profile_data(
request: HttpRequest,
user_profile: UserProfile,
data: List[Dict[str, Union[int, str]]]=REQ(validator=check_list(
check_dict([('id', check_int)])))) -> HttpResponse:
for item in data:
field_id = item['id']
try:
field = CustomProfileField.objects.get(id=field_id)
except CustomProfileField.DoesNotExist:
return json_error(_('Field id {id} not found.').format(id=field_id))
validators = CustomProfileField.FIELD_VALIDATORS
extended_validators = CustomProfileField.EXTENDED_FIELD_VALIDATORS
field_type = field.field_type
value = item['value']
var_name = '{}'.format(field.name)
if field_type in validators:
validator = validators[field_type]
result = validator(var_name, value)
else:
# Check extended validators.
extended_validator = extended_validators[field_type]
field_data = field.field_data
result = extended_validator(var_name, field_data, value)
if result is not None:
return json_error(result)
do_update_user_custom_profile_data(user_profile, data)
# We need to call this explicitly otherwise constraints are not check
return json_success()
def test_null_value_and_rendered_value(self) -> None:
self.login(self.example_email("iago"))
realm = get_realm("zulip")
quote = try_add_realm_custom_profile_field(
realm=realm,
name="Quote",
hint="Saying or phrase which you known for.",
field_type=CustomProfileField.SHORT_TEXT)
iago = self.example_user("iago")
iago_profile_quote = iago.profile_data[-1]
value = iago_profile_quote["value"]
rendered_value = iago_profile_quote["rendered_value"]
self.assertIsNone(value)
self.assertIsNone(rendered_value)
update_dict = {"id": quote.id, "value": "***beware*** of jealousy..."}
do_update_user_custom_profile_data(iago, [update_dict])
iago_profile_quote = self.example_user("iago").profile_data[-1]
value = iago_profile_quote["value"]
rendered_value = iago_profile_quote["rendered_value"]
self.assertIsNotNone(value)
self.assertIsNotNone(rendered_value)
self.assertEqual(
"<p><strong><em>beware</em></strong> of jealousy...</p>",
rendered_value)
def test_delete_field_value(self) -> None:
iago = self.example_user("iago")
self.login(iago.email)
realm = get_realm("zulip")
invalid_field_id = 1234
result = self.client_delete("/json/users/me/profile_data", {
'data': ujson.dumps([invalid_field_id])
})
self.assert_json_error(result,
u'Field id %d not found.' % (invalid_field_id))
field = CustomProfileField.objects.get(name="Mentor", realm=realm)
data = [{'id': field.id,
'value': [self.example_user("aaron").id]}] # type: List[Dict[str, Union[int, str, List[int]]]]
do_update_user_custom_profile_data(iago, data)
iago_value = CustomProfileFieldValue.objects.get(user_profile=iago, field=field)
converter = field.FIELD_CONVERTERS[field.field_type]
self.assertEqual([self.example_user("aaron").id], converter(iago_value.value))
result = self.client_delete("/json/users/me/profile_data", {
'data': ujson.dumps([field.id])
})
self.assert_json_success(result)
# Don't throw an exception here
result = self.client_delete("/json/users/me/profile_data", {
'data': ujson.dumps([field.id])
})
self.assert_json_success(result)
def test_null_value_and_rendered_value(self) -> None:
self.login(self.example_email("iago"))
realm = get_realm("zulip")
quote = try_add_realm_custom_profile_field(
realm=realm,
name="Quote",
hint="Saying or phrase which you known for.",
field_type=CustomProfileField.SHORT_TEXT
)
iago = self.example_user("iago")
iago_profile_quote = iago.profile_data[-1]
value = iago_profile_quote["value"]
rendered_value = iago_profile_quote["rendered_value"]
self.assertIsNone(value)
self.assertIsNone(rendered_value)
update_dict = {
"id": quote.id,
"value": "***beware*** of jealousy..."
}
do_update_user_custom_profile_data(iago, [update_dict])
iago_profile_quote = self.example_user("iago").profile_data[-1]
value = iago_profile_quote["value"]
rendered_value = iago_profile_quote["rendered_value"]
self.assertIsNotNone(value)
self.assertIsNotNone(rendered_value)
self.assertEqual("<p><strong><em>beware</em></strong> of jealousy...</p>", rendered_value)
def update_user_custom_profile_data(
request: HttpRequest,
user_profile: UserProfile,
data: List[Dict[str, Union[int, str, List[int]]]]=REQ(validator=check_list(
check_dict([('id', check_int)])))) -> HttpResponse:
validate_user_custom_profile_data(user_profile.realm.id, data)
do_update_user_custom_profile_data(user_profile, data)
# We need to call this explicitly otherwise constraints are not check
return json_success()
def update_user_custom_profile_data(
request: HttpRequest,
user_profile: UserProfile,
data: List[Dict[str, Union[int, str, List[int]]]] = REQ(
validator=check_list(check_dict([('id', check_int)])))
) -> HttpResponse:
validate_user_custom_profile_data(user_profile.realm.id, data)
do_update_user_custom_profile_data(user_profile, data)
# We need to call this explicitly otherwise constraints are not check
return json_success()
def update_user_backend(
request: HttpRequest,
user_profile: UserProfile,
user_id: int,
full_name: Optional[str] = REQ(default="", validator=check_string),
is_admin: Optional[bool] = REQ(default=None, validator=check_bool),
is_guest: Optional[bool] = REQ(default=None, validator=check_bool),
profile_data: List[Dict[str, Union[int, str, List[int]]]] = REQ(
default=None, validator=check_list(check_dict([('id', check_int)])))
) -> HttpResponse:
target = access_user_by_id(user_profile,
user_id,
allow_deactivated=True,
allow_bots=True)
# This condition is a bit complicated, because the user could
# already be a guest/admin, or the request could be to make the
# user a guest/admin. In any case, the point is that we outright
# reject requests that would result in a user who is both an admin
# and a guest.
if (((is_guest is None and target.is_guest) or is_guest)
and ((is_admin is None and target.is_realm_admin) or is_admin)):
return json_error(_("Guests cannot be organization administrators"))
if is_admin is not None and target.is_realm_admin != is_admin:
if not is_admin and check_last_admin(user_profile):
return json_error(
_('Cannot remove the only organization administrator'))
do_change_is_admin(target, is_admin)
if is_guest is not None and target.is_guest != is_guest:
do_change_is_guest(target, is_guest)
if (full_name is not None and target.full_name != full_name
and full_name.strip() != ""):
# We don't respect `name_changes_disabled` here because the request
# is on behalf of the administrator.
check_change_full_name(target, full_name, user_profile)
if profile_data is not None:
clean_profile_data = []
for entry in profile_data:
if not entry["value"]:
field_id = entry["id"]
check_remove_custom_profile_field_value(target, field_id)
else:
clean_profile_data.append(entry)
validate_user_custom_profile_data(target.realm.id, clean_profile_data)
do_update_user_custom_profile_data(target, clean_profile_data)
return json_success()
def test_delete_internals(self) -> None:
user_profile = self.example_user('iago')
realm = user_profile.realm
field = CustomProfileField.objects.get(name="Phone number", realm=realm)
data = [{'id': field.id, 'value': u'123456'}] # type: List[Dict[str, Union[int, str, List[int]]]]
do_update_user_custom_profile_data(user_profile, data)
self.assertTrue(self.custom_field_exists_in_realm(field.id))
self.assertEqual(user_profile.customprofilefieldvalue_set.count(), self.original_count)
do_remove_realm_custom_profile_field(realm, field)
self.assertFalse(self.custom_field_exists_in_realm(field.id))
self.assertEqual(user_profile.customprofilefieldvalue_set.count(), self.original_count - 1)
def test_delete_internals(self) -> None:
user_profile = self.example_user('iago')
realm = user_profile.realm
field = CustomProfileField.objects.get(name="Phone number", realm=realm)
data = [{'id': field.id, 'value': u'123456'}] # type: List[Dict[str, Union[int, str, List[int]]]]
do_update_user_custom_profile_data(user_profile, data)
self.assertTrue(self.custom_field_exists_in_realm(field.id))
self.assertEqual(user_profile.customprofilefieldvalue_set.count(), self.original_count)
do_remove_realm_custom_profile_field(realm, field)
self.assertFalse(self.custom_field_exists_in_realm(field.id))
self.assertEqual(user_profile.customprofilefieldvalue_set.count(), self.original_count - 1)
def test_delete(self) -> None:
user_profile = self.example_user('iago')
realm = user_profile.realm
field = CustomProfileField.objects.get(name="Phone number", realm=realm)
data = [{'id': field.id, 'value': u'123456'}] # type: List[Dict[str, Union[int, Text]]]
do_update_user_custom_profile_data(user_profile, data)
self.assertEqual(len(custom_profile_fields_for_realm(realm.id)), 3)
self.assertEqual(user_profile.customprofilefieldvalue_set.count(), 3)
do_remove_realm_custom_profile_field(realm, field)
self.assertEqual(len(custom_profile_fields_for_realm(realm.id)), 2)
self.assertEqual(user_profile.customprofilefieldvalue_set.count(), 2)
def sync_custom_profile_fields_from_ldap(self, user_profile: UserProfile,
ldap_user: _LDAPUser) -> None:
values_by_var_name = {} # type: Dict[str, Union[int, str, List[int]]]
for attr, ldap_attr in settings.AUTH_LDAP_USER_ATTR_MAP.items():
if not attr.startswith('custom_profile_field__'):
continue
var_name = attr.split('custom_profile_field__')[1]
try:
value = ldap_user.attrs[ldap_attr][0]
except KeyError:
# If this user doesn't have this field set then ignore this
# field and continue syncing other fields. `django-auth-ldap`
# automatically logs error about missing field.
continue
values_by_var_name[var_name] = value
fields_by_var_name = {} # type: Dict[str, CustomProfileField]
custom_profile_fields = custom_profile_fields_for_realm(
user_profile.realm.id)
for field in custom_profile_fields:
var_name = '_'.join(field.name.lower().split(' '))
fields_by_var_name[var_name] = field
existing_values = {}
for data in user_profile.profile_data:
var_name = '_'.join(data['name'].lower().split(
' ')) # type: ignore # data field values can also be int
existing_values[var_name] = data['value']
profile_data = [] # type: List[Dict[str, Union[int, str, List[int]]]]
for var_name, value in values_by_var_name.items():
try:
field = fields_by_var_name[var_name]
except KeyError:
raise ZulipLDAPException(
'Custom profile field with name %s not found.' %
(var_name, ))
if existing_values.get(var_name) == value:
continue
result = validate_user_custom_profile_field(
user_profile.realm.id, field, value)
if result is not None:
raise ZulipLDAPException('Invalid data for %s field: %s' %
(var_name, result))
profile_data.append({
'id': field.id,
'value': value,
})
do_update_user_custom_profile_data(user_profile, profile_data)
def test_delete(self) -> None:
user_profile = self.example_user('iago')
realm = user_profile.realm
field = try_add_realm_custom_profile_field(
realm,
u"Phone",
CustomProfileField.SHORT_TEXT
)
data = [{'id': field.id, 'value': u'123456'}] # type: List[Dict[str, Union[int, Text]]]
do_update_user_custom_profile_data(user_profile, data)
self.assertEqual(len(custom_profile_fields_for_realm(realm.id)), 1)
self.assertEqual(user_profile.customprofilefieldvalue_set.count(), 1)
do_remove_realm_custom_profile_field(realm, field)
self.assertEqual(len(custom_profile_fields_for_realm(realm.id)), 0)
self.assertEqual(user_profile.customprofilefieldvalue_set.count(), 0)
def test_delete(self) -> None:
user_profile = self.example_user('iago')
realm = user_profile.realm
field = try_add_realm_custom_profile_field(
realm, u"Phone", CustomProfileField.SHORT_TEXT)
data = [{
'id': field.id,
'value': u'123456'
}] # type: List[Dict[str, Union[int, Text]]]
do_update_user_custom_profile_data(user_profile, data)
self.assertEqual(len(custom_profile_fields_for_realm(realm.id)), 1)
self.assertEqual(user_profile.customprofilefieldvalue_set.count(), 1)
do_remove_realm_custom_profile_field(realm, field)
self.assertEqual(len(custom_profile_fields_for_realm(realm.id)), 0)
self.assertEqual(user_profile.customprofilefieldvalue_set.count(), 0)
def update_user_backend(request: HttpRequest, user_profile: UserProfile, user_id: int,
full_name: Optional[str]=REQ(default="", validator=check_string),
is_admin: Optional[bool]=REQ(default=None, validator=check_bool),
is_guest: Optional[bool]=REQ(default=None, validator=check_bool),
profile_data: List[Dict[str, Union[int, str, List[int]]]]=
REQ(default=None,
validator=check_list(check_dict([('id', check_int)])))) -> HttpResponse:
target = access_user_by_id(user_profile, user_id, allow_deactivated=True, allow_bots=True)
# This condition is a bit complicated, because the user could
# already be a guest/admin, or the request could be to make the
# user a guest/admin. In any case, the point is that we outright
# reject requests that would result in a user who is both an admin
# and a guest.
if (((is_guest is None and target.is_guest) or is_guest) and
((is_admin is None and target.is_realm_admin) or is_admin)):
return json_error(_("Guests cannot be organization administrators"))
if is_admin is not None and target.is_realm_admin != is_admin:
if not is_admin and check_last_admin(user_profile):
return json_error(_('Cannot remove the only organization administrator'))
do_change_is_admin(target, is_admin)
if is_guest is not None and target.is_guest != is_guest:
do_change_is_guest(target, is_guest)
if (full_name is not None and target.full_name != full_name and
full_name.strip() != ""):
# We don't respect `name_changes_disabled` here because the request
# is on behalf of the administrator.
check_change_full_name(target, full_name, user_profile)
if profile_data is not None:
clean_profile_data = []
for entry in profile_data:
if not entry["value"]:
field_id = entry["id"]
check_remove_custom_profile_field_value(target, field_id)
else:
clean_profile_data.append(entry)
validate_user_custom_profile_data(target.realm.id, clean_profile_data)
do_update_user_custom_profile_data(target, clean_profile_data)
return json_success()
def sync_custom_profile_fields_from_ldap(self, user_profile: UserProfile,
ldap_user: _LDAPUser) -> None:
values_by_var_name = {} # type: Dict[str, Union[int, str, List[int]]]
for attr, ldap_attr in settings.AUTH_LDAP_USER_ATTR_MAP.items():
if not attr.startswith('custom_profile_field__'):
continue
var_name = attr.split('custom_profile_field__')[1]
try:
value = ldap_user.attrs[ldap_attr][0]
except KeyError:
# If this user doesn't have this field set then ignore this
# field and continue syncing other fields. `django-auth-ldap`
# automatically logs error about missing field.
continue
values_by_var_name[var_name] = value
fields_by_var_name = {} # type: Dict[str, CustomProfileField]
custom_profile_fields = custom_profile_fields_for_realm(user_profile.realm.id)
for field in custom_profile_fields:
var_name = '_'.join(field.name.lower().split(' '))
fields_by_var_name[var_name] = field
existing_values = {}
for data in user_profile.profile_data:
var_name = '_'.join(data['name'].lower().split(' ')) # type: ignore # data field values can also be int
existing_values[var_name] = data['value']
profile_data = [] # type: List[Dict[str, Union[int, str, List[int]]]]
for var_name, value in values_by_var_name.items():
try:
field = fields_by_var_name[var_name]
except KeyError:
raise ZulipLDAPException('Custom profile field with name %s not found.' % (var_name,))
if existing_values.get(var_name) == value:
continue
result = validate_user_custom_profile_field(user_profile.realm.id, field, value)
if result is not None:
raise ZulipLDAPException('Invalid data for %s field: %s' % (var_name, result))
profile_data.append({
'id': field.id,
'value': value,
})
do_update_user_custom_profile_data(user_profile, profile_data)
def update_user_custom_profile_data(
request,
user_profile,
data=REQ(validator=check_list(check_dict([('id', check_int)])))):
# type: (HttpRequest, UserProfile, List[Dict[str, Union[int, Text]]]) -> HttpResponse
for item in data:
field_id = item['id']
try:
field = CustomProfileField.objects.get(id=field_id)
except CustomProfileField.DoesNotExist:
return json_error(_('Field id {id} not found.').format(id=field_id))
validator = CustomProfileField.FIELD_VALIDATORS[field.field_type]
result = validator('value[{}]'.format(field_id), item['value'])
if result is not None:
return json_error(result)
do_update_user_custom_profile_data(user_profile, data)
# We need to call this explicitly otherwise constraints are not check
return json_success()
def update_user_custom_profile_data(
request,
user_profile,
data=REQ(validator=check_list(check_dict([('id', check_int)])))):
# type: (HttpRequest, UserProfile, List[Dict[str, Union[int, Text]]]) -> HttpResponse
for item in data:
field_id = item['id']
try:
field = CustomProfileField.objects.get(id=field_id)
except CustomProfileField.DoesNotExist:
return json_error(
_('Field id {id} not found.').format(id=field_id))
validator = CustomProfileField.FIELD_VALIDATORS[field.field_type]
result = validator('value[{}]'.format(field_id), item['value'])
if result is not None:
return json_error(result)
do_update_user_custom_profile_data(user_profile, data)
# We need to call this explicitly otherwise constraints are not check
return json_success()
def update_user_custom_profile_data(
request: HttpRequest,
user_profile: UserProfile,
data: List[Dict[str, Union[int, str, List[int]]]] = REQ(
validator=check_list(check_dict([('id', check_int)])))
) -> HttpResponse:
for item in data:
field_id = item['id']
try:
field = CustomProfileField.objects.get(id=field_id)
except CustomProfileField.DoesNotExist:
return json_error(
_('Field id {id} not found.').format(id=field_id))
validators = CustomProfileField.FIELD_VALIDATORS
field_type = field.field_type
var_name = '{}'.format(field.name)
value = item['value']
if field_type in validators:
validator = validators[field_type]
result = validator(var_name, value)
elif field_type == CustomProfileField.CHOICE:
choice_field_validator = CustomProfileField.CHOICE_FIELD_VALIDATORS[
field_type]
field_data = field.field_data
result = choice_field_validator(var_name, field_data, value)
elif field_type == CustomProfileField.USER:
user_field_validator = CustomProfileField.USER_FIELD_VALIDATORS[
field_type]
result = user_field_validator(user_profile.realm.id,
cast(List[int], value), False)
else:
raise AssertionError("Invalid field type")
if result is not None:
return json_error(result)
do_update_user_custom_profile_data(user_profile, data)
# We need to call this explicitly otherwise constraints are not check
return json_success()
def update_user_custom_profile_data(
request: HttpRequest,
user_profile: UserProfile,
data: List[Dict[str, Union[int, str, List[int]]]]=REQ(validator=check_list(
check_dict([('id', check_int)])))) -> HttpResponse:
for item in data:
field_id = item['id']
try:
field = CustomProfileField.objects.get(id=field_id)
except CustomProfileField.DoesNotExist:
return json_error(_('Field id {id} not found.').format(id=field_id))
validators = CustomProfileField.FIELD_VALIDATORS
field_type = field.field_type
var_name = '{}'.format(field.name)
value = item['value']
if field_type in validators:
validator = validators[field_type]
result = validator(var_name, value)
elif field_type == CustomProfileField.CHOICE:
choice_field_validator = CustomProfileField.CHOICE_FIELD_VALIDATORS[field_type]
field_data = field.field_data
result = choice_field_validator(var_name, field_data, value)
elif field_type == CustomProfileField.USER:
user_field_validator = CustomProfileField.USER_FIELD_VALIDATORS[field_type]
result = user_field_validator(user_profile.realm.id, cast(List[int], value),
False)
else:
raise AssertionError("Invalid field type")
if result is not None:
return json_error(result)
do_update_user_custom_profile_data(user_profile, data)
# We need to call this explicitly otherwise constraints are not check
return json_success()
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write(
"Error! More than 100% of messages allocated.\n")
return
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our two default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
zulip_realm = Realm.objects.create(
string_id="zulip",
name="Zulip Dev",
restricted_to_domain=True,
description=
"The Zulip development environment default organization."
" It's great for testing!",
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
if options["test_suite"]:
mit_realm = Realm.objects.create(string_id="zephyr",
name="MIT",
restricted_to_domain=True,
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = Realm.objects.create(string_id="lear",
name="Lear & Co.",
restricted_to_domain=False,
invite_required=False,
org_type=Realm.CORPORATE)
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
("Polonius", "*****@*****.**"),
]
for i in range(options["extra_users"]):
names.append(
('Extra User %d' % (i, ), '*****@*****.**' % (i, )))
create_users(zulip_realm, names)
iago = get_user("*****@*****.**", zulip_realm)
do_change_is_admin(iago, True)
iago.is_staff = True
iago.save(update_fields=['is_staff'])
guest_user = get_user("*****@*****.**", zulip_realm)
guest_user.is_guest = True
guest_user.save(update_fields=['is_guest'])
# These bots are directly referenced from code and thus
# are needed for the test suite.
all_realm_bots = [
(bot['name'],
bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN, ))
for bot in settings.INTERNAL_BOTS
]
zulip_realm_bots = [
("Zulip New User Bot", "*****@*****.**"),
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
("Welcome Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(
('Extra Bot %d' % (i, ), '*****@*****.**' % (i, )))
zulip_realm_bots.extend(all_realm_bots)
create_users(zulip_realm,
zulip_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
zoe = get_user("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(zulip_realm,
zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
bot_owner=zoe)
aaron = get_user("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [("Outgoing Webhook",
"*****@*****.**")]
create_users(zulip_realm,
zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
bot_owner=aaron)
# TODO: Clean up this initial bot creation code
Service.objects.create(
name="test",
user_profile=get_user("*****@*****.**",
zulip_realm),
base_url="http://127.0.0.1:5002/bots/followup",
token="abcd1234",
interface=1)
# Create public streams.
stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
stream_dict = {
"Verona": {
"description": "A city in Italy",
"invite_only": False,
"is_web_public": False
},
"Denmark": {
"description": "A Scandinavian country",
"invite_only": False,
"is_web_public": False
},
"Scotland": {
"description": "Located in the United Kingdom",
"invite_only": False,
"is_web_public": False
},
"Venice": {
"description": "A northeastern Italian city",
"invite_only": False,
"is_web_public": False
},
"Rome": {
"description": "Yet another Italian city",
"invite_only": False,
"is_web_public": True
}
} # type: Dict[str, Dict[str, Any]]
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams = [
Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list
] # type: List[int]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users).
subscriptions_to_add = [] # type: List[Subscription]
event_time = timezone_now()
all_subscription_logs = [] # type: (List[RealmAuditLog])
profiles = UserProfile.objects.select_related().filter(
is_bot=False, is_guest=False).order_by(
"email") # type: Sequence[UserProfile]
for i, profile in enumerate(profiles):
# Subscribe to some streams.
for type_id in recipient_streams[:int(
len(recipient_streams) * float(i) / len(profiles)) +
1]:
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=type_id)
s = Subscription(recipient=r,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[
i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream_id=type_id,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(
zulip_realm,
"Phone number",
CustomProfileField.SHORT_TEXT,
hint='')
biography = try_add_realm_custom_profile_field(
zulip_realm,
"Biography",
CustomProfileField.LONG_TEXT,
hint='What are you known for?')
favorite_food = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite food",
CustomProfileField.SHORT_TEXT,
hint="Or drink, if you'd prefer")
field_data = {
'vim': {
'text': 'Vim',
'order': '1'
},
'emacs': {
'text': 'Emacs',
'order': '2'
},
}
favorite_editor = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite editor",
CustomProfileField.CHOICE,
field_data=field_data)
birthday = try_add_realm_custom_profile_field(
zulip_realm, "Birthday", CustomProfileField.DATE)
favorite_website = try_add_realm_custom_profile_field(
zulip_realm,
"GitHub profile",
CustomProfileField.URL,
hint="Or your personal blog's URL")
# Fill in values for Iago and Hamlet
hamlet = get_user("*****@*****.**", zulip_realm)
do_update_user_custom_profile_data(iago, [
{
"id": phone_number.id,
"value": "+1-234-567-8901"
},
{
"id": biography.id,
"value": "Betrayer of Othello."
},
{
"id": favorite_food.id,
"value": "Apples"
},
{
"id": favorite_editor.id,
"value": "emacs"
},
{
"id": birthday.id,
"value": "2000-1-1"
},
{
"id": favorite_website.id,
"value": "https://github.com/zulip/zulip"
},
])
do_update_user_custom_profile_data(hamlet, [
{
"id": phone_number.id,
"value": "+0-11-23-456-7890"
},
{
"id": biography.id,
"value": "Prince of Denmark, and other things!"
},
{
"id": favorite_food.id,
"value": "Dark chocolate"
},
{
"id": favorite_editor.id,
"value": "vim"
},
{
"id": birthday.id,
"value": "1900-1-1"
},
{
"id": favorite_website.id,
"value": "https://blog.zulig.org"
},
])
else:
zulip_realm = get_realm("zulip")
recipient_streams = [
klass.type_id
for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
# Extract a list of all users
user_profiles = list(UserProfile.objects.filter(
is_bot=False)) # type: List[UserProfile]
# Create a test realm emoji.
IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images',
'test-images', 'checkbox.png')
with open(IMAGE_FILE_PATH, 'rb') as fp:
check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status = UserPresence.ACTIVE # type: int
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= 'H':
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(user_profile=user,
client=client,
timestamp=date,
status=status)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [
random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])
]
# Generate a new set of test data.
create_test_data()
# prepopulate the URL preview/embed data for the links present
# in the config.generate_data.json data set. This makes it
# possible for populate_db to run happily without Internet
# access.
with open("zerver/tests/fixtures/docs_url_preview_data.json",
"r") as f:
urls_with_preview_data = ujson.load(f)
for url in urls_with_preview_data:
cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)
threads = options["threads"]
jobs = [
] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write,
random.randint(0, 10**10)))
for job in jobs:
send_messages(job)
if options["delete"]:
# Create the "website" and "API" clients; if we don't, the
# default values in zerver/decorators.py will not work
# with the Django test suite.
get_client("website")
get_client("API")
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)",
"*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm, testsuite_mit_users)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
]
create_users(lear_realm, testsuite_lear_users)
if not options["test_suite"]:
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
zulip_stream_dict = {
"devel": {
"description": "For developing",
"invite_only": False,
"is_web_public": False
},
"all": {
"description": "For everything",
"invite_only": False,
"is_web_public": False
},
"announce": {
"description": "For announcements",
"invite_only": False,
"is_web_public": False
},
"design": {
"description": "For design",
"invite_only": False,
"is_web_public": False
},
"support": {
"description": "For support",
"invite_only": False,
"is_web_public": False
},
"social": {
"description": "For socializing",
"invite_only": False,
"is_web_public": False
},
"test": {
"description": "For testing",
"invite_only": False,
"is_web_public": False
},
"errors": {
"description": "For errors",
"invite_only": False,
"is_web_public": False
},
"sales": {
"description": "For sales discussion",
"invite_only": False,
"is_web_public": False
}
} # type: Dict[str, Dict[str, Any]]
# Calculate the maximum number of digits in any extra stream's
# number, since a stream with name "Extra Stream 3" could show
# up after "Extra Stream 29". (Used later to pad numbers with
# 0s).
maximum_digits = len(str(options['extra_streams'] - 1))
for i in range(options['extra_streams']):
# Pad the number with 0s based on `maximum_digits`.
number_str = str(i).zfill(maximum_digits)
extra_stream_name = 'Extra Stream ' + number_str
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
"invite_only": False,
"is_web_public": False,
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream(
"announce", zulip_realm)
zulip_realm.save(update_fields=['notifications_stream'])
# Add a few default streams
for default_stream_name in [
"design", "devel", "social", "support"
]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(
default_stream_name,
zulip_realm))
# Now subscribe everyone to these streams
subscriptions_to_add = []
event_time = timezone_now()
all_subscription_logs = []
profiles = UserProfile.objects.select_related().filter(
realm=zulip_realm)
for i, stream_name in enumerate(zulip_stream_dict):
stream = Stream.objects.get(name=stream_name,
realm=zulip_realm)
recipient = Recipient.objects.get(type=Recipient.STREAM,
type_id=stream.id)
for profile in profiles:
# Subscribe to some streams.
s = Subscription(
recipient=recipient,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[
i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream=stream,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# These bots are not needed by the test suite
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm,
internal_zulip_users_nosubs,
bot_type=UserProfile.DEFAULT_BOT)
zulip_cross_realm_bots = [
("Zulip Feedback Bot", "*****@*****.**"),
]
create_users(zulip_realm,
zulip_cross_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
# Mark all messages as read
UserMessage.objects.all().update(flags=UserMessage.flags.read)
if not options["test_suite"]:
# Update pointer of each user to point to the last message in their
# UserMessage rows with sender_id=user_profile_id.
users = list(
UserMessage.objects.filter(message__sender_id=F(
'user_profile_id')).values('user_profile_id').annotate(
pointer=Max('message_id')))
for user in users:
UserProfile.objects.filter(
id=user['user_profile_id']).update(
pointer=user['pointer'])
create_user_groups()
self.stdout.write("Successfully populated test database.\n")
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write("Error! More than 100% of messages allocated.\n")
return
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our two default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
zulip_realm = Realm.objects.create(
string_id="zulip", name="Zulip Dev", restricted_to_domain=True,
description="The Zulip development environment default organization."
" It's great for testing!",
invite_required=False, org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
if options["test_suite"]:
mit_realm = Realm.objects.create(
string_id="zephyr", name="MIT", restricted_to_domain=True,
invite_required=False, org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = Realm.objects.create(
string_id="lear", name="Lear & Co.", restricted_to_domain=False,
invite_required=False, org_type=Realm.CORPORATE)
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
]
for i in range(options["extra_users"]):
names.append(('Extra User %d' % (i,), '*****@*****.**' % (i,)))
create_users(zulip_realm, names)
iago = get_user("*****@*****.**", zulip_realm)
do_change_is_admin(iago, True)
iago.is_staff = True
iago.save(update_fields=['is_staff'])
# These bots are directly referenced from code and thus
# are needed for the test suite.
all_realm_bots = [(bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,))
for bot in settings.INTERNAL_BOTS]
zulip_realm_bots = [
("Zulip New User Bot", "*****@*****.**"),
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
("Welcome Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(('Extra Bot %d' % (i,), '*****@*****.**' % (i,)))
zulip_realm_bots.extend(all_realm_bots)
create_users(zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT)
zoe = get_user("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(zulip_realm, zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe)
aaron = get_user("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [
("Outgoing Webhook", "*****@*****.**")
]
create_users(zulip_realm, zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, bot_owner=aaron)
# TODO: Clean up this initial bot creation code
Service.objects.create(
name="test",
user_profile=get_user("*****@*****.**", zulip_realm),
base_url="http://127.0.0.1:5002/bots/followup",
token="abcd1234",
interface=1)
# Create public streams.
stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
stream_dict = {
"Verona": {"description": "A city in Italy", "invite_only": False},
"Denmark": {"description": "A Scandinavian country", "invite_only": False},
"Scotland": {"description": "Located in the United Kingdom", "invite_only": False},
"Venice": {"description": "A northeastern Italian city", "invite_only": False},
"Rome": {"description": "Yet another Italian city", "invite_only": False}
} # type: Dict[Text, Dict[Text, Any]]
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams = [Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list] # type: List[int]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users).
subscriptions_to_add = [] # type: List[Subscription]
event_time = timezone_now()
all_subscription_logs = [] # type: (List[RealmAuditLog])
profiles = UserProfile.objects.select_related().filter(
is_bot=False).order_by("email") # type: Sequence[UserProfile]
for i, profile in enumerate(profiles):
# Subscribe to some streams.
for type_id in recipient_streams[:int(len(recipient_streams) *
float(i)/len(profiles)) + 1]:
r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id)
s = Subscription(
recipient=r,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream_id=type_id,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
if not options['test_suite']:
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(zulip_realm, "Phone number",
CustomProfileField.SHORT_TEXT)
biography = try_add_realm_custom_profile_field(zulip_realm, "Biography",
CustomProfileField.LONG_TEXT)
favorite_integer = try_add_realm_custom_profile_field(zulip_realm, "Favorite integer",
CustomProfileField.INTEGER)
# Fill in values for Iago and Hamlet
hamlet = get_user("*****@*****.**", zulip_realm)
do_update_user_custom_profile_data(iago, [
{"id": phone_number.id, "value": "+1-234-567-8901"},
{"id": biography.id, "value": "Betrayer of Othello."},
{"id": favorite_integer.id, "value": 17},
])
do_update_user_custom_profile_data(hamlet, [
{"id": phone_number.id, "value": "+0-11-23-456-7890"},
{"id": biography.id, "value": "Prince of Denmark, and other things!"},
{"id": favorite_integer.id, "value": 12},
])
else:
zulip_realm = get_realm("zulip")
recipient_streams = [klass.type_id for klass in
Recipient.objects.filter(type=Recipient.STREAM)]
# Extract a list of all users
user_profiles = list(UserProfile.objects.filter(is_bot=False)) # type: List[UserProfile]
# Create a test realm emoji.
IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images', 'test-images', 'checkbox.png')
UPLOADED_EMOJI_FILE_NAME = 'green_tick.png'
with open(IMAGE_FILE_PATH, 'rb') as fp:
upload_backend.upload_emoji_image(fp, UPLOADED_EMOJI_FILE_NAME, iago)
RealmEmoji.objects.create(
name='green_tick',
author=iago,
realm=zulip_realm,
deactivated=False,
file_name=UPLOADED_EMOJI_FILE_NAME,
)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status = UserPresence.ACTIVE # type: int
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= 'H':
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(user_profile=user,
client=client,
timestamp=date,
status=status)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])]
# Generate a new set of test data.
create_test_data()
threads = options["threads"]
jobs = [] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write, random.randint(0, 10**10)))
for job in jobs:
send_messages(job)
if options["delete"]:
# Create the "website" and "API" clients; if we don't, the
# default values in zerver/decorators.py will not work
# with the Django test suite.
get_client("website")
get_client("API")
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)", "*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm, testsuite_mit_users)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
]
create_users(lear_realm, testsuite_lear_users)
if not options["test_suite"]:
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
zulip_stream_dict = {
"devel": {"description": "For developing", "invite_only": False},
"all": {"description": "For everything", "invite_only": False},
"announce": {"description": "For announcements", "invite_only": False},
"design": {"description": "For design", "invite_only": False},
"support": {"description": "For support", "invite_only": False},
"social": {"description": "For socializing", "invite_only": False},
"test": {"description": "For testing", "invite_only": False},
"errors": {"description": "For errors", "invite_only": False},
"sales": {"description": "For sales discussion", "invite_only": False}
} # type: Dict[Text, Dict[Text, Any]]
# Calculate the maximum number of digits in any extra stream's
# number, since a stream with name "Extra Stream 3" could show
# up after "Extra Stream 29". (Used later to pad numbers with
# 0s).
maximum_digits = len(str(options['extra_streams'] - 1))
for i in range(options['extra_streams']):
# Pad the number with 0s based on `maximum_digits`.
number_str = str(i).zfill(maximum_digits)
extra_stream_name = 'Extra Stream ' + number_str
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
"invite_only": False,
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream("announce", zulip_realm)
zulip_realm.save(update_fields=['notifications_stream'])
# Add a few default streams
for default_stream_name in ["design", "devel", "social", "support"]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(default_stream_name, zulip_realm))
# Now subscribe everyone to these streams
subscriptions_to_add = []
event_time = timezone_now()
all_subscription_logs = []
profiles = UserProfile.objects.select_related().filter(realm=zulip_realm)
for i, stream_name in enumerate(zulip_stream_dict):
stream = Stream.objects.get(name=stream_name, realm=zulip_realm)
recipient = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id)
for profile in profiles:
# Subscribe to some streams.
s = Subscription(
recipient=recipient,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream=stream,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# These bots are not needed by the test suite
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm, internal_zulip_users_nosubs, bot_type=UserProfile.DEFAULT_BOT)
zulip_cross_realm_bots = [
("Zulip Feedback Bot", "*****@*****.**"),
]
create_users(zulip_realm, zulip_cross_realm_bots, bot_type=UserProfile.DEFAULT_BOT)
# Mark all messages as read
UserMessage.objects.all().update(flags=UserMessage.flags.read)
if not options["test_suite"]:
# Update pointer of each user to point to the last message in their
# UserMessage rows with sender_id=user_profile_id.
users = list(UserMessage.objects.filter(
message__sender_id=F('user_profile_id')).values(
'user_profile_id').annotate(pointer=Max('message_id')))
for user in users:
UserProfile.objects.filter(id=user['user_profile_id']).update(
pointer=user['pointer'])
create_user_groups()
self.stdout.write("Successfully populated test database.\n")
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write("Error! More than 100% of messages allocated.\n")
return
# Get consistent data for backend tests.
if options["test_suite"]:
random.seed(0)
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our two default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
zulip_realm = Realm.objects.create(
string_id="zulip", name="Zulip Dev", emails_restricted_to_domains=True,
description="The Zulip development environment default organization."
" It's great for testing!",
invite_required=False, org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
if options["test_suite"]:
mit_realm = Realm.objects.create(
string_id="zephyr", name="MIT", emails_restricted_to_domains=True,
invite_required=False, org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = Realm.objects.create(
string_id="lear", name="Lear & Co.", emails_restricted_to_domains=False,
invite_required=False, org_type=Realm.CORPORATE)
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
("Polonius", "*****@*****.**"),
]
for i in range(options["extra_users"]):
names.append(('Extra User %d' % (i,), '*****@*****.**' % (i,)))
create_users(zulip_realm, names)
iago = get_user("*****@*****.**", zulip_realm)
do_change_is_admin(iago, True)
iago.is_staff = True
iago.save(update_fields=['is_staff'])
guest_user = get_user("*****@*****.**", zulip_realm)
guest_user.is_guest = True
guest_user.save(update_fields=['is_guest'])
# These bots are directly referenced from code and thus
# are needed for the test suite.
all_realm_bots = [(bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,))
for bot in settings.INTERNAL_BOTS]
zulip_realm_bots = [
("Zulip New User Bot", "*****@*****.**"),
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
("Welcome Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(('Extra Bot %d' % (i,), '*****@*****.**' % (i,)))
zulip_realm_bots.extend(all_realm_bots)
create_users(zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT)
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
zoe = get_user("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(zulip_realm, zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe)
aaron = get_user("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [
("Outgoing Webhook", "*****@*****.**")
]
create_users(zulip_realm, zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, bot_owner=aaron)
outgoing_webhook = get_user("*****@*****.**", zulip_realm)
add_service("outgoing-webhook", user_profile=outgoing_webhook, interface=Service.GENERIC,
base_url="http://127.0.0.1:5002", token=generate_api_key())
# Add the realm internl bots to each realm.
create_if_missing_realm_internal_bots()
# Create public streams.
stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
stream_dict = {
"Verona": {"description": "A city in Italy"},
"Denmark": {"description": "A Scandinavian country"},
"Scotland": {"description": "Located in the United Kingdom"},
"Venice": {"description": "A northeastern Italian city"},
"Rome": {"description": "Yet another Italian city", "is_web_public": True}
} # type: Dict[str, Dict[str, Any]]
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams = [Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list] # type: List[int]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users). For the test suite, we have a fixed list of
# subscriptions to make sure test data is consistent
# across platforms.
subscriptions_list = [] # type: List[Tuple[UserProfile, Recipient]]
profiles = UserProfile.objects.select_related().filter(
is_bot=False).order_by("email") # type: Sequence[UserProfile]
if options["test_suite"]:
subscriptions_map = {
'*****@*****.**': ['Verona'],
'*****@*****.**': ['Verona'],
'*****@*****.**': ['Verona', 'Denmark'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice', 'Rome'],
'*****@*****.**': ['Verona'],
}
for profile in profiles:
if profile.email not in subscriptions_map:
raise Exception('Subscriptions not listed for user %s' % (profile.email,))
for stream_name in subscriptions_map[profile.email]:
stream = Stream.objects.get(name=stream_name)
r = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id)
subscriptions_list.append((profile, r))
else:
for i, profile in enumerate(profiles):
# Subscribe to some streams.
for type_id in recipient_streams[:int(len(recipient_streams) *
float(i)/len(profiles)) + 1]:
r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id)
subscriptions_list.append((profile, r))
subscriptions_to_add = [] # type: List[Subscription]
event_time = timezone_now()
all_subscription_logs = [] # type: (List[RealmAuditLog])
i = 0
for profile, recipient in subscriptions_list:
i += 1
color = STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)]
s = Subscription(
recipient=recipient,
user_profile=profile,
color=color)
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream_id=recipient.type_id,
event_last_message_id=0,
event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(zulip_realm, "Phone number",
CustomProfileField.SHORT_TEXT,
hint='')
biography = try_add_realm_custom_profile_field(zulip_realm, "Biography",
CustomProfileField.LONG_TEXT,
hint='What are you known for?')
favorite_food = try_add_realm_custom_profile_field(zulip_realm, "Favorite food",
CustomProfileField.SHORT_TEXT,
hint="Or drink, if you'd prefer")
field_data = {
'vim': {'text': 'Vim', 'order': '1'},
'emacs': {'text': 'Emacs', 'order': '2'},
}
favorite_editor = try_add_realm_custom_profile_field(zulip_realm,
"Favorite editor",
CustomProfileField.CHOICE,
field_data=field_data)
birthday = try_add_realm_custom_profile_field(zulip_realm, "Birthday",
CustomProfileField.DATE)
favorite_website = try_add_realm_custom_profile_field(zulip_realm, "GitHub profile",
CustomProfileField.URL,
hint="Or your personal blog's URL")
mentor = try_add_realm_custom_profile_field(zulip_realm, "Mentor",
CustomProfileField.USER)
# Fill in values for Iago and Hamlet
hamlet = get_user("*****@*****.**", zulip_realm)
do_update_user_custom_profile_data(iago, [
{"id": phone_number.id, "value": "+1-234-567-8901"},
{"id": biography.id, "value": "Betrayer of Othello."},
{"id": favorite_food.id, "value": "Apples"},
{"id": favorite_editor.id, "value": "emacs"},
{"id": birthday.id, "value": "2000-1-1"},
{"id": favorite_website.id, "value": "https://github.com/zulip/zulip"},
{"id": mentor.id, "value": [hamlet.id]},
])
do_update_user_custom_profile_data(hamlet, [
{"id": phone_number.id, "value": "+0-11-23-456-7890"},
{"id": biography.id, "value": "Prince of Denmark, and other things!"},
{"id": favorite_food.id, "value": "Dark chocolate"},
{"id": favorite_editor.id, "value": "vim"},
{"id": birthday.id, "value": "1900-1-1"},
{"id": favorite_website.id, "value": "https://blog.zulig.org"},
{"id": mentor.id, "value": [iago.id]},
])
else:
zulip_realm = get_realm("zulip")
recipient_streams = [klass.type_id for klass in
Recipient.objects.filter(type=Recipient.STREAM)]
# Extract a list of all users
user_profiles = list(UserProfile.objects.filter(is_bot=False)) # type: List[UserProfile]
# Create a test realm emoji.
IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images', 'test-images', 'checkbox.png')
with open(IMAGE_FILE_PATH, 'rb') as fp:
check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status = UserPresence.ACTIVE # type: int
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= 'H':
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(user_profile=user,
client=client,
timestamp=date,
status=status)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])]
# Generate a new set of test data.
create_test_data()
# prepopulate the URL preview/embed data for the links present
# in the config.generate_data.json data set. This makes it
# possible for populate_db to run happily without Internet
# access.
with open("zerver/tests/fixtures/docs_url_preview_data.json", "r") as f:
urls_with_preview_data = ujson.load(f)
for url in urls_with_preview_data:
cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)
threads = options["threads"]
jobs = [] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write, random.randint(0, 10**10)))
for job in jobs:
send_messages(job)
if options["delete"]:
# Create the "website" and "API" clients; if we don't, the
# default values in zerver/decorators.py will not work
# with the Django test suite.
get_client("website")
get_client("API")
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)", "*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm, testsuite_mit_users)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
]
create_users(lear_realm, testsuite_lear_users)
if not options["test_suite"]:
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
zulip_stream_dict = {
"devel": {"description": "For developing"},
"all": {"description": "For everything"},
"announce": {"description": "For announcements", 'is_announcement_only': True},
"design": {"description": "For design"},
"support": {"description": "For support"},
"social": {"description": "For socializing"},
"test": {"description": "For testing"},
"errors": {"description": "For errors"},
"sales": {"description": "For sales discussion"}
} # type: Dict[str, Dict[str, Any]]
# Calculate the maximum number of digits in any extra stream's
# number, since a stream with name "Extra Stream 3" could show
# up after "Extra Stream 29". (Used later to pad numbers with
# 0s).
maximum_digits = len(str(options['extra_streams'] - 1))
for i in range(options['extra_streams']):
# Pad the number with 0s based on `maximum_digits`.
number_str = str(i).zfill(maximum_digits)
extra_stream_name = 'Extra Stream ' + number_str
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream("announce", zulip_realm)
zulip_realm.save(update_fields=['notifications_stream'])
# Add a few default streams
for default_stream_name in ["design", "devel", "social", "support"]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(default_stream_name, zulip_realm))
# Now subscribe everyone to these streams
subscribe_users_to_streams(zulip_realm, zulip_stream_dict)
# These bots are not needed by the test suite
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm, internal_zulip_users_nosubs, bot_type=UserProfile.DEFAULT_BOT)
zulip_cross_realm_bots = [
("Zulip Feedback Bot", "*****@*****.**"),
]
create_users(zulip_realm, zulip_cross_realm_bots, bot_type=UserProfile.DEFAULT_BOT)
# Mark all messages as read
UserMessage.objects.all().update(flags=UserMessage.flags.read)
if not options["test_suite"]:
# Update pointer of each user to point to the last message in their
# UserMessage rows with sender_id=user_profile_id.
users = list(UserMessage.objects.filter(
message__sender_id=F('user_profile_id')).values(
'user_profile_id').annotate(pointer=Max('message_id')))
for user in users:
UserProfile.objects.filter(id=user['user_profile_id']).update(
pointer=user['pointer'])
create_user_groups()
if not options["test_suite"]:
# We populate the analytics database here for
# development purpose only
call_command('populate_analytics_db')
self.stdout.write("Successfully populated test database.\n")
