def test_plan_type(self) -> None:
realm = get_realm('zulip')
realm.plan_type = Realm.STANDARD
realm.save(update_fields=['plan_type'])
self._setup_export_files()
self._export_realm(realm)
with patch('logging.info'):
with self.settings(BILLING_ENABLED=True):
realm = do_import_realm(
get_or_create_dev_uuid_var_path(
'test-backend/test-export'), 'test-zulip-1')
self.assertTrue(realm.plan_type, Realm.LIMITED)
self.assertTrue(
RealmAuditLog.objects.filter(
realm=realm,
event_type=RealmAuditLog.REALM_PLAN_TYPE_CHANGED).
exists())
with self.settings(BILLING_ENABLED=False):
realm = do_import_realm(
get_or_create_dev_uuid_var_path(
'test-backend/test-export'), 'test-zulip-2')
self.assertTrue(realm.plan_type, Realm.SELF_HOSTED)
self.assertTrue(
RealmAuditLog.objects.filter(
realm=realm,
event_type=RealmAuditLog.REALM_PLAN_TYPE_CHANGED).
exists())
def create_test_data() -> None:
gens = load_generators(config) # returns a dictionary of generators
paragraphs = parse_file(config, gens, config["corpus"]["filename"])
write_file(paragraphs, os.path.join(get_or_create_dev_uuid_var_path('test-backend'),
"test_messages.json"))
def set_up_django(external_host: str) -> None:
os.environ['FULL_STACK_ZULIP_TEST'] = '1'
os.environ['EXTERNAL_HOST'] = external_host
os.environ["LOCAL_UPLOADS_DIR"] = get_or_create_dev_uuid_var_path(
'test-backend/test_uploads')
os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.test_settings'
django.setup()
os.environ['PYTHONUNBUFFERED'] = 'y'
def set_up_django(external_host: str) -> None:
os.environ['EXTERNAL_HOST'] = external_host
os.environ["TORNADO_SERVER"] = "http://127.0.0.1:9983"
os.environ["LOCAL_UPLOADS_DIR"] = get_or_create_dev_uuid_var_path(
'test-backend/test_uploads')
os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.test_settings'
django.setup()
os.environ['PYTHONUNBUFFERED'] = 'y'
def set_up_django(external_host: str) -> None:
os.environ["FULL_STACK_ZULIP_TEST"] = "1"
os.environ["TEST_EXTERNAL_HOST"] = external_host
os.environ["LOCAL_UPLOADS_DIR"] = get_or_create_dev_uuid_var_path(
"test-backend/test_uploads")
os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.test_settings"
django.setup()
os.environ["PYTHONUNBUFFERED"] = "y"
def initialize_worker_path(worker_id: int) -> None:
# Allow each test worker process to write to a unique directory
# within `TEST_RUN_DIR`.
worker_path = os.path.join(TEST_RUN_DIR, 'worker_{}'.format(_worker_id))
os.makedirs(worker_path, exist_ok=True)
settings.TEST_WORKER_DIR = worker_path
# Every process should upload to a separate directory so that
# race conditions can be avoided.
settings.LOCAL_UPLOADS_DIR = get_or_create_dev_uuid_var_path(
os.path.join("test-backend", os.path.basename(TEST_RUN_DIR),
os.path.basename(worker_path), "test_uploads"))
def test_import_files_from_s3(self) -> None:
uploads_bucket, avatar_bucket = create_s3_buckets(
settings.S3_AUTH_UPLOADS_BUCKET, settings.S3_AVATAR_BUCKET)
realm = Realm.objects.get(string_id='zulip')
self._setup_export_files()
self._export_realm(realm)
with patch('logging.info'):
do_import_realm(
get_or_create_dev_uuid_var_path('test-backend/test-export'),
'test-zulip')
imported_realm = Realm.objects.get(string_id='test-zulip')
test_image_data = open(get_test_image_file('img.png').name,
'rb').read()
# Test attachments
uploaded_file = Attachment.objects.get(realm=imported_realm)
self.assertEqual(len(b'zulip!'), uploaded_file.size)
attachment_content = uploads_bucket.get_key(
uploaded_file.path_id).get_contents_as_string()
self.assertEqual(b"zulip!", attachment_content)
# Test emojis
realm_emoji = RealmEmoji.objects.get(realm=imported_realm)
emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format(
realm_id=imported_realm.id,
emoji_file_name=realm_emoji.file_name,
)
emoji_key = avatar_bucket.get_key(emoji_path)
self.assertIsNotNone(emoji_key)
self.assertEqual(emoji_key.key, emoji_path)
# Test avatars
user_email = Message.objects.all()[0].sender.email
user_profile = UserProfile.objects.get(email=user_email,
realm=imported_realm)
avatar_path_id = user_avatar_path(user_profile) + ".original"
original_image_key = avatar_bucket.get_key(avatar_path_id)
self.assertEqual(original_image_key.key, avatar_path_id)
image_data = original_image_key.get_contents_as_string()
self.assertEqual(image_data, test_image_data)
def test_import_files_from_local(self) -> None:
realm = Realm.objects.get(string_id='zulip')
self._setup_export_files()
self._export_realm(realm)
with patch('logging.info'):
do_import_realm(
get_or_create_dev_uuid_var_path('test-backend/test-export'),
'test-zulip')
imported_realm = Realm.objects.get(string_id='test-zulip')
# Test attachments
uploaded_file = Attachment.objects.get(realm=imported_realm)
self.assertEqual(len(b'zulip!'), uploaded_file.size)
attachment_file_path = os.path.join(settings.LOCAL_UPLOADS_DIR,
'files', uploaded_file.path_id)
self.assertTrue(os.path.isfile(attachment_file_path))
# Test emojis
realm_emoji = RealmEmoji.objects.get(realm=imported_realm)
emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format(
realm_id=imported_realm.id,
emoji_file_name=realm_emoji.file_name,
)
emoji_file_path = os.path.join(settings.LOCAL_UPLOADS_DIR, "avatars",
emoji_path)
self.assertTrue(os.path.isfile(emoji_file_path))
# Test avatars
user_email = Message.objects.all()[0].sender.email
user_profile = UserProfile.objects.get(email=user_email,
realm=imported_realm)
avatar_path_id = user_avatar_path(user_profile) + ".original"
avatar_file_path = os.path.join(settings.LOCAL_UPLOADS_DIR, "avatars",
avatar_path_id)
self.assertTrue(os.path.isfile(avatar_file_path))
def generate_and_send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open(
os.path.join(get_or_create_dev_uuid_var_path("test-backend"),
"test_messages.json"), "rb") as infile:
dialog = orjson.loads(infile.read())
random.shuffle(dialog)
texts = itertools.cycle(dialog)
# We need to filter out streams from the analytics realm as we don't want to generate
# messages to its streams - and they might also have no subscribers, which would break
# our message generation mechanism below.
stream_ids = Stream.objects.filter(realm=get_realm("zulip")).values_list(
"id", flat=True)
recipient_streams: List[int] = [
recipient.id
for recipient in Recipient.objects.filter(type=Recipient.STREAM,
type_id__in=stream_ids)
]
recipient_huddles: List[int] = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
]
huddle_members: Dict[int, List[int]] = {}
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
# Generate different topics for each stream
possible_topics = {}
for stream_id in recipient_streams:
possible_topics[stream_id] = generate_topics(options["max_topics"])
message_batch_size = options["batch_size"]
num_messages = 0
random_max = 1000000
recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {}
messages: List[Message] = []
while num_messages < tot_messages:
saved_data: Dict[str, Any] = {}
message = Message()
message.sending_client = get_client("populate_db")
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100.0 / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data["personals_pair"]
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data["subject"]
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif randkey <= random_max * options["percent_huddles"] / 100.0:
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.0):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif randkey <= random_max * 1.0:
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data["personals_pair"] = personals_pair
elif message_type == Recipient.STREAM:
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = random.choice(
possible_topics[message.recipient.id])
saved_data["subject"] = message.subject
message.date_sent = choose_date_sent(num_messages, tot_messages,
options["threads"])
messages.append(message)
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
if (num_messages % message_batch_size) == 0:
# Send the batch and empty the list:
send_messages(messages)
messages = []
if len(messages) > 0:
# If there are unsent messages after exiting the loop, send them:
send_messages(messages)
return tot_messages
import unittest
import shutil
from multiprocessing.sharedctypes import Synchronized
from scripts.lib.zulip_tools import get_dev_uuid_var_path, TEMPLATE_DATABASE_DIR, \
get_or_create_dev_uuid_var_path
# We need to pick an ID for this test-backend invocation, and store it
# in this global so it can be used in init_worker; this is used to
# ensure the database IDs we select are unique for each `test-backend`
# run. This probably should use a locking mechanism rather than the
# below hack, which fails 1/10000000 of the time.
random_id_range_start = random.randint(1, 10000000) * 100
# The root directory for this run of the test suite.
TEST_RUN_DIR = get_or_create_dev_uuid_var_path(
os.path.join('test-backend', 'run_{}'.format(random_id_range_start)))
_worker_id = 0 # Used to identify the worker process.
ReturnT = TypeVar('ReturnT') # Constrain return type to match
def slow(
slowness_reason: str
) -> Callable[[Callable[..., ReturnT]], Callable[..., ReturnT]]:
'''
This is a decorate that annotates a test as being "known
to be slow." The decorator will set expected_run_time and slowness_reason
as attributes of the function. Other code can use this annotation
as needed, e.g. to exclude these tests in "fast" mode.
'''
# We need to pick an ID for this test-backend invocation, and store it
# in this global so it can be used in init_worker; this is used to
# ensure the database IDs we select are unique for each `test-backend`
# run. This probably should use a locking mechanism rather than the
# below hack, which fails 1/10000000 of the time.
random_id_range_start = str(random.randint(1, 10000000))
def get_database_id(worker_id: Optional[int] = None) -> str:
if worker_id:
return f"{random_id_range_start}_{worker_id}"
return random_id_range_start
# The root directory for this run of the test suite.
TEST_RUN_DIR = get_or_create_dev_uuid_var_path(
os.path.join("test-backend", f"run_{get_database_id()}"))
_worker_id = 0 # Used to identify the worker process.
class TextTestResult(runner.TextTestResult):
"""
This class has unpythonic function names because base class follows
this style.
"""
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.failed_tests: List[str] = []
def addInfo(self, test: TestCase, msg: str) -> None:
self.stream.write(msg)
# We need to pick an ID for this test-backend invocation, and store it
# in this global so it can be used in init_worker; this is used to
# ensure the database IDs we select are unique for each `test-backend`
# run. This probably should use a locking mechanism rather than the
# below hack, which fails 1/10000000 of the time.
random_id_range_start = str(random.randint(1, 10000000))
def get_database_id(worker_id: Optional[int] = None) -> str:
if worker_id:
return "{}_{}".format(random_id_range_start, worker_id)
return random_id_range_start
# The root directory for this run of the test suite.
TEST_RUN_DIR = get_or_create_dev_uuid_var_path(
os.path.join('test-backend', 'run_{}'.format(get_database_id())))
_worker_id = 0 # Used to identify the worker process.
ReturnT = TypeVar('ReturnT') # Constrain return type to match
def slow(
slowness_reason: str
) -> Callable[[Callable[..., ReturnT]], Callable[..., ReturnT]]:
'''
This is a decorate that annotates a test as being "known
to be slow." The decorator will set expected_run_time and slowness_reason
as attributes of the function. Other code can use this annotation
as needed, e.g. to exclude these tests in "fast" mode.
'''
def send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open(
os.path.join(get_or_create_dev_uuid_var_path('test-backend'),
"test_messages.json"), "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
] # type: List[int]
recipient_huddles = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100. / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + str(random.randint(1, 3))
saved_data['subject'] = message.subject
# Spoofing time not supported with threading
if options['threads'] != 1:
message.pub_date = timezone_now()
else:
# Distrubutes 80% of messages starting from 5 days ago, over a period
# of 3 days. Then, distributes remaining messages over past 24 hours.
spoofed_date = timezone_now() - timezone_timedelta(days=5)
if (num_messages < tot_messages * 0.8):
# Maximum of 3 days ahead, convert to minutes
time_ahead = 3 * 24 * 60
time_ahead //= int(tot_messages * 0.8)
else:
time_ahead = 24 * 60
time_ahead //= int(tot_messages * 0.2)
spoofed_minute = random.randint(time_ahead * num_messages,
time_ahead * (num_messages + 1))
spoofed_date += timezone_timedelta(minutes=spoofed_minute)
message.pub_date = spoofed_date
# We disable USING_RABBITMQ here, so that deferred work is
# executed in do_send_message_messages, rather than being
# queued. This is important, because otherwise, if run-dev.py
# wasn't running when populate_db was run, a developer can end
# up with queued events that reference objects from a previous
# life of the database, which naturally throws exceptions.
settings.USING_RABBITMQ = False
do_send_messages([{'message': message}])
settings.USING_RABBITMQ = True
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
return tot_messages
set_loglevel('zulip.requests', 'CRITICAL')
set_loglevel('zulip.management', 'CRITICAL')
set_loglevel('django.request', 'ERROR')
set_loglevel('fakeldap', 'ERROR')
set_loglevel('zulip.send_email', 'ERROR')
set_loglevel('zerver.lib.push_notifications', 'WARNING')
set_loglevel('zerver.lib.digest', 'ERROR')
set_loglevel('zerver.lib.email_mirror', 'ERROR')
set_loglevel('zerver.worker.queue_processors', 'WARNING')
set_loglevel('stripe', 'WARNING')
# Enable file:/// hyperlink support by default in tests
ENABLE_FILE_LINKS = True
LOCAL_UPLOADS_DIR = get_or_create_dev_uuid_var_path(
'test-backend/test_uploads')
S3_KEY = 'test-key'
S3_SECRET_KEY = 'test-secret-key'
S3_AUTH_UPLOADS_BUCKET = 'test-authed-bucket'
S3_AVATAR_BUCKET = 'test-avatar-bucket'
# Test Custom TOS template rendering
TERMS_OF_SERVICE = 'corporate/terms.md'
INLINE_URL_EMBED_PREVIEW = False
HOME_NOT_LOGGED_IN = '/login/'
LOGIN_URL = '/accounts/login/'
# By default will not send emails when login occurs.
def make_import_output_dir(self, exported_from: str) -> str:
output_dir = tempfile.mkdtemp(
dir=get_or_create_dev_uuid_var_path('test-backend'),
prefix="test-" + exported_from + "-import-")
os.makedirs(output_dir, exist_ok=True)
return output_dir
def create_tarball_path() -> str:
tarball_path = os.path.join(
get_or_create_dev_uuid_var_path('test-backend'), 'test-export.tar.gz')
with open(tarball_path, 'w') as f:
f.write('zulip!')
return tarball_path
def test_import_realm(self) -> None:
original_realm = Realm.objects.get(string_id='zulip')
RealmEmoji.objects.get(realm=original_realm).delete()
# data to test import of huddles
huddle = [self.example_email('hamlet'), self.example_email('othello')]
self.send_huddle_message(self.example_email('cordelia'), huddle,
'test huddle message')
user_mention_message = '@**King Hamlet** Hello'
self.send_stream_message(self.example_email("iago"), "Verona",
user_mention_message)
stream_mention_message = 'Subscribe to #**Denmark**'
self.send_stream_message(self.example_email("hamlet"), "Verona",
stream_mention_message)
user_group_mention_message = 'Hello @*hamletcharacters*'
self.send_stream_message(self.example_email("othello"), "Verona",
user_group_mention_message)
special_characters_message = "```\n'\n```\n@**Polonius**"
self.send_stream_message(self.example_email("iago"), "Denmark",
special_characters_message)
# data to test import of hotspots
sample_user = self.example_user('hamlet')
UserHotspot.objects.create(user=sample_user, hotspot='intro_streams')
# data to test import of muted topic
stream = get_stream(u'Verona', original_realm)
add_topic_mute(user_profile=sample_user,
stream_id=stream.id,
recipient_id=get_stream_recipient(stream.id).id,
topic_name=u'Verona2')
# data to test import of botstoragedata and botconfigdata
bot_profile = do_create_user(email="*****@*****.**",
password="******",
realm=original_realm,
full_name="bot",
short_name="bot",
bot_type=UserProfile.EMBEDDED_BOT,
bot_owner=sample_user)
storage = StateHandler(bot_profile)
storage.put('some key', 'some value')
set_bot_config(bot_profile, 'entry 1', 'value 1')
self._export_realm(original_realm)
with patch('logging.info'):
with self.settings(BILLING_ENABLED=False):
do_import_realm(
get_or_create_dev_uuid_var_path(
'test-backend/test-export'), 'test-zulip')
# sanity checks
# test realm
self.assertTrue(Realm.objects.filter(string_id='test-zulip').exists())
imported_realm = Realm.objects.get(string_id='test-zulip')
self.assertNotEqual(imported_realm.id, original_realm.id)
def assert_realm_values(f: Callable[[Realm], Any],
equal: bool = True) -> None:
orig_realm_result = f(original_realm)
imported_realm_result = f(imported_realm)
# orig_realm_result should be truthy and have some values, otherwise
# the test is kind of meaningless
assert (orig_realm_result)
if equal:
self.assertEqual(orig_realm_result, imported_realm_result)
else:
self.assertNotEqual(orig_realm_result, imported_realm_result)
# test users
assert_realm_values(
lambda r: {user.email
for user in r.get_admin_users_and_bots()})
assert_realm_values(
lambda r: {user.email
for user in r.get_active_users()})
# test stream
assert_realm_values(
lambda r: {stream.name
for stream in get_active_streams(r)})
# test recipients
def get_recipient_stream(r: Realm) -> Stream:
return get_stream_recipient(
Stream.objects.get(name='Verona', realm=r).id)
def get_recipient_user(r: Realm) -> UserProfile:
return get_personal_recipient(
UserProfile.objects.get(full_name='Iago', realm=r).id)
assert_realm_values(lambda r: get_recipient_stream(r).type)
assert_realm_values(lambda r: get_recipient_user(r).type)
# test subscription
def get_subscribers(recipient: Recipient) -> Set[str]:
subscriptions = Subscription.objects.filter(recipient=recipient)
users = {sub.user_profile.email for sub in subscriptions}
return users
assert_realm_values(lambda r: get_subscribers(get_recipient_stream(r)))
assert_realm_values(lambda r: get_subscribers(get_recipient_user(r)))
# test custom profile fields
def get_custom_profile_field_names(r: Realm) -> Set[str]:
custom_profile_fields = CustomProfileField.objects.filter(realm=r)
custom_profile_field_names = {
field.name
for field in custom_profile_fields
}
return custom_profile_field_names
assert_realm_values(get_custom_profile_field_names)
def get_custom_profile_with_field_type_user(
r: Realm) -> Tuple[Set[Any], Set[Any], Set[FrozenSet[str]]]:
fields = CustomProfileField.objects.filter(
field_type=CustomProfileField.USER, realm=r)
def get_email(user_id: int) -> str:
return UserProfile.objects.get(id=user_id).email
def get_email_from_value(
field_value: CustomProfileFieldValue) -> Set[str]:
user_id_list = ujson.loads(field_value.value)
return {get_email(user_id) for user_id in user_id_list}
def custom_profile_field_values_for(
fields: List[CustomProfileField]) -> Set[FrozenSet[str]]:
user_emails = set() # type: Set[FrozenSet[str]]
for field in fields:
values = CustomProfileFieldValue.objects.filter(
field=field)
for value in values:
user_emails.add(frozenset(get_email_from_value(value)))
return user_emails
field_names, field_hints = (set() for i in range(2))
for field in fields:
field_names.add(field.name)
field_hints.add(field.hint)
return (field_hints, field_names,
custom_profile_field_values_for(fields))
assert_realm_values(get_custom_profile_with_field_type_user)
# test realmauditlog
def get_realm_audit_log_event_type(r: Realm) -> Set[str]:
realmauditlogs = RealmAuditLog.objects.filter(realm=r).exclude(
event_type=RealmAuditLog.REALM_PLAN_TYPE_CHANGED)
realmauditlog_event_type = {
log.event_type
for log in realmauditlogs
}
return realmauditlog_event_type
assert_realm_values(get_realm_audit_log_event_type)
# test huddles
def get_huddle_hashes(r: str) -> str:
short_names = ['cordelia', 'hamlet', 'othello']
user_id_list = [
UserProfile.objects.get(realm=r, short_name=name).id
for name in short_names
]
huddle_hash = get_huddle_hash(user_id_list)
return huddle_hash
assert_realm_values(get_huddle_hashes, equal=False)
def get_huddle_message(r: str) -> str:
huddle_hash = get_huddle_hashes(r)
huddle_id = Huddle.objects.get(huddle_hash=huddle_hash).id
huddle_recipient = Recipient.objects.get(type_id=huddle_id, type=3)
huddle_message = Message.objects.get(recipient=huddle_recipient)
return huddle_message.content
assert_realm_values(get_huddle_message)
self.assertEqual(get_huddle_message(imported_realm),
'test huddle message')
# test userhotspot
def get_user_hotspots(r: str) -> Set[str]:
user_profile = UserProfile.objects.get(realm=r,
short_name='hamlet')
hotspots = UserHotspot.objects.filter(user=user_profile)
user_hotspots = {hotspot.hotspot for hotspot in hotspots}
return user_hotspots
assert_realm_values(get_user_hotspots)
# test muted topics
def get_muted_topics(r: Realm) -> Set[str]:
user_profile = UserProfile.objects.get(realm=r,
short_name='hamlet')
muted_topics = MutedTopic.objects.filter(user_profile=user_profile)
topic_names = {
muted_topic.topic_name
for muted_topic in muted_topics
}
return topic_names
assert_realm_values(get_muted_topics)
# test usergroups
assert_realm_values(
lambda r:
{group.name
for group in UserGroup.objects.filter(realm=r)})
def get_user_membership(r: str) -> Set[str]:
usergroup = UserGroup.objects.get(realm=r, name='hamletcharacters')
usergroup_membership = UserGroupMembership.objects.filter(
user_group=usergroup)
users = {
membership.user_profile.email
for membership in usergroup_membership
}
return users
assert_realm_values(get_user_membership)
# test botstoragedata and botconfigdata
def get_botstoragedata(r: Realm) -> Dict[str, Any]:
bot_profile = UserProfile.objects.get(full_name="bot", realm=r)
bot_storage_data = BotStorageData.objects.get(
bot_profile=bot_profile)
return {
'key': bot_storage_data.key,
'data': bot_storage_data.value
}
assert_realm_values(get_botstoragedata)
def get_botconfigdata(r: Realm) -> Dict[str, Any]:
bot_profile = UserProfile.objects.get(full_name="bot", realm=r)
bot_config_data = BotConfigData.objects.get(
bot_profile=bot_profile)
return {'key': bot_config_data.key, 'data': bot_config_data.value}
assert_realm_values(get_botconfigdata)
# test messages
def get_stream_messages(r: Realm) -> Message:
recipient = get_recipient_stream(r)
messages = Message.objects.filter(recipient=recipient)
return messages
def get_stream_topics(r: Realm) -> Set[str]:
messages = get_stream_messages(r)
topics = {m.topic_name() for m in messages}
return topics
assert_realm_values(get_stream_topics)
# test usermessages
def get_usermessages_user(r: Realm) -> Set[Any]:
messages = get_stream_messages(r).order_by('content')
usermessage = UserMessage.objects.filter(message=messages[0])
usermessage_user = {um.user_profile.email for um in usermessage}
return usermessage_user
assert_realm_values(get_usermessages_user)
# tests to make sure that various data-*-ids in rendered_content
# are replaced correctly with the values of newer realm.
def get_user_mention(r: Realm) -> Set[Any]:
mentioned_user = UserProfile.objects.get(
delivery_email=self.example_email("hamlet"), realm=r)
data_user_id = 'data-user-id="{}"'.format(mentioned_user.id)
mention_message = get_stream_messages(r).get(
rendered_content__contains=data_user_id)
return mention_message.content
assert_realm_values(get_user_mention)
def get_stream_mention(r: Realm) -> Set[Any]:
mentioned_stream = get_stream(u'Denmark', r)
data_stream_id = 'data-stream-id="{}"'.format(mentioned_stream.id)
mention_message = get_stream_messages(r).get(
rendered_content__contains=data_stream_id)
return mention_message.content
assert_realm_values(get_stream_mention)
def get_user_group_mention(r: Realm) -> Set[Any]:
user_group = UserGroup.objects.get(realm=r,
name='hamletcharacters')
data_usergroup_id = 'data-user-group-id="{}"'.format(user_group.id)
mention_message = get_stream_messages(r).get(
rendered_content__contains=data_usergroup_id)
return mention_message.content
assert_realm_values(get_user_group_mention)
# test to highlight that bs4 which we use to do data-**id
# replacements modifies the HTML sometimes. eg replacing <br>
# with </br>, ' with \' etc. The modifications doesn't
# affect how the browser displays the rendered_content so we
# are okay with using bs4 for this. lxml package also has
# similar behavior.
orig_polonius_user = UserProfile.objects.get(
email=self.example_email("polonius"), realm=original_realm)
original_msg = Message.objects.get(content=special_characters_message,
sender__realm=original_realm)
self.assertEqual(original_msg.rendered_content, (
'<div class="codehilite"><pre><span></span>'\n</pre></div>\n\n\n'
'<p><span class="user-mention" data-user-id="%s">@Polonius</span></p>'
% (orig_polonius_user.id, )))
imported_polonius_user = UserProfile.objects.get(
email=self.example_email("polonius"), realm=imported_realm)
imported_msg = Message.objects.get(content=special_characters_message,
sender__realm=imported_realm)
self.assertEqual(imported_msg.rendered_content, (
'<div class="codehilite"><pre><span></span>\'\n</pre></div>\n'
'<p><span class="user-mention" data-user-id="%s">@Polonius</span></p>'
% (imported_polonius_user.id, )))
def _make_output_dir(self) -> str:
output_dir = get_or_create_dev_uuid_var_path(
'test-backend/test-export')
self.rm_tree(output_dir)
os.makedirs(output_dir, exist_ok=True)
return output_dir
# We need to pick an ID for this test-backend invocation, and store it
# in this global so it can be used in init_worker; this is used to
# ensure the database IDs we select are unique for each `test-backend`
# run. This probably should use a locking mechanism rather than the
# below hack, which fails 1/10000000 of the time.
random_id_range_start = str(random.randint(1, 10000000))
def get_database_id(worker_id: Optional[int] = None) -> str:
if worker_id:
return f"{random_id_range_start}_{worker_id}"
return random_id_range_start
# The root directory for this run of the test suite.
TEST_RUN_DIR = get_or_create_dev_uuid_var_path(
os.path.join('test-backend', f'run_{get_database_id()}'))
_worker_id = 0 # Used to identify the worker process.
class TextTestResult(runner.TextTestResult):
"""
This class has unpythonic function names because base class follows
this style.
"""
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.failed_tests: List[str] = []
def addInfo(self, test: TestCase, msg: str) -> None:
self.stream.write(msg)
def generate_and_send_messages(data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open(os.path.join(get_or_create_dev_uuid_var_path('test-backend'),
"test_messages.json")) as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams: List[int] = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
recipient_huddles: List[int] = [h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)]
huddle_members: Dict[int, List[int]] = {}
for h in recipient_huddles:
huddle_members[h] = [s.user_profile.id for s in
Subscription.objects.filter(recipient_id=h)]
# Generate different topics for each stream
possible_topics = dict()
for stream_id in recipient_streams:
possible_topics[stream_id] = generate_topics(options["max_topics"])
message_batch_size = options['batch_size']
num_messages = 0
random_max = 1000000
recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {}
messages = []
messages_add_reaction = []
while num_messages < tot_messages:
saved_data: Dict[str, Any] = {}
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0 and
random.randint(1, random_max) * 100. / random_max < options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages - 1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(random.choice(recipient_huddles))
elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(type=Recipient.PERSONAL,
type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
# Pick a random subscriber to the stream
message.sender = random.choice(Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = random.choice(possible_topics[message.recipient.id])
saved_data['subject'] = message.subject
message.date_sent = choose_date_sent(num_messages, tot_messages, options['threads'])
messages.append(message)
messages_add_reaction.append(message)
recipients[num_messages] = (message_type, message.recipient.id, saved_data)
num_messages += 1
if (num_messages % message_batch_size) == 0:
# Send the batch and empty the list:
send_messages(messages)
messages = []
if len(messages) > 0:
# If there are unsent messages after exiting the loop, send them:
send_messages(messages)
reactions_message = []
add_emojis = ["1f44d", "1f642", "1f60a"]
for rmessage in messages_add_reaction:
if random.random() > 0.9:
reactedmessage = Reaction(user_profile=rmessage.sender,
message=rmessage,
emoji_name="+1",
emoji_code=add_emojis[random.randint(0, 2)],
reaction_type="unicode_emoji")
reactions_message.append(reactedmessage)
Reaction.objects.bulk_create(reactions_message)
return tot_messages
def generate_and_send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open(
os.path.join(get_or_create_dev_uuid_var_path('test-backend'),
"test_messages.json"), "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
] # type: List[int]
recipient_huddles = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
message_batch_size = options['batch_size']
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
messages = []
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100. / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + str(random.randint(1, 3))
saved_data['subject'] = message.subject
message.date_sent = choose_date_sent(num_messages, tot_messages,
options['threads'])
messages.append(message)
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
if (num_messages % message_batch_size) == 0:
# Send the batch and empty the list:
send_messages(messages)
messages = []
if len(messages) > 0:
# If there are unsent messages after exiting the loop, send them:
send_messages(messages)
return tot_messages
def test_convert_slack_workspace_messages(
self, mock_get_messages_iterator: mock.Mock,
mock_message: mock.Mock) -> None:
output_dir = get_or_create_dev_uuid_var_path(
'test-backend/test-slack-import')
added_channels = {
'random': ('c5', 1),
'general': ('c6', 2)
} # type: Dict[str, Tuple[str, int]]
time = float(timezone_now().timestamp())
zerver_message = [{'id': 1, 'ts': time}, {'id': 5, 'ts': time}]
def fake_get_messages_iter(
slack_data_dir: str, added_channels: AddedChannelsT,
added_mpims: AddedMPIMsT) -> Iterator[ZerverFieldsT]:
import copy
return iter(copy.deepcopy(zerver_message))
realm = {'zerver_subscription': []} # type: Dict[str, Any]
user_list = [] # type: List[Dict[str, Any]]
reactions = [{"name": "grinning", "users": ["U061A5N1G"], "count": 1}]
attachments = uploads = [] # type: List[Dict[str, Any]]
zerver_usermessage = [{'id': 3}, {'id': 5}, {'id': 6}, {'id': 9}]
mock_get_messages_iterator.side_effect = fake_get_messages_iter
mock_message.side_effect = [[
zerver_message[:1], zerver_usermessage[:2], attachments, uploads,
reactions[:1]
],
[
zerver_message[1:2],
zerver_usermessage[2:5], attachments,
uploads, reactions[1:1]
]]
# Hacky: We should include a zerver_userprofile, not the empty []
test_reactions, uploads, zerver_attachment = convert_slack_workspace_messages(
'./random_path',
user_list,
2, {}, {},
added_channels, {},
realm, [], [],
'domain',
output_dir=output_dir,
chunk_size=1)
messages_file_1 = os.path.join(output_dir, 'messages-000001.json')
self.assertTrue(os.path.exists(messages_file_1))
messages_file_2 = os.path.join(output_dir, 'messages-000002.json')
self.assertTrue(os.path.exists(messages_file_2))
with open(messages_file_1) as f:
message_json = ujson.load(f)
self.assertEqual(message_json['zerver_message'], zerver_message[:1])
self.assertEqual(message_json['zerver_usermessage'],
zerver_usermessage[:2])
with open(messages_file_2) as f:
message_json = ujson.load(f)
self.assertEqual(message_json['zerver_message'], zerver_message[1:2])
self.assertEqual(message_json['zerver_usermessage'],
zerver_usermessage[2:5])
self.assertEqual(test_reactions, reactions)
