def test_plan_type(self) -> None: realm = get_realm('zulip') realm.plan_type = Realm.STANDARD realm.save(update_fields=['plan_type']) self._setup_export_files() self._export_realm(realm) with patch('logging.info'): with self.settings(BILLING_ENABLED=True): realm = do_import_realm( get_or_create_dev_uuid_var_path( 'test-backend/test-export'), 'test-zulip-1') self.assertTrue(realm.plan_type, Realm.LIMITED) self.assertTrue( RealmAuditLog.objects.filter( realm=realm, event_type=RealmAuditLog.REALM_PLAN_TYPE_CHANGED). exists()) with self.settings(BILLING_ENABLED=False): realm = do_import_realm( get_or_create_dev_uuid_var_path( 'test-backend/test-export'), 'test-zulip-2') self.assertTrue(realm.plan_type, Realm.SELF_HOSTED) self.assertTrue( RealmAuditLog.objects.filter( realm=realm, event_type=RealmAuditLog.REALM_PLAN_TYPE_CHANGED). exists())
def create_test_data() -> None: gens = load_generators(config) # returns a dictionary of generators paragraphs = parse_file(config, gens, config["corpus"]["filename"]) write_file(paragraphs, os.path.join(get_or_create_dev_uuid_var_path('test-backend'), "test_messages.json"))
def set_up_django(external_host: str) -> None: os.environ['FULL_STACK_ZULIP_TEST'] = '1' os.environ['EXTERNAL_HOST'] = external_host os.environ["LOCAL_UPLOADS_DIR"] = get_or_create_dev_uuid_var_path( 'test-backend/test_uploads') os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.test_settings' django.setup() os.environ['PYTHONUNBUFFERED'] = 'y'
def set_up_django(external_host: str) -> None: os.environ['EXTERNAL_HOST'] = external_host os.environ["TORNADO_SERVER"] = "http://127.0.0.1:9983" os.environ["LOCAL_UPLOADS_DIR"] = get_or_create_dev_uuid_var_path( 'test-backend/test_uploads') os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.test_settings' django.setup() os.environ['PYTHONUNBUFFERED'] = 'y'
def set_up_django(external_host: str) -> None: os.environ["FULL_STACK_ZULIP_TEST"] = "1" os.environ["TEST_EXTERNAL_HOST"] = external_host os.environ["LOCAL_UPLOADS_DIR"] = get_or_create_dev_uuid_var_path( "test-backend/test_uploads") os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.test_settings" django.setup() os.environ["PYTHONUNBUFFERED"] = "y"
def initialize_worker_path(worker_id: int) -> None: # Allow each test worker process to write to a unique directory # within `TEST_RUN_DIR`. worker_path = os.path.join(TEST_RUN_DIR, 'worker_{}'.format(_worker_id)) os.makedirs(worker_path, exist_ok=True) settings.TEST_WORKER_DIR = worker_path # Every process should upload to a separate directory so that # race conditions can be avoided. settings.LOCAL_UPLOADS_DIR = get_or_create_dev_uuid_var_path( os.path.join("test-backend", os.path.basename(TEST_RUN_DIR), os.path.basename(worker_path), "test_uploads"))
def test_import_files_from_s3(self) -> None: uploads_bucket, avatar_bucket = create_s3_buckets( settings.S3_AUTH_UPLOADS_BUCKET, settings.S3_AVATAR_BUCKET) realm = Realm.objects.get(string_id='zulip') self._setup_export_files() self._export_realm(realm) with patch('logging.info'): do_import_realm( get_or_create_dev_uuid_var_path('test-backend/test-export'), 'test-zulip') imported_realm = Realm.objects.get(string_id='test-zulip') test_image_data = open(get_test_image_file('img.png').name, 'rb').read() # Test attachments uploaded_file = Attachment.objects.get(realm=imported_realm) self.assertEqual(len(b'zulip!'), uploaded_file.size) attachment_content = uploads_bucket.get_key( uploaded_file.path_id).get_contents_as_string() self.assertEqual(b"zulip!", attachment_content) # Test emojis realm_emoji = RealmEmoji.objects.get(realm=imported_realm) emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format( realm_id=imported_realm.id, emoji_file_name=realm_emoji.file_name, ) emoji_key = avatar_bucket.get_key(emoji_path) self.assertIsNotNone(emoji_key) self.assertEqual(emoji_key.key, emoji_path) # Test avatars user_email = Message.objects.all()[0].sender.email user_profile = UserProfile.objects.get(email=user_email, realm=imported_realm) avatar_path_id = user_avatar_path(user_profile) + ".original" original_image_key = avatar_bucket.get_key(avatar_path_id) self.assertEqual(original_image_key.key, avatar_path_id) image_data = original_image_key.get_contents_as_string() self.assertEqual(image_data, test_image_data)
def test_import_files_from_local(self) -> None: realm = Realm.objects.get(string_id='zulip') self._setup_export_files() self._export_realm(realm) with patch('logging.info'): do_import_realm( get_or_create_dev_uuid_var_path('test-backend/test-export'), 'test-zulip') imported_realm = Realm.objects.get(string_id='test-zulip') # Test attachments uploaded_file = Attachment.objects.get(realm=imported_realm) self.assertEqual(len(b'zulip!'), uploaded_file.size) attachment_file_path = os.path.join(settings.LOCAL_UPLOADS_DIR, 'files', uploaded_file.path_id) self.assertTrue(os.path.isfile(attachment_file_path)) # Test emojis realm_emoji = RealmEmoji.objects.get(realm=imported_realm) emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format( realm_id=imported_realm.id, emoji_file_name=realm_emoji.file_name, ) emoji_file_path = os.path.join(settings.LOCAL_UPLOADS_DIR, "avatars", emoji_path) self.assertTrue(os.path.isfile(emoji_file_path)) # Test avatars user_email = Message.objects.all()[0].sender.email user_profile = UserProfile.objects.get(email=user_email, realm=imported_realm) avatar_path_id = user_avatar_path(user_profile) + ".original" avatar_file_path = os.path.join(settings.LOCAL_UPLOADS_DIR, "avatars", avatar_path_id) self.assertTrue(os.path.isfile(avatar_file_path))
def generate_and_send_messages( data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int] ) -> int: (tot_messages, personals_pairs, options, output, random_seed) = data random.seed(random_seed) with open( os.path.join(get_or_create_dev_uuid_var_path("test-backend"), "test_messages.json"), "rb") as infile: dialog = orjson.loads(infile.read()) random.shuffle(dialog) texts = itertools.cycle(dialog) # We need to filter out streams from the analytics realm as we don't want to generate # messages to its streams - and they might also have no subscribers, which would break # our message generation mechanism below. stream_ids = Stream.objects.filter(realm=get_realm("zulip")).values_list( "id", flat=True) recipient_streams: List[int] = [ recipient.id for recipient in Recipient.objects.filter(type=Recipient.STREAM, type_id__in=stream_ids) ] recipient_huddles: List[int] = [ h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE) ] huddle_members: Dict[int, List[int]] = {} for h in recipient_huddles: huddle_members[h] = [ s.user_profile.id for s in Subscription.objects.filter(recipient_id=h) ] # Generate different topics for each stream possible_topics = {} for stream_id in recipient_streams: possible_topics[stream_id] = generate_topics(options["max_topics"]) message_batch_size = options["batch_size"] num_messages = 0 random_max = 1000000 recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {} messages: List[Message] = [] while num_messages < tot_messages: saved_data: Dict[str, Any] = {} message = Message() message.sending_client = get_client("populate_db") message.content = next(texts) randkey = random.randint(1, random_max) if (num_messages > 0 and random.randint(1, random_max) * 100.0 / random_max < options["stickyness"]): # Use an old recipient message_type, recipient_id, saved_data = recipients[num_messages - 1] if message_type == Recipient.PERSONAL: personals_pair = saved_data["personals_pair"] random.shuffle(personals_pair) elif message_type == Recipient.STREAM: message.subject = saved_data["subject"] message.recipient = get_recipient_by_id(recipient_id) elif message_type == Recipient.HUDDLE: message.recipient = get_recipient_by_id(recipient_id) elif randkey <= random_max * options["percent_huddles"] / 100.0: message_type = Recipient.HUDDLE message.recipient = get_recipient_by_id( random.choice(recipient_huddles)) elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.0): message_type = Recipient.PERSONAL personals_pair = random.choice(personals_pairs) random.shuffle(personals_pair) elif randkey <= random_max * 1.0: message_type = Recipient.STREAM message.recipient = get_recipient_by_id( random.choice(recipient_streams)) if message_type == Recipient.HUDDLE: sender_id = random.choice(huddle_members[message.recipient.id]) message.sender = get_user_profile_by_id(sender_id) elif message_type == Recipient.PERSONAL: message.recipient = Recipient.objects.get( type=Recipient.PERSONAL, type_id=personals_pair[0]) message.sender = get_user_profile_by_id(personals_pair[1]) saved_data["personals_pair"] = personals_pair elif message_type == Recipient.STREAM: # Pick a random subscriber to the stream message.sender = random.choice( Subscription.objects.filter( recipient=message.recipient)).user_profile message.subject = random.choice( possible_topics[message.recipient.id]) saved_data["subject"] = message.subject message.date_sent = choose_date_sent(num_messages, tot_messages, options["threads"]) messages.append(message) recipients[num_messages] = (message_type, message.recipient.id, saved_data) num_messages += 1 if (num_messages % message_batch_size) == 0: # Send the batch and empty the list: send_messages(messages) messages = [] if len(messages) > 0: # If there are unsent messages after exiting the loop, send them: send_messages(messages) return tot_messages
import unittest import shutil from multiprocessing.sharedctypes import Synchronized from scripts.lib.zulip_tools import get_dev_uuid_var_path, TEMPLATE_DATABASE_DIR, \ get_or_create_dev_uuid_var_path # We need to pick an ID for this test-backend invocation, and store it # in this global so it can be used in init_worker; this is used to # ensure the database IDs we select are unique for each `test-backend` # run. This probably should use a locking mechanism rather than the # below hack, which fails 1/10000000 of the time. random_id_range_start = random.randint(1, 10000000) * 100 # The root directory for this run of the test suite. TEST_RUN_DIR = get_or_create_dev_uuid_var_path( os.path.join('test-backend', 'run_{}'.format(random_id_range_start))) _worker_id = 0 # Used to identify the worker process. ReturnT = TypeVar('ReturnT') # Constrain return type to match def slow( slowness_reason: str ) -> Callable[[Callable[..., ReturnT]], Callable[..., ReturnT]]: ''' This is a decorate that annotates a test as being "known to be slow." The decorator will set expected_run_time and slowness_reason as attributes of the function. Other code can use this annotation as needed, e.g. to exclude these tests in "fast" mode. '''
# We need to pick an ID for this test-backend invocation, and store it # in this global so it can be used in init_worker; this is used to # ensure the database IDs we select are unique for each `test-backend` # run. This probably should use a locking mechanism rather than the # below hack, which fails 1/10000000 of the time. random_id_range_start = str(random.randint(1, 10000000)) def get_database_id(worker_id: Optional[int] = None) -> str: if worker_id: return f"{random_id_range_start}_{worker_id}" return random_id_range_start # The root directory for this run of the test suite. TEST_RUN_DIR = get_or_create_dev_uuid_var_path( os.path.join("test-backend", f"run_{get_database_id()}")) _worker_id = 0 # Used to identify the worker process. class TextTestResult(runner.TextTestResult): """ This class has unpythonic function names because base class follows this style. """ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self.failed_tests: List[str] = [] def addInfo(self, test: TestCase, msg: str) -> None: self.stream.write(msg)
# We need to pick an ID for this test-backend invocation, and store it # in this global so it can be used in init_worker; this is used to # ensure the database IDs we select are unique for each `test-backend` # run. This probably should use a locking mechanism rather than the # below hack, which fails 1/10000000 of the time. random_id_range_start = str(random.randint(1, 10000000)) def get_database_id(worker_id: Optional[int] = None) -> str: if worker_id: return "{}_{}".format(random_id_range_start, worker_id) return random_id_range_start # The root directory for this run of the test suite. TEST_RUN_DIR = get_or_create_dev_uuid_var_path( os.path.join('test-backend', 'run_{}'.format(get_database_id()))) _worker_id = 0 # Used to identify the worker process. ReturnT = TypeVar('ReturnT') # Constrain return type to match def slow( slowness_reason: str ) -> Callable[[Callable[..., ReturnT]], Callable[..., ReturnT]]: ''' This is a decorate that annotates a test as being "known to be slow." The decorator will set expected_run_time and slowness_reason as attributes of the function. Other code can use this annotation as needed, e.g. to exclude these tests in "fast" mode. '''
def send_messages( data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int] ) -> int: (tot_messages, personals_pairs, options, output, random_seed) = data random.seed(random_seed) with open( os.path.join(get_or_create_dev_uuid_var_path('test-backend'), "test_messages.json"), "r") as infile: dialog = ujson.load(infile) random.shuffle(dialog) texts = itertools.cycle(dialog) recipient_streams = [ klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] # type: List[int] recipient_huddles = [ h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE) ] # type: List[int] huddle_members = {} # type: Dict[int, List[int]] for h in recipient_huddles: huddle_members[h] = [ s.user_profile.id for s in Subscription.objects.filter(recipient_id=h) ] num_messages = 0 random_max = 1000000 recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]] while num_messages < tot_messages: saved_data = {} # type: Dict[str, Any] message = Message() message.sending_client = get_client('populate_db') message.content = next(texts) randkey = random.randint(1, random_max) if (num_messages > 0 and random.randint(1, random_max) * 100. / random_max < options["stickyness"]): # Use an old recipient message_type, recipient_id, saved_data = recipients[num_messages - 1] if message_type == Recipient.PERSONAL: personals_pair = saved_data['personals_pair'] random.shuffle(personals_pair) elif message_type == Recipient.STREAM: message.subject = saved_data['subject'] message.recipient = get_recipient_by_id(recipient_id) elif message_type == Recipient.HUDDLE: message.recipient = get_recipient_by_id(recipient_id) elif (randkey <= random_max * options["percent_huddles"] / 100.): message_type = Recipient.HUDDLE message.recipient = get_recipient_by_id( random.choice(recipient_huddles)) elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.): message_type = Recipient.PERSONAL personals_pair = random.choice(personals_pairs) random.shuffle(personals_pair) elif (randkey <= random_max * 1.0): message_type = Recipient.STREAM message.recipient = get_recipient_by_id( random.choice(recipient_streams)) if message_type == Recipient.HUDDLE: sender_id = random.choice(huddle_members[message.recipient.id]) message.sender = get_user_profile_by_id(sender_id) elif message_type == Recipient.PERSONAL: message.recipient = Recipient.objects.get( type=Recipient.PERSONAL, type_id=personals_pair[0]) message.sender = get_user_profile_by_id(personals_pair[1]) saved_data['personals_pair'] = personals_pair elif message_type == Recipient.STREAM: stream = Stream.objects.get(id=message.recipient.type_id) # Pick a random subscriber to the stream message.sender = random.choice( Subscription.objects.filter( recipient=message.recipient)).user_profile message.subject = stream.name + str(random.randint(1, 3)) saved_data['subject'] = message.subject # Spoofing time not supported with threading if options['threads'] != 1: message.pub_date = timezone_now() else: # Distrubutes 80% of messages starting from 5 days ago, over a period # of 3 days. Then, distributes remaining messages over past 24 hours. spoofed_date = timezone_now() - timezone_timedelta(days=5) if (num_messages < tot_messages * 0.8): # Maximum of 3 days ahead, convert to minutes time_ahead = 3 * 24 * 60 time_ahead //= int(tot_messages * 0.8) else: time_ahead = 24 * 60 time_ahead //= int(tot_messages * 0.2) spoofed_minute = random.randint(time_ahead * num_messages, time_ahead * (num_messages + 1)) spoofed_date += timezone_timedelta(minutes=spoofed_minute) message.pub_date = spoofed_date # We disable USING_RABBITMQ here, so that deferred work is # executed in do_send_message_messages, rather than being # queued. This is important, because otherwise, if run-dev.py # wasn't running when populate_db was run, a developer can end # up with queued events that reference objects from a previous # life of the database, which naturally throws exceptions. settings.USING_RABBITMQ = False do_send_messages([{'message': message}]) settings.USING_RABBITMQ = True recipients[num_messages] = (message_type, message.recipient.id, saved_data) num_messages += 1 return tot_messages
set_loglevel('zulip.requests', 'CRITICAL') set_loglevel('zulip.management', 'CRITICAL') set_loglevel('django.request', 'ERROR') set_loglevel('fakeldap', 'ERROR') set_loglevel('zulip.send_email', 'ERROR') set_loglevel('zerver.lib.push_notifications', 'WARNING') set_loglevel('zerver.lib.digest', 'ERROR') set_loglevel('zerver.lib.email_mirror', 'ERROR') set_loglevel('zerver.worker.queue_processors', 'WARNING') set_loglevel('stripe', 'WARNING') # Enable file:/// hyperlink support by default in tests ENABLE_FILE_LINKS = True LOCAL_UPLOADS_DIR = get_or_create_dev_uuid_var_path( 'test-backend/test_uploads') S3_KEY = 'test-key' S3_SECRET_KEY = 'test-secret-key' S3_AUTH_UPLOADS_BUCKET = 'test-authed-bucket' S3_AVATAR_BUCKET = 'test-avatar-bucket' # Test Custom TOS template rendering TERMS_OF_SERVICE = 'corporate/terms.md' INLINE_URL_EMBED_PREVIEW = False HOME_NOT_LOGGED_IN = '/login/' LOGIN_URL = '/accounts/login/' # By default will not send emails when login occurs.
def make_import_output_dir(self, exported_from: str) -> str: output_dir = tempfile.mkdtemp( dir=get_or_create_dev_uuid_var_path('test-backend'), prefix="test-" + exported_from + "-import-") os.makedirs(output_dir, exist_ok=True) return output_dir
def create_tarball_path() -> str: tarball_path = os.path.join( get_or_create_dev_uuid_var_path('test-backend'), 'test-export.tar.gz') with open(tarball_path, 'w') as f: f.write('zulip!') return tarball_path
def test_import_realm(self) -> None: original_realm = Realm.objects.get(string_id='zulip') RealmEmoji.objects.get(realm=original_realm).delete() # data to test import of huddles huddle = [self.example_email('hamlet'), self.example_email('othello')] self.send_huddle_message(self.example_email('cordelia'), huddle, 'test huddle message') user_mention_message = '@**King Hamlet** Hello' self.send_stream_message(self.example_email("iago"), "Verona", user_mention_message) stream_mention_message = 'Subscribe to #**Denmark**' self.send_stream_message(self.example_email("hamlet"), "Verona", stream_mention_message) user_group_mention_message = 'Hello @*hamletcharacters*' self.send_stream_message(self.example_email("othello"), "Verona", user_group_mention_message) special_characters_message = "```\n'\n```\n@**Polonius**" self.send_stream_message(self.example_email("iago"), "Denmark", special_characters_message) # data to test import of hotspots sample_user = self.example_user('hamlet') UserHotspot.objects.create(user=sample_user, hotspot='intro_streams') # data to test import of muted topic stream = get_stream(u'Verona', original_realm) add_topic_mute(user_profile=sample_user, stream_id=stream.id, recipient_id=get_stream_recipient(stream.id).id, topic_name=u'Verona2') # data to test import of botstoragedata and botconfigdata bot_profile = do_create_user(email="*****@*****.**", password="******", realm=original_realm, full_name="bot", short_name="bot", bot_type=UserProfile.EMBEDDED_BOT, bot_owner=sample_user) storage = StateHandler(bot_profile) storage.put('some key', 'some value') set_bot_config(bot_profile, 'entry 1', 'value 1') self._export_realm(original_realm) with patch('logging.info'): with self.settings(BILLING_ENABLED=False): do_import_realm( get_or_create_dev_uuid_var_path( 'test-backend/test-export'), 'test-zulip') # sanity checks # test realm self.assertTrue(Realm.objects.filter(string_id='test-zulip').exists()) imported_realm = Realm.objects.get(string_id='test-zulip') self.assertNotEqual(imported_realm.id, original_realm.id) def assert_realm_values(f: Callable[[Realm], Any], equal: bool = True) -> None: orig_realm_result = f(original_realm) imported_realm_result = f(imported_realm) # orig_realm_result should be truthy and have some values, otherwise # the test is kind of meaningless assert (orig_realm_result) if equal: self.assertEqual(orig_realm_result, imported_realm_result) else: self.assertNotEqual(orig_realm_result, imported_realm_result) # test users assert_realm_values( lambda r: {user.email for user in r.get_admin_users_and_bots()}) assert_realm_values( lambda r: {user.email for user in r.get_active_users()}) # test stream assert_realm_values( lambda r: {stream.name for stream in get_active_streams(r)}) # test recipients def get_recipient_stream(r: Realm) -> Stream: return get_stream_recipient( Stream.objects.get(name='Verona', realm=r).id) def get_recipient_user(r: Realm) -> UserProfile: return get_personal_recipient( UserProfile.objects.get(full_name='Iago', realm=r).id) assert_realm_values(lambda r: get_recipient_stream(r).type) assert_realm_values(lambda r: get_recipient_user(r).type) # test subscription def get_subscribers(recipient: Recipient) -> Set[str]: subscriptions = Subscription.objects.filter(recipient=recipient) users = {sub.user_profile.email for sub in subscriptions} return users assert_realm_values(lambda r: get_subscribers(get_recipient_stream(r))) assert_realm_values(lambda r: get_subscribers(get_recipient_user(r))) # test custom profile fields def get_custom_profile_field_names(r: Realm) -> Set[str]: custom_profile_fields = CustomProfileField.objects.filter(realm=r) custom_profile_field_names = { field.name for field in custom_profile_fields } return custom_profile_field_names assert_realm_values(get_custom_profile_field_names) def get_custom_profile_with_field_type_user( r: Realm) -> Tuple[Set[Any], Set[Any], Set[FrozenSet[str]]]: fields = CustomProfileField.objects.filter( field_type=CustomProfileField.USER, realm=r) def get_email(user_id: int) -> str: return UserProfile.objects.get(id=user_id).email def get_email_from_value( field_value: CustomProfileFieldValue) -> Set[str]: user_id_list = ujson.loads(field_value.value) return {get_email(user_id) for user_id in user_id_list} def custom_profile_field_values_for( fields: List[CustomProfileField]) -> Set[FrozenSet[str]]: user_emails = set() # type: Set[FrozenSet[str]] for field in fields: values = CustomProfileFieldValue.objects.filter( field=field) for value in values: user_emails.add(frozenset(get_email_from_value(value))) return user_emails field_names, field_hints = (set() for i in range(2)) for field in fields: field_names.add(field.name) field_hints.add(field.hint) return (field_hints, field_names, custom_profile_field_values_for(fields)) assert_realm_values(get_custom_profile_with_field_type_user) # test realmauditlog def get_realm_audit_log_event_type(r: Realm) -> Set[str]: realmauditlogs = RealmAuditLog.objects.filter(realm=r).exclude( event_type=RealmAuditLog.REALM_PLAN_TYPE_CHANGED) realmauditlog_event_type = { log.event_type for log in realmauditlogs } return realmauditlog_event_type assert_realm_values(get_realm_audit_log_event_type) # test huddles def get_huddle_hashes(r: str) -> str: short_names = ['cordelia', 'hamlet', 'othello'] user_id_list = [ UserProfile.objects.get(realm=r, short_name=name).id for name in short_names ] huddle_hash = get_huddle_hash(user_id_list) return huddle_hash assert_realm_values(get_huddle_hashes, equal=False) def get_huddle_message(r: str) -> str: huddle_hash = get_huddle_hashes(r) huddle_id = Huddle.objects.get(huddle_hash=huddle_hash).id huddle_recipient = Recipient.objects.get(type_id=huddle_id, type=3) huddle_message = Message.objects.get(recipient=huddle_recipient) return huddle_message.content assert_realm_values(get_huddle_message) self.assertEqual(get_huddle_message(imported_realm), 'test huddle message') # test userhotspot def get_user_hotspots(r: str) -> Set[str]: user_profile = UserProfile.objects.get(realm=r, short_name='hamlet') hotspots = UserHotspot.objects.filter(user=user_profile) user_hotspots = {hotspot.hotspot for hotspot in hotspots} return user_hotspots assert_realm_values(get_user_hotspots) # test muted topics def get_muted_topics(r: Realm) -> Set[str]: user_profile = UserProfile.objects.get(realm=r, short_name='hamlet') muted_topics = MutedTopic.objects.filter(user_profile=user_profile) topic_names = { muted_topic.topic_name for muted_topic in muted_topics } return topic_names assert_realm_values(get_muted_topics) # test usergroups assert_realm_values( lambda r: {group.name for group in UserGroup.objects.filter(realm=r)}) def get_user_membership(r: str) -> Set[str]: usergroup = UserGroup.objects.get(realm=r, name='hamletcharacters') usergroup_membership = UserGroupMembership.objects.filter( user_group=usergroup) users = { membership.user_profile.email for membership in usergroup_membership } return users assert_realm_values(get_user_membership) # test botstoragedata and botconfigdata def get_botstoragedata(r: Realm) -> Dict[str, Any]: bot_profile = UserProfile.objects.get(full_name="bot", realm=r) bot_storage_data = BotStorageData.objects.get( bot_profile=bot_profile) return { 'key': bot_storage_data.key, 'data': bot_storage_data.value } assert_realm_values(get_botstoragedata) def get_botconfigdata(r: Realm) -> Dict[str, Any]: bot_profile = UserProfile.objects.get(full_name="bot", realm=r) bot_config_data = BotConfigData.objects.get( bot_profile=bot_profile) return {'key': bot_config_data.key, 'data': bot_config_data.value} assert_realm_values(get_botconfigdata) # test messages def get_stream_messages(r: Realm) -> Message: recipient = get_recipient_stream(r) messages = Message.objects.filter(recipient=recipient) return messages def get_stream_topics(r: Realm) -> Set[str]: messages = get_stream_messages(r) topics = {m.topic_name() for m in messages} return topics assert_realm_values(get_stream_topics) # test usermessages def get_usermessages_user(r: Realm) -> Set[Any]: messages = get_stream_messages(r).order_by('content') usermessage = UserMessage.objects.filter(message=messages[0]) usermessage_user = {um.user_profile.email for um in usermessage} return usermessage_user assert_realm_values(get_usermessages_user) # tests to make sure that various data-*-ids in rendered_content # are replaced correctly with the values of newer realm. def get_user_mention(r: Realm) -> Set[Any]: mentioned_user = UserProfile.objects.get( delivery_email=self.example_email("hamlet"), realm=r) data_user_id = 'data-user-id="{}"'.format(mentioned_user.id) mention_message = get_stream_messages(r).get( rendered_content__contains=data_user_id) return mention_message.content assert_realm_values(get_user_mention) def get_stream_mention(r: Realm) -> Set[Any]: mentioned_stream = get_stream(u'Denmark', r) data_stream_id = 'data-stream-id="{}"'.format(mentioned_stream.id) mention_message = get_stream_messages(r).get( rendered_content__contains=data_stream_id) return mention_message.content assert_realm_values(get_stream_mention) def get_user_group_mention(r: Realm) -> Set[Any]: user_group = UserGroup.objects.get(realm=r, name='hamletcharacters') data_usergroup_id = 'data-user-group-id="{}"'.format(user_group.id) mention_message = get_stream_messages(r).get( rendered_content__contains=data_usergroup_id) return mention_message.content assert_realm_values(get_user_group_mention) # test to highlight that bs4 which we use to do data-**id # replacements modifies the HTML sometimes. eg replacing <br> # with </br>, ' with \' etc. The modifications doesn't # affect how the browser displays the rendered_content so we # are okay with using bs4 for this. lxml package also has # similar behavior. orig_polonius_user = UserProfile.objects.get( email=self.example_email("polonius"), realm=original_realm) original_msg = Message.objects.get(content=special_characters_message, sender__realm=original_realm) self.assertEqual(original_msg.rendered_content, ( '<div class="codehilite"><pre><span></span>'\n</pre></div>\n\n\n' '<p><span class="user-mention" data-user-id="%s">@Polonius</span></p>' % (orig_polonius_user.id, ))) imported_polonius_user = UserProfile.objects.get( email=self.example_email("polonius"), realm=imported_realm) imported_msg = Message.objects.get(content=special_characters_message, sender__realm=imported_realm) self.assertEqual(imported_msg.rendered_content, ( '<div class="codehilite"><pre><span></span>\'\n</pre></div>\n' '<p><span class="user-mention" data-user-id="%s">@Polonius</span></p>' % (imported_polonius_user.id, )))
def _make_output_dir(self) -> str: output_dir = get_or_create_dev_uuid_var_path( 'test-backend/test-export') self.rm_tree(output_dir) os.makedirs(output_dir, exist_ok=True) return output_dir
# We need to pick an ID for this test-backend invocation, and store it # in this global so it can be used in init_worker; this is used to # ensure the database IDs we select are unique for each `test-backend` # run. This probably should use a locking mechanism rather than the # below hack, which fails 1/10000000 of the time. random_id_range_start = str(random.randint(1, 10000000)) def get_database_id(worker_id: Optional[int] = None) -> str: if worker_id: return f"{random_id_range_start}_{worker_id}" return random_id_range_start # The root directory for this run of the test suite. TEST_RUN_DIR = get_or_create_dev_uuid_var_path( os.path.join('test-backend', f'run_{get_database_id()}')) _worker_id = 0 # Used to identify the worker process. class TextTestResult(runner.TextTestResult): """ This class has unpythonic function names because base class follows this style. """ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self.failed_tests: List[str] = [] def addInfo(self, test: TestCase, msg: str) -> None: self.stream.write(msg)
def generate_and_send_messages(data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int]) -> int: (tot_messages, personals_pairs, options, output, random_seed) = data random.seed(random_seed) with open(os.path.join(get_or_create_dev_uuid_var_path('test-backend'), "test_messages.json")) as infile: dialog = ujson.load(infile) random.shuffle(dialog) texts = itertools.cycle(dialog) recipient_streams: List[int] = [ klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] recipient_huddles: List[int] = [h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)] huddle_members: Dict[int, List[int]] = {} for h in recipient_huddles: huddle_members[h] = [s.user_profile.id for s in Subscription.objects.filter(recipient_id=h)] # Generate different topics for each stream possible_topics = dict() for stream_id in recipient_streams: possible_topics[stream_id] = generate_topics(options["max_topics"]) message_batch_size = options['batch_size'] num_messages = 0 random_max = 1000000 recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {} messages = [] messages_add_reaction = [] while num_messages < tot_messages: saved_data: Dict[str, Any] = {} message = Message() message.sending_client = get_client('populate_db') message.content = next(texts) randkey = random.randint(1, random_max) if (num_messages > 0 and random.randint(1, random_max) * 100. / random_max < options["stickyness"]): # Use an old recipient message_type, recipient_id, saved_data = recipients[num_messages - 1] if message_type == Recipient.PERSONAL: personals_pair = saved_data['personals_pair'] random.shuffle(personals_pair) elif message_type == Recipient.STREAM: message.subject = saved_data['subject'] message.recipient = get_recipient_by_id(recipient_id) elif message_type == Recipient.HUDDLE: message.recipient = get_recipient_by_id(recipient_id) elif (randkey <= random_max * options["percent_huddles"] / 100.): message_type = Recipient.HUDDLE message.recipient = get_recipient_by_id(random.choice(recipient_huddles)) elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.): message_type = Recipient.PERSONAL personals_pair = random.choice(personals_pairs) random.shuffle(personals_pair) elif (randkey <= random_max * 1.0): message_type = Recipient.STREAM message.recipient = get_recipient_by_id(random.choice(recipient_streams)) if message_type == Recipient.HUDDLE: sender_id = random.choice(huddle_members[message.recipient.id]) message.sender = get_user_profile_by_id(sender_id) elif message_type == Recipient.PERSONAL: message.recipient = Recipient.objects.get(type=Recipient.PERSONAL, type_id=personals_pair[0]) message.sender = get_user_profile_by_id(personals_pair[1]) saved_data['personals_pair'] = personals_pair elif message_type == Recipient.STREAM: # Pick a random subscriber to the stream message.sender = random.choice(Subscription.objects.filter( recipient=message.recipient)).user_profile message.subject = random.choice(possible_topics[message.recipient.id]) saved_data['subject'] = message.subject message.date_sent = choose_date_sent(num_messages, tot_messages, options['threads']) messages.append(message) messages_add_reaction.append(message) recipients[num_messages] = (message_type, message.recipient.id, saved_data) num_messages += 1 if (num_messages % message_batch_size) == 0: # Send the batch and empty the list: send_messages(messages) messages = [] if len(messages) > 0: # If there are unsent messages after exiting the loop, send them: send_messages(messages) reactions_message = [] add_emojis = ["1f44d", "1f642", "1f60a"] for rmessage in messages_add_reaction: if random.random() > 0.9: reactedmessage = Reaction(user_profile=rmessage.sender, message=rmessage, emoji_name="+1", emoji_code=add_emojis[random.randint(0, 2)], reaction_type="unicode_emoji") reactions_message.append(reactedmessage) Reaction.objects.bulk_create(reactions_message) return tot_messages
def generate_and_send_messages( data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int] ) -> int: (tot_messages, personals_pairs, options, output, random_seed) = data random.seed(random_seed) with open( os.path.join(get_or_create_dev_uuid_var_path('test-backend'), "test_messages.json"), "r") as infile: dialog = ujson.load(infile) random.shuffle(dialog) texts = itertools.cycle(dialog) recipient_streams = [ klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] # type: List[int] recipient_huddles = [ h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE) ] # type: List[int] huddle_members = {} # type: Dict[int, List[int]] for h in recipient_huddles: huddle_members[h] = [ s.user_profile.id for s in Subscription.objects.filter(recipient_id=h) ] message_batch_size = options['batch_size'] num_messages = 0 random_max = 1000000 recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]] messages = [] while num_messages < tot_messages: saved_data = {} # type: Dict[str, Any] message = Message() message.sending_client = get_client('populate_db') message.content = next(texts) randkey = random.randint(1, random_max) if (num_messages > 0 and random.randint(1, random_max) * 100. / random_max < options["stickyness"]): # Use an old recipient message_type, recipient_id, saved_data = recipients[num_messages - 1] if message_type == Recipient.PERSONAL: personals_pair = saved_data['personals_pair'] random.shuffle(personals_pair) elif message_type == Recipient.STREAM: message.subject = saved_data['subject'] message.recipient = get_recipient_by_id(recipient_id) elif message_type == Recipient.HUDDLE: message.recipient = get_recipient_by_id(recipient_id) elif (randkey <= random_max * options["percent_huddles"] / 100.): message_type = Recipient.HUDDLE message.recipient = get_recipient_by_id( random.choice(recipient_huddles)) elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.): message_type = Recipient.PERSONAL personals_pair = random.choice(personals_pairs) random.shuffle(personals_pair) elif (randkey <= random_max * 1.0): message_type = Recipient.STREAM message.recipient = get_recipient_by_id( random.choice(recipient_streams)) if message_type == Recipient.HUDDLE: sender_id = random.choice(huddle_members[message.recipient.id]) message.sender = get_user_profile_by_id(sender_id) elif message_type == Recipient.PERSONAL: message.recipient = Recipient.objects.get( type=Recipient.PERSONAL, type_id=personals_pair[0]) message.sender = get_user_profile_by_id(personals_pair[1]) saved_data['personals_pair'] = personals_pair elif message_type == Recipient.STREAM: stream = Stream.objects.get(id=message.recipient.type_id) # Pick a random subscriber to the stream message.sender = random.choice( Subscription.objects.filter( recipient=message.recipient)).user_profile message.subject = stream.name + str(random.randint(1, 3)) saved_data['subject'] = message.subject message.date_sent = choose_date_sent(num_messages, tot_messages, options['threads']) messages.append(message) recipients[num_messages] = (message_type, message.recipient.id, saved_data) num_messages += 1 if (num_messages % message_batch_size) == 0: # Send the batch and empty the list: send_messages(messages) messages = [] if len(messages) > 0: # If there are unsent messages after exiting the loop, send them: send_messages(messages) return tot_messages
def test_convert_slack_workspace_messages( self, mock_get_messages_iterator: mock.Mock, mock_message: mock.Mock) -> None: output_dir = get_or_create_dev_uuid_var_path( 'test-backend/test-slack-import') added_channels = { 'random': ('c5', 1), 'general': ('c6', 2) } # type: Dict[str, Tuple[str, int]] time = float(timezone_now().timestamp()) zerver_message = [{'id': 1, 'ts': time}, {'id': 5, 'ts': time}] def fake_get_messages_iter( slack_data_dir: str, added_channels: AddedChannelsT, added_mpims: AddedMPIMsT) -> Iterator[ZerverFieldsT]: import copy return iter(copy.deepcopy(zerver_message)) realm = {'zerver_subscription': []} # type: Dict[str, Any] user_list = [] # type: List[Dict[str, Any]] reactions = [{"name": "grinning", "users": ["U061A5N1G"], "count": 1}] attachments = uploads = [] # type: List[Dict[str, Any]] zerver_usermessage = [{'id': 3}, {'id': 5}, {'id': 6}, {'id': 9}] mock_get_messages_iterator.side_effect = fake_get_messages_iter mock_message.side_effect = [[ zerver_message[:1], zerver_usermessage[:2], attachments, uploads, reactions[:1] ], [ zerver_message[1:2], zerver_usermessage[2:5], attachments, uploads, reactions[1:1] ]] # Hacky: We should include a zerver_userprofile, not the empty [] test_reactions, uploads, zerver_attachment = convert_slack_workspace_messages( './random_path', user_list, 2, {}, {}, added_channels, {}, realm, [], [], 'domain', output_dir=output_dir, chunk_size=1) messages_file_1 = os.path.join(output_dir, 'messages-000001.json') self.assertTrue(os.path.exists(messages_file_1)) messages_file_2 = os.path.join(output_dir, 'messages-000002.json') self.assertTrue(os.path.exists(messages_file_2)) with open(messages_file_1) as f: message_json = ujson.load(f) self.assertEqual(message_json['zerver_message'], zerver_message[:1]) self.assertEqual(message_json['zerver_usermessage'], zerver_usermessage[:2]) with open(messages_file_2) as f: message_json = ujson.load(f) self.assertEqual(message_json['zerver_message'], zerver_message[1:2]) self.assertEqual(message_json['zerver_usermessage'], zerver_usermessage[2:5]) self.assertEqual(test_reactions, reactions)