def gather_new_streams( user_profile: UserProfile, threshold: datetime.datetime) -> Tuple[int, Dict[str, List[str]]]: if user_profile.is_guest: new_streams = list( get_active_streams(user_profile.realm).filter( is_web_public=True, date_created__gt=threshold)) elif user_profile.can_access_public_streams(): new_streams = list( get_active_streams(user_profile.realm).filter( invite_only=False, date_created__gt=threshold)) base_url = f"{user_profile.realm.uri}/#narrow/stream/" streams_html = [] streams_plain = [] for stream in new_streams: narrow_url = base_url + encode_stream(stream.id, stream.name) stream_link = f"<a href='{narrow_url}'>{stream.name}</a>" streams_html.append(stream_link) streams_plain.append(stream.name) return len(new_streams), {"html": streams_html, "plain": streams_plain}
def by_stream(self, query, operand, maybe_negate): stream = get_stream(operand, self.user_profile.realm) if stream is None: raise BadNarrowOperator('unknown stream ' + operand) if self.user_profile.realm.domain == "mit.edu": # MIT users expect narrowing to "social" to also show messages to /^(un)*social(.d)*$/ # (unsocial, ununsocial, social.d, etc) m = re.search(r'^(?:un)*(.+?)(?:\.d)*$', stream.name, re.IGNORECASE) if m: base_stream_name = m.group(1) else: base_stream_name = stream.name matching_streams = get_active_streams( self.user_profile.realm).filter( name__iregex=r'^(un)*%s(\.d)*$' % (self._pg_re_escape(base_stream_name), )) matching_stream_ids = [ matching_stream.id for matching_stream in matching_streams ] recipients_map = bulk_get_recipients(Recipient.STREAM, matching_stream_ids) cond = column("recipient_id").in_( [recipient.id for recipient in recipients_map.values()]) return query.where(maybe_negate(cond)) recipient = get_recipient(Recipient.STREAM, type_id=stream.id) cond = column("recipient_id") == recipient.id return query.where(maybe_negate(cond))
def by_stream(self, query: Query, operand: Union[str, int], maybe_negate: ConditionTransform) -> Query: try: # Because you can see your own message history for # private streams you are no longer subscribed to, we # need get_stream_by_narrow_operand_access_unchecked here. stream = get_stream_by_narrow_operand_access_unchecked(operand, self.user_profile.realm) except Stream.DoesNotExist: raise BadNarrowOperator('unknown stream ' + str(operand)) if self.user_profile.realm.is_zephyr_mirror_realm: # MIT users expect narrowing to "social" to also show messages to # /^(un)*social(.d)*$/ (unsocial, ununsocial, social.d, ...). # In `ok_to_include_history`, we assume that a non-negated # `stream` term for a public stream will limit the query to # that specific stream. So it would be a bug to hit this # codepath after relying on this term there. But all streams in # a Zephyr realm are private, so that doesn't happen. assert(not stream.is_public()) m = re.search(r'^(?:un)*(.+?)(?:\.d)*$', stream.name, re.IGNORECASE) # Since the regex has a `.+` in it and "" is invalid as a # stream name, this will always match assert(m is not None) base_stream_name = m.group(1) matching_streams = get_active_streams(self.user_profile.realm).filter( name__iregex=fr'^(un)*{self._pg_re_escape(base_stream_name)}(\.d)*$') recipient_ids = [matching_stream.recipient_id for matching_stream in matching_streams] cond = column("recipient_id").in_(recipient_ids) return query.where(maybe_negate(cond)) recipient = stream.recipient cond = column("recipient_id") == recipient.id return query.where(maybe_negate(cond))
def by_stream(self, query, operand, maybe_negate): # type: (Query, str, ConditionTransform) -> Query stream = get_stream(operand, self.user_profile.realm) if stream is None: raise BadNarrowOperator('unknown stream ' + operand) if self.user_profile.realm.is_zephyr_mirror_realm: # MIT users expect narrowing to "social" to also show messages to /^(un)*social(.d)*$/ # (unsocial, ununsocial, social.d, etc) m = re.search(r'^(?:un)*(.+?)(?:\.d)*$', stream.name, re.IGNORECASE) # Since the regex has a `.+` in it and "" is invalid as a # stream name, this will always match assert(m is not None) base_stream_name = m.group(1) matching_streams = get_active_streams(self.user_profile.realm).filter( name__iregex=r'^(un)*%s(\.d)*$' % (self._pg_re_escape(base_stream_name),)) matching_stream_ids = [matching_stream.id for matching_stream in matching_streams] recipients_map = bulk_get_recipients(Recipient.STREAM, matching_stream_ids) cond = column("recipient_id").in_([recipient.id for recipient in recipients_map.values()]) return query.where(maybe_negate(cond)) recipient = get_recipient(Recipient.STREAM, type_id=stream.id) cond = column("recipient_id") == recipient.id return query.where(maybe_negate(cond))
def gather_new_streams(user_profile, threshold): if user_profile.realm.domain == "mit.edu": new_streams = [] else: new_streams = list(get_active_streams(user_profile.realm).filter( invite_only=False, date_created__gt=threshold)) base_url = "https://%s/#narrow/stream/" % (settings.EXTERNAL_HOST,) streams_html = [] streams_plain = [] for stream in new_streams: narrow_url = base_url + hashchange_encode(stream.name) stream_link = "<a href='%s'>%s</a>" % (narrow_url, stream.name) streams_html.append(stream_link) streams_plain.append(stream.name) return len(new_streams), {"html": streams_html, "plain": streams_plain}
def gather_new_streams(user_profile: UserProfile, threshold: datetime.datetime) -> Tuple[int, Dict[str, List[Text]]]: if user_profile.realm.is_zephyr_mirror_realm: new_streams = [] # type: List[Stream] else: new_streams = list(get_active_streams(user_profile.realm).filter( invite_only=False, date_created__gt=threshold)) base_url = u"%s/#narrow/stream/" % (user_profile.realm.uri,) streams_html = [] streams_plain = [] for stream in new_streams: narrow_url = base_url + hash_util_encode(stream.name) stream_link = u"<a href='%s'>%s</a>" % (narrow_url, stream.name) streams_html.append(stream_link) streams_plain.append(stream.name) return len(new_streams), {"html": streams_html, "plain": streams_plain}
def gather_new_streams(user_profile, threshold): # type: (UserProfile, datetime.datetime) -> Tuple[int, Dict[str, List[text_type]]] if user_profile.realm.is_zephyr_mirror_realm: new_streams = [] # type: List[Stream] else: new_streams = list(get_active_streams(user_profile.realm).filter( invite_only=False, date_created__gt=threshold)) base_url = u"https://%s/#narrow/stream/" % (settings.EXTERNAL_HOST,) streams_html = [] streams_plain = [] for stream in new_streams: narrow_url = base_url + hashchange_encode(stream.name) stream_link = u"<a href='%s'>%s</a>" % (narrow_url, stream.name) streams_html.append(stream_link) streams_plain.append(stream.name) return len(new_streams), {"html": streams_html, "plain": streams_plain}
def gather_new_streams(user_profile: UserProfile, threshold: datetime.datetime) -> Tuple[int, Dict[str, List[str]]]: if user_profile.can_access_public_streams(): new_streams = list(get_active_streams(user_profile.realm).filter( invite_only=False, date_created__gt=threshold)) else: new_streams = [] base_url = "%s/#narrow/stream/" % (user_profile.realm.uri,) streams_html = [] streams_plain = [] for stream in new_streams: narrow_url = base_url + encode_stream(stream.id, stream.name) stream_link = "<a href='%s'>%s</a>" % (narrow_url, stream.name) streams_html.append(stream_link) streams_plain.append(stream.name) return len(new_streams), {"html": streams_html, "plain": streams_plain}
def get_stream_name_info(realm: Realm, stream_names: Set[str]) -> Dict[str, FullNameInfo]: if not stream_names: return {} q_list = {Q(name=name) for name in stream_names} rows = ( get_active_streams( realm=realm, ) .filter( functools.reduce(lambda a, b: a | b, q_list), ) .values( "id", "name", ) ) dct = {row["name"]: row for row in rows} return dct
def by_stream(self, query, operand, maybe_negate): stream = get_stream(operand, self.user_profile.realm) if stream is None: raise BadNarrowOperator('unknown stream ' + operand) if self.user_profile.realm.domain == "mit.edu": # MIT users expect narrowing to "social" to also show messages to /^(un)*social(.d)*$/ # (unsocial, ununsocial, social.d, etc) m = re.search(r'^(?:un)*(.+?)(?:\.d)*$', stream.name, re.IGNORECASE) if m: base_stream_name = m.group(1) else: base_stream_name = stream.name matching_streams = get_active_streams(self.user_profile.realm).filter( name__iregex=r'^(un)*%s(\.d)*$' % (self._pg_re_escape(base_stream_name),)) matching_stream_ids = [matching_stream.id for matching_stream in matching_streams] recipients_map = bulk_get_recipients(Recipient.STREAM, matching_stream_ids) cond = column("recipient_id").in_([recipient.id for recipient in recipients_map.values()]) return query.where(maybe_negate(cond)) recipient = get_recipient(Recipient.STREAM, type_id=stream.id) cond = column("recipient_id") == recipient.id return query.where(maybe_negate(cond))
def test_import_realm(self) -> None: original_realm = Realm.objects.get(string_id='zulip') RealmEmoji.objects.get(realm=original_realm).delete() # data to test import of huddles huddle = [ self.example_email('hamlet'), self.example_email('othello') ] self.send_huddle_message( self.example_email('cordelia'), huddle, 'test huddle message' ) # data to test import of hotspots sample_user = self.example_user('hamlet') UserHotspot.objects.create( user=sample_user, hotspot='intro_streams' ) # data to test import of muted topic stream = get_stream(u'Verona', original_realm) add_topic_mute( user_profile=sample_user, stream_id=stream.id, recipient_id=get_stream_recipient(stream.id).id, topic_name=u'Verona2') # data to test import of botstoragedata and botconfigdata bot_profile = do_create_user( email="*****@*****.**", password="******", realm=original_realm, full_name="bot", short_name="bot", bot_type=UserProfile.EMBEDDED_BOT, bot_owner=sample_user) storage = StateHandler(bot_profile) storage.put('some key', 'some value') set_bot_config(bot_profile, 'entry 1', 'value 1') self._export_realm(original_realm) with patch('logging.info'): do_import_realm('var/test-export', 'test-zulip') # sanity checks # test realm self.assertTrue(Realm.objects.filter(string_id='test-zulip').exists()) imported_realm = Realm.objects.get(string_id='test-zulip') self.assertNotEqual(imported_realm.id, original_realm.id) def assert_realm_values(f: Callable[[Realm], Any], equal: bool=True) -> None: orig_realm_result = f(original_realm) imported_realm_result = f(imported_realm) # orig_realm_result should be truthy and have some values, otherwise # the test is kind of meaningless assert(orig_realm_result) if equal: self.assertEqual(orig_realm_result, imported_realm_result) else: self.assertNotEqual(orig_realm_result, imported_realm_result) # test users assert_realm_values( lambda r: {user.email for user in r.get_admin_users()} ) assert_realm_values( lambda r: {user.email for user in r.get_active_users()} ) # test stream assert_realm_values( lambda r: {stream.name for stream in get_active_streams(r)} ) # test recipients def get_recipient_stream(r: Realm) -> Stream: return get_stream_recipient( Stream.objects.get(name='Verona', realm=r).id ) def get_recipient_user(r: Realm) -> UserProfile: return get_personal_recipient( UserProfile.objects.get(full_name='Iago', realm=r).id ) assert_realm_values(lambda r: get_recipient_stream(r).type) assert_realm_values(lambda r: get_recipient_user(r).type) # test subscription def get_subscribers(recipient: Recipient) -> Set[str]: subscriptions = Subscription.objects.filter(recipient=recipient) users = {sub.user_profile.email for sub in subscriptions} return users assert_realm_values( lambda r: get_subscribers(get_recipient_stream(r)) ) assert_realm_values( lambda r: get_subscribers(get_recipient_user(r)) ) # test custom profile fields def get_custom_profile_field_names(r: Realm) -> Set[str]: custom_profile_fields = CustomProfileField.objects.filter(realm=r) custom_profile_field_names = {field.name for field in custom_profile_fields} return custom_profile_field_names assert_realm_values(get_custom_profile_field_names) def get_custom_profile_with_field_type_user(r: Realm) -> Tuple[Set[Any], Set[Any], Set[FrozenSet[str]]]: fields = CustomProfileField.objects.filter( field_type=CustomProfileField.USER, realm=r) def get_email(user_id: int) -> str: return UserProfile.objects.get(id=user_id).email def get_email_from_value(field_value: CustomProfileFieldValue) -> Set[str]: user_id_list = ujson.loads(field_value.value) return {get_email(user_id) for user_id in user_id_list} def custom_profile_field_values_for(fields: List[CustomProfileField]) -> Set[FrozenSet[str]]: user_emails = set() # type: Set[FrozenSet[str]] for field in fields: values = CustomProfileFieldValue.objects.filter(field=field) for value in values: user_emails.add(frozenset(get_email_from_value(value))) return user_emails field_names, field_hints = (set() for i in range(2)) for field in fields: field_names.add(field.name) field_hints.add(field.hint) return (field_hints, field_names, custom_profile_field_values_for(fields)) assert_realm_values(get_custom_profile_with_field_type_user) # test realmauditlog def get_realm_audit_log_event_type(r: Realm) -> Set[str]: realmauditlogs = RealmAuditLog.objects.filter(realm=r) realmauditlog_event_type = {log.event_type for log in realmauditlogs} return realmauditlog_event_type assert_realm_values(get_realm_audit_log_event_type) # test huddles def get_huddle_hashes(r: str) -> str: short_names = ['cordelia', 'hamlet', 'othello'] user_id_list = [UserProfile.objects.get(realm=r, short_name=name).id for name in short_names] huddle_hash = get_huddle_hash(user_id_list) return huddle_hash assert_realm_values(get_huddle_hashes, equal=False) def get_huddle_message(r: str) -> str: huddle_hash = get_huddle_hashes(r) huddle_id = Huddle.objects.get(huddle_hash=huddle_hash).id huddle_recipient = Recipient.objects.get(type_id=huddle_id, type=3) huddle_message = Message.objects.get(recipient=huddle_recipient) return huddle_message.content assert_realm_values(get_huddle_message) self.assertEqual(get_huddle_message(imported_realm), 'test huddle message') # test userhotspot def get_user_hotspots(r: str) -> Set[str]: user_profile = UserProfile.objects.get(realm=r, short_name='hamlet') hotspots = UserHotspot.objects.filter(user=user_profile) user_hotspots = {hotspot.hotspot for hotspot in hotspots} return user_hotspots assert_realm_values(get_user_hotspots) # test muted topics def get_muted_topics(r: Realm) -> Set[str]: user_profile = UserProfile.objects.get(realm=r, short_name='hamlet') muted_topics = MutedTopic.objects.filter(user_profile=user_profile) topic_names = {muted_topic.topic_name for muted_topic in muted_topics} return topic_names assert_realm_values(get_muted_topics) # test usergroups assert_realm_values( lambda r: {group.name for group in UserGroup.objects.filter(realm=r)} ) def get_user_membership(r: str) -> Set[str]: usergroup = UserGroup.objects.get(realm=r, name='hamletcharacters') usergroup_membership = UserGroupMembership.objects.filter(user_group=usergroup) users = {membership.user_profile.email for membership in usergroup_membership} return users assert_realm_values(get_user_membership) # test botstoragedata and botconfigdata def get_botstoragedata(r: Realm) -> Dict[str, Any]: bot_profile = UserProfile.objects.get(full_name="bot", realm=r) bot_storage_data = BotStorageData.objects.get(bot_profile=bot_profile) return {'key': bot_storage_data.key, 'data': bot_storage_data.value} assert_realm_values(get_botstoragedata) def get_botconfigdata(r: Realm) -> Dict[str, Any]: bot_profile = UserProfile.objects.get(full_name="bot", realm=r) bot_config_data = BotConfigData.objects.get(bot_profile=bot_profile) return {'key': bot_config_data.key, 'data': bot_config_data.value} assert_realm_values(get_botconfigdata) # test messages def get_stream_messages(r: Realm) -> Message: recipient = get_recipient_stream(r) messages = Message.objects.filter(recipient=recipient) return messages def get_stream_topics(r: Realm) -> Set[str]: messages = get_stream_messages(r) topics = {m.topic_name() for m in messages} return topics assert_realm_values(get_stream_topics) # test usermessages def get_usermessages_user(r: Realm) -> Set[Any]: messages = get_stream_messages(r).order_by('content') usermessage = UserMessage.objects.filter(message=messages[0]) usermessage_user = {um.user_profile.email for um in usermessage} return usermessage_user assert_realm_values(get_usermessages_user)
def test_import_realm(self) -> None: original_realm = Realm.objects.get(string_id='zulip') RealmEmoji.objects.get(realm=original_realm).delete() # data to test import of huddles huddle = [self.example_email('hamlet'), self.example_email('othello')] self.send_huddle_message(self.example_email('cordelia'), huddle, 'test huddle message') # data to test import of hotspots sample_user = self.example_user('hamlet') UserHotspot.objects.create(user=sample_user, hotspot='intro_streams') # data to test import of muted topic stream = get_stream(u'Verona', original_realm) add_topic_mute(user_profile=sample_user, stream_id=stream.id, recipient_id=get_stream_recipient(stream.id).id, topic_name=u'Verona2') # data to test import of botstoragedata and botconfigdata bot_profile = do_create_user(email="*****@*****.**", password="******", realm=original_realm, full_name="bot", short_name="bot", bot_type=UserProfile.EMBEDDED_BOT, bot_owner=sample_user) storage = StateHandler(bot_profile) storage.put('some key', 'some value') set_bot_config(bot_profile, 'entry 1', 'value 1') self._export_realm(original_realm) with patch('logging.info'): with self.settings(BILLING_ENABLED=False): do_import_realm('var/test-export', 'test-zulip') # sanity checks # test realm self.assertTrue(Realm.objects.filter(string_id='test-zulip').exists()) imported_realm = Realm.objects.get(string_id='test-zulip') self.assertNotEqual(imported_realm.id, original_realm.id) def assert_realm_values(f: Callable[[Realm], Any], equal: bool = True) -> None: orig_realm_result = f(original_realm) imported_realm_result = f(imported_realm) # orig_realm_result should be truthy and have some values, otherwise # the test is kind of meaningless assert (orig_realm_result) if equal: self.assertEqual(orig_realm_result, imported_realm_result) else: self.assertNotEqual(orig_realm_result, imported_realm_result) # test users assert_realm_values( lambda r: {user.email for user in r.get_admin_users()}) assert_realm_values( lambda r: {user.email for user in r.get_active_users()}) # test stream assert_realm_values( lambda r: {stream.name for stream in get_active_streams(r)}) # test recipients def get_recipient_stream(r: Realm) -> Stream: return get_stream_recipient( Stream.objects.get(name='Verona', realm=r).id) def get_recipient_user(r: Realm) -> UserProfile: return get_personal_recipient( UserProfile.objects.get(full_name='Iago', realm=r).id) assert_realm_values(lambda r: get_recipient_stream(r).type) assert_realm_values(lambda r: get_recipient_user(r).type) # test subscription def get_subscribers(recipient: Recipient) -> Set[str]: subscriptions = Subscription.objects.filter(recipient=recipient) users = {sub.user_profile.email for sub in subscriptions} return users assert_realm_values(lambda r: get_subscribers(get_recipient_stream(r))) assert_realm_values(lambda r: get_subscribers(get_recipient_user(r))) # test custom profile fields def get_custom_profile_field_names(r: Realm) -> Set[str]: custom_profile_fields = CustomProfileField.objects.filter(realm=r) custom_profile_field_names = { field.name for field in custom_profile_fields } return custom_profile_field_names assert_realm_values(get_custom_profile_field_names) def get_custom_profile_with_field_type_user( r: Realm) -> Tuple[Set[Any], Set[Any], Set[FrozenSet[str]]]: fields = CustomProfileField.objects.filter( field_type=CustomProfileField.USER, realm=r) def get_email(user_id: int) -> str: return UserProfile.objects.get(id=user_id).email def get_email_from_value( field_value: CustomProfileFieldValue) -> Set[str]: user_id_list = ujson.loads(field_value.value) return {get_email(user_id) for user_id in user_id_list} def custom_profile_field_values_for( fields: List[CustomProfileField]) -> Set[FrozenSet[str]]: user_emails = set() # type: Set[FrozenSet[str]] for field in fields: values = CustomProfileFieldValue.objects.filter( field=field) for value in values: user_emails.add(frozenset(get_email_from_value(value))) return user_emails field_names, field_hints = (set() for i in range(2)) for field in fields: field_names.add(field.name) field_hints.add(field.hint) return (field_hints, field_names, custom_profile_field_values_for(fields)) assert_realm_values(get_custom_profile_with_field_type_user) # test realmauditlog def get_realm_audit_log_event_type(r: Realm) -> Set[str]: realmauditlogs = RealmAuditLog.objects.filter(realm=r).exclude( event_type=RealmAuditLog.REALM_PLAN_TYPE_CHANGED) realmauditlog_event_type = { log.event_type for log in realmauditlogs } return realmauditlog_event_type assert_realm_values(get_realm_audit_log_event_type) # test huddles def get_huddle_hashes(r: str) -> str: short_names = ['cordelia', 'hamlet', 'othello'] user_id_list = [ UserProfile.objects.get(realm=r, short_name=name).id for name in short_names ] huddle_hash = get_huddle_hash(user_id_list) return huddle_hash assert_realm_values(get_huddle_hashes, equal=False) def get_huddle_message(r: str) -> str: huddle_hash = get_huddle_hashes(r) huddle_id = Huddle.objects.get(huddle_hash=huddle_hash).id huddle_recipient = Recipient.objects.get(type_id=huddle_id, type=3) huddle_message = Message.objects.get(recipient=huddle_recipient) return huddle_message.content assert_realm_values(get_huddle_message) self.assertEqual(get_huddle_message(imported_realm), 'test huddle message') # test userhotspot def get_user_hotspots(r: str) -> Set[str]: user_profile = UserProfile.objects.get(realm=r, short_name='hamlet') hotspots = UserHotspot.objects.filter(user=user_profile) user_hotspots = {hotspot.hotspot for hotspot in hotspots} return user_hotspots assert_realm_values(get_user_hotspots) # test muted topics def get_muted_topics(r: Realm) -> Set[str]: user_profile = UserProfile.objects.get(realm=r, short_name='hamlet') muted_topics = MutedTopic.objects.filter(user_profile=user_profile) topic_names = { muted_topic.topic_name for muted_topic in muted_topics } return topic_names assert_realm_values(get_muted_topics) # test usergroups assert_realm_values( lambda r: {group.name for group in UserGroup.objects.filter(realm=r)}) def get_user_membership(r: str) -> Set[str]: usergroup = UserGroup.objects.get(realm=r, name='hamletcharacters') usergroup_membership = UserGroupMembership.objects.filter( user_group=usergroup) users = { membership.user_profile.email for membership in usergroup_membership } return users assert_realm_values(get_user_membership) # test botstoragedata and botconfigdata def get_botstoragedata(r: Realm) -> Dict[str, Any]: bot_profile = UserProfile.objects.get(full_name="bot", realm=r) bot_storage_data = BotStorageData.objects.get( bot_profile=bot_profile) return { 'key': bot_storage_data.key, 'data': bot_storage_data.value } assert_realm_values(get_botstoragedata) def get_botconfigdata(r: Realm) -> Dict[str, Any]: bot_profile = UserProfile.objects.get(full_name="bot", realm=r) bot_config_data = BotConfigData.objects.get( bot_profile=bot_profile) return {'key': bot_config_data.key, 'data': bot_config_data.value} assert_realm_values(get_botconfigdata) # test messages def get_stream_messages(r: Realm) -> Message: recipient = get_recipient_stream(r) messages = Message.objects.filter(recipient=recipient) return messages def get_stream_topics(r: Realm) -> Set[str]: messages = get_stream_messages(r) topics = {m.topic_name() for m in messages} return topics assert_realm_values(get_stream_topics) # test usermessages def get_usermessages_user(r: Realm) -> Set[Any]: messages = get_stream_messages(r).order_by('content') usermessage = UserMessage.objects.filter(message=messages[0]) usermessage_user = {um.user_profile.email for um in usermessage} return usermessage_user assert_realm_values(get_usermessages_user)
def test_import_realm(self) -> None: original_realm = Realm.objects.get(string_id='zulip') RealmEmoji.objects.get(realm=original_realm).delete() self._export_realm(original_realm) with patch('logging.info'): do_import_realm('var/test-export', 'test-zulip') # sanity checks # test realm self.assertTrue(Realm.objects.filter(string_id='test-zulip').exists()) imported_realm = Realm.objects.get(string_id='test-zulip') realms = [original_realm, imported_realm] self.assertNotEqual(imported_realm.id, original_realm.id) # test users emails = [{user.email for user in realm.get_admin_users()} for realm in realms] self.assertEqual(emails[0], emails[1]) emails = [{ user.email for user in realm.get_active_users().filter(is_bot=False) } for realm in realms] self.assertEqual(emails[0], emails[1]) # test stream stream_names = [{stream.name for stream in get_active_streams(realm)} for realm in realms] self.assertEqual(stream_names[0], stream_names[1]) # test recipients stream_recipients = [ get_stream_recipient( Stream.objects.get(name='Verona', realm=realm).id) for realm in realms ] self.assertEqual(stream_recipients[0].type, stream_recipients[1].type) user_recipients = [ get_personal_recipient( UserProfile.objects.get(full_name='Iago', realm=realm).id) for realm in realms ] self.assertEqual(user_recipients[0].type, user_recipients[1].type) # test subscription stream_subscription = [ Subscription.objects.filter(recipient=recipient) for recipient in stream_recipients ] subscription_stream_user = [{ sub.user_profile.email for sub in subscription } for subscription in stream_subscription] self.assertEqual(subscription_stream_user[0], subscription_stream_user[1]) user_subscription = [ Subscription.objects.filter(recipient=recipient) for recipient in user_recipients ] subscription_user = [{sub.user_profile.email for sub in subscription} for subscription in user_subscription] self.assertEqual(subscription_user[0], subscription_user[1]) # test custom profile fields custom_profile_field = [ CustomProfileField.objects.filter(realm=realm) for realm in realms ] custom_profile_field_name = [{ custom_profile_field.name for custom_profile_field in realm_custom_profile_field } for realm_custom_profile_field in custom_profile_field] self.assertEqual(custom_profile_field_name[0], custom_profile_field_name[1]) # test messages stream_message = [ Message.objects.filter(recipient=recipient) for recipient in stream_recipients ] stream_message_subject = [{ message.subject for message in stream_message } for stream_message in stream_message] self.assertEqual(stream_message_subject[0], stream_message_subject[1]) # test usermessages stream_message = [ stream_message.order_by('content') for stream_message in stream_message ] usermessage = [ UserMessage.objects.filter(message=stream_message[0]) for stream_message in stream_message ] usermessage_user = [{ user_message.user_profile.email for user_message in usermessage } for usermessage in usermessage] self.assertEqual(usermessage_user[0], usermessage_user[1])
def test_import_realm(self) -> None: original_realm = Realm.objects.get(string_id='zulip') RealmEmoji.objects.get(realm=original_realm).delete() # data to test import of huddles huddle = [ self.example_email('hamlet'), self.example_email('othello') ] self.send_huddle_message( self.example_email('cordelia'), huddle, 'test huddle message' ) self._export_realm(original_realm) with patch('logging.info'): do_import_realm('var/test-export', 'test-zulip') # sanity checks # test realm self.assertTrue(Realm.objects.filter(string_id='test-zulip').exists()) imported_realm = Realm.objects.get(string_id='test-zulip') self.assertNotEqual(imported_realm.id, original_realm.id) def assert_realm_values(f: Callable[[Realm], Any], equal: bool=True) -> None: orig_realm_result = f(original_realm) imported_realm_result = f(imported_realm) if equal: self.assertEqual(orig_realm_result, imported_realm_result) else: self.assertNotEqual(orig_realm_result, imported_realm_result) # test users assert_realm_values( lambda r: {user.email for user in r.get_admin_users()} ) assert_realm_values( lambda r: {user.email for user in r.get_active_users()} ) # test stream assert_realm_values( lambda r: {stream.name for stream in get_active_streams(r)} ) # test recipients def get_recipient_stream(r: Realm) -> Stream: return get_stream_recipient( Stream.objects.get(name='Verona', realm=r).id ) def get_recipient_user(r: Realm) -> UserProfile: return get_personal_recipient( UserProfile.objects.get(full_name='Iago', realm=r).id ) assert_realm_values(lambda r: get_recipient_stream(r).type) assert_realm_values(lambda r: get_recipient_user(r).type) # test subscription def get_subscribers(recipient: Recipient) -> Set[str]: subscriptions = Subscription.objects.filter(recipient=recipient) users = {sub.user_profile.email for sub in subscriptions} return users assert_realm_values( lambda r: get_subscribers(get_recipient_stream(r)) ) assert_realm_values( lambda r: get_subscribers(get_recipient_user(r)) ) # test custom profile fields def get_custom_profile_field_names(r: Realm) -> Set[str]: custom_profile_fields = CustomProfileField.objects.filter(realm=r) custom_profile_field_names = {field.name for field in custom_profile_fields} return custom_profile_field_names assert_realm_values(get_custom_profile_field_names) # test realmauditlog def get_realm_audit_log_event_type(r: Realm) -> Set[str]: realmauditlogs = RealmAuditLog.objects.filter(realm=r) realmauditlog_event_type = {log.event_type for log in realmauditlogs} return realmauditlog_event_type assert_realm_values(get_realm_audit_log_event_type) # test huddles def get_huddle_hashes(r: str) -> str: short_names = ['cordelia', 'hamlet', 'othello'] user_id_list = [UserProfile.objects.get(realm=r, short_name=name).id for name in short_names] huddle_hash = get_huddle_hash(user_id_list) return huddle_hash assert_realm_values(get_huddle_hashes, equal=False) def get_huddle_message(r: str) -> str: huddle_hash = get_huddle_hashes(r) huddle_id = Huddle.objects.get(huddle_hash=huddle_hash).id huddle_recipient = Recipient.objects.get(type_id=huddle_id, type=3) huddle_message = Message.objects.get(recipient=huddle_recipient) return huddle_message.content assert_realm_values(get_huddle_message) self.assertEqual(get_huddle_message(imported_realm), 'test huddle message') # test messages def get_stream_messages(r: Realm) -> Message: recipient = get_recipient_stream(r) messages = Message.objects.filter(recipient=recipient) return messages def get_stream_topics(r: Realm) -> Set[str]: messages = get_stream_messages(r) topics = {m.subject for m in messages} return topics assert_realm_values(get_stream_topics) # test usermessages def get_usermessages_user(r: Realm) -> Set[Any]: messages = get_stream_messages(r).order_by('content') usermessage = UserMessage.objects.filter(message=messages[0]) usermessage_user = {um.user_profile.email for um in usermessage} return usermessage_user assert_realm_values(get_usermessages_user)
def test_import_realm(self) -> None: original_realm = Realm.objects.get(string_id='zulip') RealmEmoji.objects.get(realm=original_realm).delete() # data to test import of huddles huddle = [self.example_email('hamlet'), self.example_email('othello')] self.send_huddle_message(self.example_email('cordelia'), huddle, 'test huddle message') user_mention_message = '@**King Hamlet** Hello' self.send_stream_message(self.example_email("iago"), "Verona", user_mention_message) stream_mention_message = 'Subscribe to #**Denmark**' self.send_stream_message(self.example_email("hamlet"), "Verona", stream_mention_message) user_group_mention_message = 'Hello @*hamletcharacters*' self.send_stream_message(self.example_email("othello"), "Verona", user_group_mention_message) special_characters_message = "```\n'\n```\n@**Polonius**" self.send_stream_message(self.example_email("iago"), "Denmark", special_characters_message) # data to test import of hotspots sample_user = self.example_user('hamlet') UserHotspot.objects.create(user=sample_user, hotspot='intro_streams') # data to test import of muted topic stream = get_stream(u'Verona', original_realm) add_topic_mute(user_profile=sample_user, stream_id=stream.id, recipient_id=get_stream_recipient(stream.id).id, topic_name=u'Verona2') # data to test import of botstoragedata and botconfigdata bot_profile = do_create_user(email="*****@*****.**", password="******", realm=original_realm, full_name="bot", short_name="bot", bot_type=UserProfile.EMBEDDED_BOT, bot_owner=sample_user) storage = StateHandler(bot_profile) storage.put('some key', 'some value') set_bot_config(bot_profile, 'entry 1', 'value 1') self._export_realm(original_realm) with patch('logging.info'): with self.settings(BILLING_ENABLED=False): do_import_realm( os.path.join(settings.TEST_WORKER_DIR, 'test-export'), 'test-zulip') # sanity checks # test realm self.assertTrue(Realm.objects.filter(string_id='test-zulip').exists()) imported_realm = Realm.objects.get(string_id='test-zulip') self.assertNotEqual(imported_realm.id, original_realm.id) def assert_realm_values(f: Callable[[Realm], Any], equal: bool = True) -> None: orig_realm_result = f(original_realm) imported_realm_result = f(imported_realm) # orig_realm_result should be truthy and have some values, otherwise # the test is kind of meaningless assert (orig_realm_result) if equal: self.assertEqual(orig_realm_result, imported_realm_result) else: self.assertNotEqual(orig_realm_result, imported_realm_result) # test users assert_realm_values( lambda r: {user.email for user in r.get_admin_users_and_bots()}) assert_realm_values( lambda r: {user.email for user in r.get_active_users()}) # test stream assert_realm_values( lambda r: {stream.name for stream in get_active_streams(r)}) # test recipients def get_recipient_stream(r: Realm) -> Stream: return get_stream_recipient( Stream.objects.get(name='Verona', realm=r).id) def get_recipient_user(r: Realm) -> UserProfile: return get_personal_recipient( UserProfile.objects.get(full_name='Iago', realm=r).id) assert_realm_values(lambda r: get_recipient_stream(r).type) assert_realm_values(lambda r: get_recipient_user(r).type) # test subscription def get_subscribers(recipient: Recipient) -> Set[str]: subscriptions = Subscription.objects.filter(recipient=recipient) users = {sub.user_profile.email for sub in subscriptions} return users assert_realm_values(lambda r: get_subscribers(get_recipient_stream(r))) assert_realm_values(lambda r: get_subscribers(get_recipient_user(r))) # test custom profile fields def get_custom_profile_field_names(r: Realm) -> Set[str]: custom_profile_fields = CustomProfileField.objects.filter(realm=r) custom_profile_field_names = { field.name for field in custom_profile_fields } return custom_profile_field_names assert_realm_values(get_custom_profile_field_names) def get_custom_profile_with_field_type_user( r: Realm) -> Tuple[Set[Any], Set[Any], Set[FrozenSet[str]]]: fields = CustomProfileField.objects.filter( field_type=CustomProfileField.USER, realm=r) def get_email(user_id: int) -> str: return UserProfile.objects.get(id=user_id).email def get_email_from_value( field_value: CustomProfileFieldValue) -> Set[str]: user_id_list = ujson.loads(field_value.value) return {get_email(user_id) for user_id in user_id_list} def custom_profile_field_values_for( fields: List[CustomProfileField]) -> Set[FrozenSet[str]]: user_emails = set() # type: Set[FrozenSet[str]] for field in fields: values = CustomProfileFieldValue.objects.filter( field=field) for value in values: user_emails.add(frozenset(get_email_from_value(value))) return user_emails field_names, field_hints = (set() for i in range(2)) for field in fields: field_names.add(field.name) field_hints.add(field.hint) return (field_hints, field_names, custom_profile_field_values_for(fields)) assert_realm_values(get_custom_profile_with_field_type_user) # test realmauditlog def get_realm_audit_log_event_type(r: Realm) -> Set[str]: realmauditlogs = RealmAuditLog.objects.filter(realm=r).exclude( event_type=RealmAuditLog.REALM_PLAN_TYPE_CHANGED) realmauditlog_event_type = { log.event_type for log in realmauditlogs } return realmauditlog_event_type assert_realm_values(get_realm_audit_log_event_type) # test huddles def get_huddle_hashes(r: str) -> str: short_names = ['cordelia', 'hamlet', 'othello'] user_id_list = [ UserProfile.objects.get(realm=r, short_name=name).id for name in short_names ] huddle_hash = get_huddle_hash(user_id_list) return huddle_hash assert_realm_values(get_huddle_hashes, equal=False) def get_huddle_message(r: str) -> str: huddle_hash = get_huddle_hashes(r) huddle_id = Huddle.objects.get(huddle_hash=huddle_hash).id huddle_recipient = Recipient.objects.get(type_id=huddle_id, type=3) huddle_message = Message.objects.get(recipient=huddle_recipient) return huddle_message.content assert_realm_values(get_huddle_message) self.assertEqual(get_huddle_message(imported_realm), 'test huddle message') # test userhotspot def get_user_hotspots(r: str) -> Set[str]: user_profile = UserProfile.objects.get(realm=r, short_name='hamlet') hotspots = UserHotspot.objects.filter(user=user_profile) user_hotspots = {hotspot.hotspot for hotspot in hotspots} return user_hotspots assert_realm_values(get_user_hotspots) # test muted topics def get_muted_topics(r: Realm) -> Set[str]: user_profile = UserProfile.objects.get(realm=r, short_name='hamlet') muted_topics = MutedTopic.objects.filter(user_profile=user_profile) topic_names = { muted_topic.topic_name for muted_topic in muted_topics } return topic_names assert_realm_values(get_muted_topics) # test usergroups assert_realm_values( lambda r: {group.name for group in UserGroup.objects.filter(realm=r)}) def get_user_membership(r: str) -> Set[str]: usergroup = UserGroup.objects.get(realm=r, name='hamletcharacters') usergroup_membership = UserGroupMembership.objects.filter( user_group=usergroup) users = { membership.user_profile.email for membership in usergroup_membership } return users assert_realm_values(get_user_membership) # test botstoragedata and botconfigdata def get_botstoragedata(r: Realm) -> Dict[str, Any]: bot_profile = UserProfile.objects.get(full_name="bot", realm=r) bot_storage_data = BotStorageData.objects.get( bot_profile=bot_profile) return { 'key': bot_storage_data.key, 'data': bot_storage_data.value } assert_realm_values(get_botstoragedata) def get_botconfigdata(r: Realm) -> Dict[str, Any]: bot_profile = UserProfile.objects.get(full_name="bot", realm=r) bot_config_data = BotConfigData.objects.get( bot_profile=bot_profile) return {'key': bot_config_data.key, 'data': bot_config_data.value} assert_realm_values(get_botconfigdata) # test messages def get_stream_messages(r: Realm) -> Message: recipient = get_recipient_stream(r) messages = Message.objects.filter(recipient=recipient) return messages def get_stream_topics(r: Realm) -> Set[str]: messages = get_stream_messages(r) topics = {m.topic_name() for m in messages} return topics assert_realm_values(get_stream_topics) # test usermessages def get_usermessages_user(r: Realm) -> Set[Any]: messages = get_stream_messages(r).order_by('content') usermessage = UserMessage.objects.filter(message=messages[0]) usermessage_user = {um.user_profile.email for um in usermessage} return usermessage_user assert_realm_values(get_usermessages_user) # tests to make sure that various data-*-ids in rendered_content # are replaced correctly with the values of newer realm. def get_user_mention(r: Realm) -> Set[Any]: mentioned_user = UserProfile.objects.get( delivery_email=self.example_email("hamlet"), realm=r) data_user_id = 'data-user-id="{}"'.format(mentioned_user.id) mention_message = get_stream_messages(r).get( rendered_content__contains=data_user_id) return mention_message.content assert_realm_values(get_user_mention) def get_stream_mention(r: Realm) -> Set[Any]: mentioned_stream = get_stream(u'Denmark', r) data_stream_id = 'data-stream-id="{}"'.format(mentioned_stream.id) mention_message = get_stream_messages(r).get( rendered_content__contains=data_stream_id) return mention_message.content assert_realm_values(get_stream_mention) def get_user_group_mention(r: Realm) -> Set[Any]: user_group = UserGroup.objects.get(realm=r, name='hamletcharacters') data_usergroup_id = 'data-user-group-id="{}"'.format(user_group.id) mention_message = get_stream_messages(r).get( rendered_content__contains=data_usergroup_id) return mention_message.content assert_realm_values(get_user_group_mention) # test to highlight that bs4 which we use to do data-**id # replacements modifies the HTML sometimes. eg replacing <br> # with </br>, ' with \' etc. The modifications doesn't # affect how the browser displays the rendered_content so we # are okay with using bs4 for this. lxml package also has # similar behavior. orig_polonius_user = UserProfile.objects.get( email=self.example_email("polonius"), realm=original_realm) original_msg = Message.objects.get(content=special_characters_message, sender__realm=original_realm) self.assertEqual(original_msg.rendered_content, ( '<div class="codehilite"><pre><span></span>'\n</pre></div>\n\n\n' '<p><span class="user-mention" data-user-id="%s">@Polonius</span></p>' % (orig_polonius_user.id, ))) imported_polonius_user = UserProfile.objects.get( email=self.example_email("polonius"), realm=imported_realm) imported_msg = Message.objects.get(content=special_characters_message, sender__realm=imported_realm) self.assertEqual(imported_msg.rendered_content, ( '<div class="codehilite"><pre><span></span>\'\n</pre></div>\n' '<p><span class="user-mention" data-user-id="%s">@Polonius</span></p>' % (imported_polonius_user.id, ))) # Check recipient_id was generated correctly for the imported users and streams. for user_profile in UserProfile.objects.filter(realm=imported_realm): self.assertEqual(user_profile.recipient_id, get_personal_recipient(user_profile.id).id) for stream in Stream.objects.filter(realm=imported_realm): self.assertEqual(stream.recipient_id, get_stream_recipient(stream.id).id)
def gather_subscriptions_helper( user_profile: UserProfile, include_subscribers: bool = True, ) -> SubscriptionInfo: realm = user_profile.realm all_streams: QuerySet[RawStreamDict] = get_active_streams(realm).values( *Stream.API_FIELDS, # The realm_id and recipient_id are generally not needed in the API. "realm_id", "recipient_id", # email_token isn't public to some users with access to # the stream, so doesn't belong in API_FIELDS. "email_token", ) recip_id_to_stream_id: Dict[int, int] = { stream["recipient_id"]: stream["id"] for stream in all_streams } all_streams_map: Dict[int, RawStreamDict] = {stream["id"]: stream for stream in all_streams} sub_dicts_query: Iterable[RawSubscriptionDict] = ( get_stream_subscriptions_for_user(user_profile) .values( *Subscription.API_FIELDS, "recipient_id", "active", ) .order_by("recipient_id") ) # We only care about subscriptions for active streams. sub_dicts: List[RawSubscriptionDict] = [ sub_dict for sub_dict in sub_dicts_query if recip_id_to_stream_id.get(sub_dict["recipient_id"]) ] def get_stream_id(sub_dict: RawSubscriptionDict) -> int: return recip_id_to_stream_id[sub_dict["recipient_id"]] traffic_stream_ids = {get_stream_id(sub_dict) for sub_dict in sub_dicts} recent_traffic = get_streams_traffic(stream_ids=traffic_stream_ids) # Okay, now we finally get to populating our main results, which # will be these three lists. subscribed: List[SubscriptionStreamDict] = [] unsubscribed: List[SubscriptionStreamDict] = [] never_subscribed: List[NeverSubscribedStreamDict] = [] sub_unsub_stream_ids = set() for sub_dict in sub_dicts: stream_id = get_stream_id(sub_dict) sub_unsub_stream_ids.add(stream_id) raw_stream_dict = all_streams_map[stream_id] stream_dict = build_stream_dict_for_sub( user=user_profile, sub_dict=sub_dict, raw_stream_dict=raw_stream_dict, recent_traffic=recent_traffic, ) # is_active is represented in this structure by which list we include it in. is_active = sub_dict["active"] if is_active: subscribed.append(stream_dict) else: unsubscribed.append(stream_dict) if user_profile.can_access_public_streams(): never_subscribed_stream_ids = set(all_streams_map) - sub_unsub_stream_ids else: web_public_stream_ids = {stream["id"] for stream in all_streams if stream["is_web_public"]} never_subscribed_stream_ids = web_public_stream_ids - sub_unsub_stream_ids never_subscribed_streams = [ all_streams_map[stream_id] for stream_id in never_subscribed_stream_ids ] for raw_stream_dict in never_subscribed_streams: is_public = not raw_stream_dict["invite_only"] if is_public or user_profile.is_realm_admin: slim_stream_dict = build_stream_dict_for_never_sub( raw_stream_dict=raw_stream_dict, recent_traffic=recent_traffic ) never_subscribed.append(slim_stream_dict) if include_subscribers: # The highly optimized bulk_get_subscriber_user_ids wants to know which # streams we are subscribed to, for validation purposes, and it uses that # info to know if it's allowed to find OTHER subscribers. subscribed_stream_ids = { get_stream_id(sub_dict) for sub_dict in sub_dicts if sub_dict["active"] } subscriber_map = bulk_get_subscriber_user_ids( all_streams, user_profile, subscribed_stream_ids, ) for lst in [subscribed, unsubscribed]: for stream_dict in lst: assert isinstance(stream_dict["stream_id"], int) stream_id = stream_dict["stream_id"] stream_dict["subscribers"] = subscriber_map[stream_id] for slim_stream_dict in never_subscribed: assert isinstance(slim_stream_dict["stream_id"], int) stream_id = slim_stream_dict["stream_id"] slim_stream_dict["subscribers"] = subscriber_map[stream_id] subscribed.sort(key=lambda x: x["name"]) unsubscribed.sort(key=lambda x: x["name"]) never_subscribed.sort(key=lambda x: x["name"]) return SubscriptionInfo( subscriptions=subscribed, unsubscribed=unsubscribed, never_subscribed=never_subscribed, )
def get_recent_streams(realm: Realm, threshold: datetime.datetime) -> List[Stream]: fields = ["id", "name", "is_web_public", "invite_only"] return list( get_active_streams(realm).filter(date_created__gt=threshold).only( *fields))