def update_active_status_backend(request: HttpRequest, user_profile: UserProfile, status: str=REQ(), ping_only: bool=REQ(validator=check_bool, default=False), new_user_input: bool=REQ(validator=check_bool, default=False) ) -> HttpResponse: status_val = UserPresence.status_from_string(status) if status_val is None: raise JsonableError(_("Invalid status: %s") % (status,)) else: update_user_presence(user_profile, request.client, timezone_now(), status_val, new_user_input) if ping_only: ret = {} # type: Dict[str, Any] else: ret = get_status_list(user_profile) if user_profile.realm.is_zephyr_mirror_realm: # In zephyr mirroring realms, users can't see the presence of other # users, but each user **is** interested in whether their mirror bot # (running as their user) has been active. try: activity = UserActivity.objects.get(user_profile = user_profile, query="get_events_backend", client__name="zephyr_mirror") ret['zephyr_mirror_active'] = \ (activity.last_visit > timezone_now() - datetime.timedelta(minutes=5)) except UserActivity.DoesNotExist: ret['zephyr_mirror_active'] = False return json_success(ret)
def update_messages_for_topic_edit(message: Message, propagate_mode: str, orig_topic_name: str, topic_name: str) -> List[Message]: propagate_query = Q(recipient = message.recipient, subject = orig_topic_name) # We only change messages up to 2 days in the past, to avoid hammering our # DB by changing an unbounded amount of messages if propagate_mode == 'change_all': before_bound = timezone_now() - datetime.timedelta(days=2) propagate_query = (propagate_query & ~Q(id = message.id) & Q(pub_date__range=(before_bound, timezone_now()))) if propagate_mode == 'change_later': propagate_query = propagate_query & Q(id__gt = message.id) messages = Message.objects.filter(propagate_query).select_related() # Evaluate the query before running the update messages_list = list(messages) messages.update(subject=topic_name) for m in messages_list: # The cached ORM object is not changed by messages.update() # and the remote cache update requires the new value m.set_topic_name(topic_name) return messages_list
def handle(self, *args: Any, **options: Any) -> None: if settings.EMAIL_DELIVERER_DISABLED: # Here doing a check and sleeping indefinitely on this setting might # not sound right. Actually we do this check to avoid running this # process on every server that might be in service to a realm. See # the comment in zproject/settings.py file about renaming this setting. sleep_forever() with lockfile("/tmp/zulip_scheduled_message_deliverer.lockfile"): while True: messages_to_deliver = ScheduledMessage.objects.filter( scheduled_timestamp__lte=timezone_now(), delivered=False) if messages_to_deliver: for message in messages_to_deliver: with transaction.atomic(): do_send_messages([self.construct_message(message)]) message.delivered = True message.save(update_fields=['delivered']) cur_time = timezone_now() time_next_min = (cur_time + timedelta(minutes=1)).replace(second=0, microsecond=0) sleep_time = (time_next_min - cur_time).total_seconds() time.sleep(sleep_time)
def test_expired_messages_in_each_realm(self): # type: () -> None # Check result realm messages order and result content # when all realm has expired messages. expired_mit_messages = self._make_mit_messages(3, timezone_now() - timedelta(days=101)) self._make_mit_messages(4, timezone_now() - timedelta(days=50)) zulip_messages_ids = Message.objects.order_by('id').filter( sender__realm=self.zulip_realm).values_list('id', flat=True)[3:10] expired_zulip_messages = self._change_messages_pub_date(zulip_messages_ids, timezone_now() - timedelta(days=31)) # Iterate by result expired_messages_result = [messages_list for messages_list in get_expired_messages()] self.assertEqual(len(expired_messages_result), 2) # Check mit.edu realm expired messages. self.assertEqual(len(expired_messages_result[0]['expired_messages']), 3) self.assertEqual(expired_messages_result[0]['realm_id'], self.mit_realm.id) # Check zulip.com realm expired messages. self.assertEqual(len(expired_messages_result[1]['expired_messages']), 7) self.assertEqual(expired_messages_result[1]['realm_id'], self.zulip_realm.id) # Compare expected messages ids with result messages ids. self.assertEqual( sorted([message.id for message in expired_mit_messages]), [message.id for message in expired_messages_result[0]['expired_messages']] ) self.assertEqual( sorted([message.id for message in expired_zulip_messages]), [message.id for message in expired_messages_result[1]['expired_messages']] )
def filter(self, record: logging.LogRecord) -> bool: from django.conf import settings from django.core.cache import cache # Track duplicate errors duplicate = False rate = getattr(settings, '%s_LIMIT' % self.__class__.__name__.upper(), 600) # seconds if rate > 0: # Test if the cache works try: cache.set('RLF_TEST_KEY', 1, 1) use_cache = cache.get('RLF_TEST_KEY') == 1 except Exception: use_cache = False if use_cache: if record.exc_info is not None: tb = '\n'.join(traceback.format_exception(*record.exc_info)) else: tb = str(record) key = self.__class__.__name__.upper() + hashlib.sha1(tb.encode()).hexdigest() duplicate = cache.get(key) == 1 if not duplicate: cache.set(key, 1, rate) else: min_date = timezone_now() - timedelta(seconds=rate) duplicate = (self.last_error >= min_date) if not duplicate: self.last_error = timezone_now() return not duplicate
def save(self, *args, **kwargs): if not self.pk: self.created = timezone_now() else: # 为了保证我们一直拥有创建数据,添加下面的条件语句 if not self.created: self.created = timezone_now() self.modified = timezone_now() super(CreationModificationDateMixin, self).save(*args, **kwargs)
def save(self, *args, **kwargs): if not self.pk: self.created = timezone_now() else: # To ensure we have a creation date always we add this one if not self.created: self.created = timezone_now() self.modified = timezone_now() super(CreationModificationDateMixin, self).save(*args, **kwargs)
def test_multiple_stream_senders(self, mock_send_future_email: mock.MagicMock, mock_enough_traffic: mock.MagicMock) -> None: client = 'website' # this makes `sent_by_human` return True othello = self.example_user('othello') self.subscribe(othello, 'Verona') one_day_ago = timezone_now() - datetime.timedelta(days=1) Message.objects.all().update(pub_date=one_day_ago) one_sec_ago = timezone_now() - datetime.timedelta(seconds=1) cutoff = time.mktime(one_sec_ago.timetuple()) senders = ['hamlet', 'cordelia', 'iago', 'prospero', 'ZOE'] for sender_name in senders: email = self.example_email(sender_name) self.login(email) content = 'some content for ' + email payload = dict( type='stream', client=client, to='Verona', topic='lunch', content=content, ) result = self.client_post("/json/messages", payload) self.assert_json_success(result) flush_per_request_caches() with queries_captured() as queries: handle_digest_email(othello.id, cutoff) self.assertTrue(24 <= len(queries) <= 25) self.assertEqual(mock_send_future_email.call_count, 1) kwargs = mock_send_future_email.call_args[1] self.assertEqual(kwargs['to_user_ids'], [othello.id]) hot_convo = kwargs['context']['hot_conversations'][0] expected_participants = { self.example_user(sender).full_name for sender in senders } self.assertEqual(set(hot_convo['participants']), expected_participants) self.assertEqual(hot_convo['count'], 5 - 2) # 5 messages, but 2 shown teaser_messages = hot_convo['first_few_messages'][0]['senders'] self.assertIn('some content', teaser_messages[0]['content'][0]['plain']) self.assertIn(teaser_messages[0]['sender'], expected_participants)
def gitter_workspace_to_realm(domain_name: str, gitter_data: GitterDataT, realm_subdomain: str) -> Tuple[ZerverFieldsT, List[ZerverFieldsT], Dict[str, int]]: """ Returns: 1. realm, Converted Realm data 2. avatars, which is list to map avatars to zulip avatar records.json 3. user_map, which is a dictionary to map from gitter user id to zulip user id """ NOW = float(timezone_now().timestamp()) zerver_realm = build_zerver_realm(realm_id, realm_subdomain, NOW, 'Gitter') # type: List[ZerverFieldsT] realm = build_realm(zerver_realm, realm_id, domain_name) zerver_userprofile, avatars, user_map = build_userprofile(int(NOW), domain_name, gitter_data) zerver_stream, zerver_defaultstream = build_stream_and_defaultstream(int(NOW)) zerver_recipient, zerver_subscription = build_recipient_and_subscription( zerver_userprofile, zerver_stream) realm['zerver_userprofile'] = zerver_userprofile realm['zerver_stream'] = zerver_stream realm['zerver_defaultstream'] = zerver_defaultstream realm['zerver_recipient'] = zerver_recipient realm['zerver_subscription'] = zerver_subscription return realm, avatars, user_map
def test_disabled(self, mock_django_timezone: mock.MagicMock, mock_queue_digest_recipient: mock.MagicMock) -> None: cutoff = timezone_now() # A Tuesday mock_django_timezone.return_value = datetime.datetime(year=2016, month=1, day=5) enqueue_emails(cutoff) mock_queue_digest_recipient.assert_not_called()
def compute_activity(self, user_activity_objects: QuerySet) -> None: # Report data from the past week. # # This is a rough report of client activity because we inconsistently # register activity from various clients; think of it as telling you # approximately how many people from a group have used a particular # client recently. For example, this might be useful to get a sense of # how popular different versions of a desktop client are. # # Importantly, this does NOT tell you anything about the relative # volumes of requests from clients. threshold = timezone_now() - datetime.timedelta(days=7) client_counts = user_activity_objects.filter( last_visit__gt=threshold).values("client__name").annotate( count=Count('client__name')) total = 0 counts = [] for client_type in client_counts: count = client_type["count"] client = client_type["client__name"] total += count counts.append((count, client)) counts.sort() for count in counts: print("%25s %15d" % (count[1], count[0])) print("Total:", total)
def get_presence_backend(request: HttpRequest, user_profile: UserProfile, email: Text) -> HttpResponse: try: target = get_user(email, user_profile.realm) except UserProfile.DoesNotExist: return json_error(_('No such user')) if not target.is_active: return json_error(_('No such user')) if target.is_bot: return json_error(_('Presence is not supported for bot users.')) presence_dict = UserPresence.get_status_dict_by_user(target) if len(presence_dict) == 0: return json_error(_('No presence data for %s' % (target.email,))) # For initial version, we just include the status and timestamp keys result = dict(presence=presence_dict[target.email]) aggregated_info = result['presence']['aggregated'] aggr_status_duration = datetime_to_timestamp(timezone_now()) - aggregated_info['timestamp'] if aggr_status_duration > settings.OFFLINE_THRESHOLD_SECS: aggregated_info['status'] = 'offline' for val in result['presence'].values(): val.pop('client', None) val.pop('pushable', None) return json_success(result)
def get_realm_expired_messages(realm: Any) -> Optional[Dict[str, Any]]: expired_date = timezone_now() - timedelta(days=realm.message_retention_days) expired_messages = Message.objects.order_by('id').filter(sender__realm=realm, pub_date__lt=expired_date) if not expired_messages.exists(): return None return {'realm_id': realm.id, 'expired_messages': expired_messages}
def create_confirmation_link(obj: ContentType, host: str, confirmation_type: int, url_args: Optional[Dict[str, str]]=None) -> str: key = generate_key() Confirmation.objects.create(content_object=obj, date_sent=timezone_now(), confirmation_key=key, realm=obj.realm, type=confirmation_type) return confirmation_url(key, host, confirmation_type, url_args)
def delete_realm_user_sessions(realm): # type: (Realm) -> None realm_user_ids = [user_profile.id for user_profile in UserProfile.objects.filter(realm=realm)] for session in Session.objects.filter(expire_date__gte=timezone_now()): if get_session_user(session) in realm_user_ids: delete_session(session)
def check_key_is_valid(creation_key: Text) -> bool: if not RealmCreationKey.objects.filter(creation_key=creation_key).exists(): return False time_elapsed = timezone_now() - RealmCreationKey.objects.get(creation_key=creation_key).date_created if time_elapsed.total_seconds() > settings.REALM_CREATION_LINK_VALIDITY_DAYS * 24 * 3600: return False return True
def send_future_email(template_prefix: str, realm: Realm, to_user_id: Optional[int]=None, to_email: Optional[str]=None, from_name: Optional[str]=None, from_address: Optional[str]=None, context: Dict[str, Any]={}, delay: datetime.timedelta=datetime.timedelta(0)) -> None: template_name = template_prefix.split('/')[-1] email_fields = {'template_prefix': template_prefix, 'to_user_id': to_user_id, 'to_email': to_email, 'from_name': from_name, 'from_address': from_address, 'context': context} if settings.DEVELOPMENT and not settings.TEST_SUITE: send_email(template_prefix, to_user_id=to_user_id, to_email=to_email, from_name=from_name, from_address=from_address, context=context) # For logging the email assert (to_user_id is None) ^ (to_email is None) if to_user_id is not None: # The realm is redundant if we have a to_user_id; this assert just # expresses that fact assert(UserProfile.objects.filter(id=to_user_id, realm=realm).exists()) to_field = {'user_id': to_user_id} # type: Dict[str, Any] else: to_field = {'address': parseaddr(to_email)[1]} ScheduledEmail.objects.create( type=EMAIL_TYPES[template_name], scheduled_timestamp=timezone_now() + delay, realm=realm, data=ujson.dumps(email_fields), **to_field)
def generate_realm_creation_url() -> Text: key = generate_key() RealmCreationKey.objects.create(creation_key=key, date_created=timezone_now()) return u'%s%s%s' % (settings.EXTERNAL_URI_SCHEME, settings.EXTERNAL_HOST, reverse('zerver.views.create_realm', kwargs={'creation_key': key}))
def test_show_billing(self) -> None: customer = Customer.objects.create(realm=get_realm("zulip"), stripe_customer_id="cus_id") # realm admin, but no CustomerPlan -> no billing link user = self.example_user('iago') self.login(user.email) result_html = self._get_home_page().content.decode('utf-8') self.assertNotIn('Billing', result_html) # realm admin, with inactive CustomerPlan -> show billing link CustomerPlan.objects.create(customer=customer, billing_cycle_anchor=timezone_now(), billing_schedule=CustomerPlan.ANNUAL, next_invoice_date=timezone_now(), tier=CustomerPlan.STANDARD, status=CustomerPlan.ENDED) result_html = self._get_home_page().content.decode('utf-8') self.assertIn('Billing', result_html) # billing admin, with CustomerPlan -> show billing link user.is_realm_admin = False user.is_billing_admin = True user.save(update_fields=['is_realm_admin', 'is_billing_admin']) result_html = self._get_home_page().content.decode('utf-8') self.assertIn('Billing', result_html) # billing admin, but no CustomerPlan -> no billing link CustomerPlan.objects.all().delete() result_html = self._get_home_page().content.decode('utf-8') self.assertNotIn('Billing', result_html) # billing admin, no customer object -> make sure it doesn't crash customer.delete() result = self._get_home_page() self.assertEqual(result.status_code, 200)
def create_user_profile(realm: Realm, email: str, password: Optional[str], active: bool, bot_type: Optional[int], full_name: str, short_name: str, bot_owner: Optional[UserProfile], is_mirror_dummy: bool, tos_version: Optional[str], timezone: Optional[str], tutorial_status: Optional[str] = UserProfile.TUTORIAL_WAITING, enter_sends: bool = False) -> UserProfile: now = timezone_now() email = UserManager.normalize_email(email) user_profile = UserProfile(is_staff=False, is_active=active, full_name=full_name, short_name=short_name, last_login=now, date_joined=now, realm=realm, pointer=-1, is_bot=bool(bot_type), bot_type=bot_type, bot_owner=bot_owner, is_mirror_dummy=is_mirror_dummy, tos_version=tos_version, timezone=timezone, tutorial_status=tutorial_status, enter_sends=enter_sends, onboarding_steps=ujson.dumps([]), default_language=realm.default_language, twenty_four_hour_time=realm.default_twenty_four_hour_time, delivery_email=email) if bot_type or not active: password = None if realm.email_address_visibility == Realm.EMAIL_ADDRESS_VISIBILITY_EVERYONE: # If emails are visible to everyone, we can set this here and save a DB query user_profile.email = get_display_email_address(user_profile, realm) user_profile.set_password(password) user_profile.api_key = generate_api_key() return user_profile
def test_regenerate_api_key(self) -> None: now = timezone_now() user = self.example_user('hamlet') do_regenerate_api_key(user, user) self.assertEqual(RealmAuditLog.objects.filter(event_type=RealmAuditLog.USER_API_KEY_CHANGED, event_time__gte=now).count(), 1) self.assertTrue(user.api_key)
def receiver_is_idle(user_profile_id, realm_presences): # type: (int, Optional[Dict[int, Dict[Text, Dict[str, Any]]]]) -> bool # If a user has no message-receiving event queues, they've got no open zulip # session so we notify them all_client_descriptors = get_client_descriptors_for_user(user_profile_id) message_event_queues = [client for client in all_client_descriptors if client.accepts_messages()] off_zulip = len(message_event_queues) == 0 # It's possible a recipient is not in the realm of a sender. We don't have # presence information in this case (and it's hard to get without an additional # db query) so we simply don't try to guess if this cross-realm recipient # has been idle for too long if realm_presences is None or user_profile_id not in realm_presences: return off_zulip # We want to find the newest "active" presence entity and compare that to the # activity expiry threshold. user_presence = realm_presences[user_profile_id] latest_active_timestamp = None idle = False for client, status in six.iteritems(user_presence): if (latest_active_timestamp is None or status['timestamp'] > latest_active_timestamp) and \ status['status'] == 'active': latest_active_timestamp = status['timestamp'] if latest_active_timestamp is None: idle = True else: active_datetime = timestamp_to_datetime(latest_active_timestamp) # 140 seconds is consistent with presence.js:OFFLINE_THRESHOLD_SECS idle = timezone_now() - active_datetime > datetime.timedelta(seconds=140) return off_zulip or idle
def save(self): user = self._user if user is None: raise RuntimeError("ResponseForm without user, can't be saved.") logger.debug("Saving response data to database and requesing doing update to submission_url") # Get the instance and update metadata instance = super().save(commit=False) instance.response_time = timezone_now() instance.response_by = user instance.response_notify = self.get_notify() # prepare for upload upload = async_response_upload(instance) if self.has_changed() else None # save to db and update form internal state fields = self._meta.fields + ('response_time', 'response_by', 'response_notify') instance.save(update_fields=fields) self.original_fields(instance, update=True) # if there is upload, do it now after instance is saved if upload: logger.debug("Instance has changes, requesting upload to aplus.") upload() else: logger.debug("No changes to response, so no aplus updated needed.") return instance
def email_on_new_login(sender: Any, user: UserProfile, request: Any, **kwargs: Any) -> None: # We import here to minimize the dependencies of this module, # since it runs as part of `manage.py` initialization from zerver.context_processors import common_context if not settings.SEND_LOGIN_EMAILS: return if request: # If the user's account was just created, avoid sending an email. if getattr(user, "just_registered", False): return user_agent = request.META.get('HTTP_USER_AGENT', "").lower() context = common_context(user) context['user_email'] = user.email user_tz = user.timezone if user_tz == '': user_tz = timezone_get_current_timezone_name() local_time = timezone_now().astimezone(get_timezone(user_tz)) context['login_time'] = local_time.strftime('%A, %B %d, %Y at %I:%M%p ') + user_tz context['device_ip'] = request.META.get('REMOTE_ADDR') or _("Unknown IP address") context['device_os'] = get_device_os(user_agent) context['device_browser'] = get_device_browser(user_agent) email_dict = { 'template_prefix': 'zerver/emails/notify_new_login', 'to_user_id': user.id, 'from_name': 'Zulip Account Security', 'from_address': FromAddress.NOREPLY, 'context': context} queue_json_publish("email_senders", email_dict)
def test_delete_old_unclaimed_attachments(self) -> None: # Upload some files and make them older than a weeek self.login(self.example_email("hamlet")) d1 = StringIO("zulip!") d1.name = "dummy_1.txt" result = self.client_post("/json/user_uploads", {'file': d1}) d1_path_id = re.sub('/user_uploads/', '', result.json()['uri']) d2 = StringIO("zulip!") d2.name = "dummy_2.txt" result = self.client_post("/json/user_uploads", {'file': d2}) d2_path_id = re.sub('/user_uploads/', '', result.json()['uri']) two_week_ago = timezone_now() - datetime.timedelta(weeks=2) d1_attachment = Attachment.objects.get(path_id = d1_path_id) d1_attachment.create_time = two_week_ago d1_attachment.save() self.assertEqual(str(d1_attachment), u'<Attachment: dummy_1.txt>') d2_attachment = Attachment.objects.get(path_id = d2_path_id) d2_attachment.create_time = two_week_ago d2_attachment.save() # Send message referring only dummy_1 self.subscribe(self.example_user("hamlet"), "Denmark") body = "Some files here ...[zulip.txt](http://localhost:9991/user_uploads/" + d1_path_id + ")" self.send_stream_message(self.example_email("hamlet"), "Denmark", body, "test") # dummy_2 should not exist in database or the uploads folder do_delete_old_unclaimed_attachments(2) self.assertTrue(not Attachment.objects.filter(path_id = d2_path_id).exists()) self.assertTrue(not delete_message_image(d2_path_id))
def create_user_profile(realm, email, password, active, bot_type, full_name, short_name, bot_owner, is_mirror_dummy, tos_version, timezone, tutorial_status=UserProfile.TUTORIAL_WAITING, enter_sends=False): # type: (Realm, Text, Optional[Text], bool, Optional[int], Text, Text, Optional[UserProfile], bool, Text, Optional[Text], Optional[Text], bool) -> UserProfile now = timezone_now() email = UserManager.normalize_email(email) user_profile = UserProfile(email=email, is_staff=False, is_active=active, full_name=full_name, short_name=short_name, last_login=now, date_joined=now, realm=realm, pointer=-1, is_bot=bool(bot_type), bot_type=bot_type, bot_owner=bot_owner, is_mirror_dummy=is_mirror_dummy, tos_version=tos_version, timezone=timezone, tutorial_status=tutorial_status, enter_sends=enter_sends, onboarding_steps=ujson.dumps([]), default_language=realm.default_language) if bot_type or not active: password = None user_profile.set_password(password) user_profile.api_key = random_api_key() return user_profile
def create_user_profile(realm: Realm, email: str, password: Optional[str], active: bool, bot_type: Optional[int], full_name: str, short_name: str, bot_owner: Optional[UserProfile], is_mirror_dummy: bool, tos_version: Optional[str], timezone: Optional[str], tutorial_status: Optional[str] = UserProfile.TUTORIAL_WAITING, enter_sends: bool = False) -> UserProfile: now = timezone_now() email = UserManager.normalize_email(email) user_profile = UserProfile(email=email, is_staff=False, is_active=active, full_name=full_name, short_name=short_name, last_login=now, date_joined=now, realm=realm, pointer=-1, is_bot=bool(bot_type), bot_type=bot_type, bot_owner=bot_owner, is_mirror_dummy=is_mirror_dummy, tos_version=tos_version, timezone=timezone, tutorial_status=tutorial_status, enter_sends=enter_sends, onboarding_steps=ujson.dumps([]), default_language=realm.default_language, twenty_four_hour_time=realm.default_twenty_four_hour_time, delivery_email=email) if bot_type or not active: password = None user_profile.set_password(password) user_profile.api_key = generate_api_key() return user_profile
def subscribe_users_to_streams(realm: Realm, stream_dict: Dict[str, Dict[str, Any]]) -> None: subscriptions_to_add = [] event_time = timezone_now() all_subscription_logs = [] profiles = UserProfile.objects.select_related().filter(realm=realm) for i, stream_name in enumerate(stream_dict): stream = Stream.objects.get(name=stream_name, realm=realm) recipient = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id) for profile in profiles: # Subscribe to some streams. s = Subscription( recipient=recipient, user_profile=profile, color=STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)]) subscriptions_to_add.append(s) log = RealmAuditLog(realm=profile.realm, modified_user=profile, modified_stream=stream, event_last_message_id=0, event_type=RealmAuditLog.SUBSCRIPTION_CREATED, event_time=event_time) all_subscription_logs.append(log) Subscription.objects.bulk_create(subscriptions_to_add) RealmAuditLog.objects.bulk_create(all_subscription_logs)
def test_get_users_for_soft_deactivation(self) -> None: users = [ self.example_user('hamlet'), self.example_user('iago'), self.example_user('cordelia'), self.example_user('ZOE'), self.example_user('othello'), self.example_user('prospero'), self.example_user('aaron'), ] client, _ = Client.objects.get_or_create(name='website') query = '/json/users/me/pointer' last_visit = timezone_now() count = 150 for user_profile in UserProfile.objects.all(): UserActivity.objects.get_or_create( user_profile=user_profile, client=client, query=query, count=count, last_visit=last_visit ) filter_kwargs = dict(user_profile__realm=get_realm('zulip')) users_to_deactivate = get_users_for_soft_deactivation(-1, filter_kwargs) self.assert_length(users_to_deactivate, 7) for user in users_to_deactivate: self.assertTrue(user in users)
def get_mirror_user(self, realm_id: int, name: str) -> Dict[str, Any]: if name in self.name_to_mirror_user_map: user = self.name_to_mirror_user_map[name] return user user_id = self._new_mirror_user_id() short_name = name full_name = name email = 'mirror-{user_id}@example.com'.format(user_id=user_id) delivery_email = email avatar_source = 'G' date_joined = int(timezone_now().timestamp()) timezone = 'UTC' user = build_user_profile( avatar_source=avatar_source, date_joined=date_joined, delivery_email=delivery_email, email=email, full_name=full_name, id=user_id, is_active=False, is_realm_admin=False, is_guest=False, is_mirror_dummy=True, realm_id=realm_id, short_name=short_name, timezone=timezone, ) self.name_to_mirror_user_map[name] = user return user
def check_update_message( user_profile: UserProfile, message_id: int, stream_id: Optional[int] = None, topic_name: Optional[str] = None, propagate_mode: str = "change_one", send_notification_to_old_thread: bool = True, send_notification_to_new_thread: bool = True, content: Optional[str] = None, ) -> int: """This will update a message given the message id and user profile. It checks whether the user profile has the permission to edit the message and raises a JsonableError if otherwise. It returns the number changed. """ message, ignored_user_message = access_message(user_profile, message_id) if not user_profile.realm.allow_message_editing: raise JsonableError( _("Your organization has turned off message editing")) # The zerver/views/message_edit.py call point already strips this # via REQ_topic; so we can delete this line if we arrange a # contract where future callers in the embedded bots system strip # use REQ_topic as well (or otherwise are guaranteed to strip input). if topic_name is not None: topic_name = topic_name.strip() if topic_name == message.topic_name(): topic_name = None validate_message_edit_payload(message, stream_id, topic_name, propagate_mode, content) is_no_topic_msg = message.topic_name() == "(no topic)" if content is not None or topic_name is not None: if not can_edit_content_or_topic(message, user_profile, is_no_topic_msg, content, topic_name): raise JsonableError( _("You don't have permission to edit this message")) # If there is a change to the content, check that it hasn't been too long # Allow an extra 20 seconds since we potentially allow editing 15 seconds # past the limit, and in case there are network issues, etc. The 15 comes # from (min_seconds_to_edit + seconds_left_buffer) in message_edit.js; if # you change this value also change those two parameters in message_edit.js. edit_limit_buffer = 20 if content is not None and user_profile.realm.message_content_edit_limit_seconds > 0: deadline_seconds = user_profile.realm.message_content_edit_limit_seconds + edit_limit_buffer if (timezone_now() - message.date_sent) > datetime.timedelta( seconds=deadline_seconds): raise JsonableError( _("The time limit for editing this message has passed")) # If there is a change to the topic, check that the user is allowed to # edit it and that it has not been too long. If this is not the user who # sent the message, they are not the admin, and the time limit for editing # topics is passed, raise an error. if (topic_name is not None and message.sender != user_profile and not user_profile.is_realm_admin and not user_profile.is_moderator and not is_no_topic_msg): deadline_seconds = Realm.DEFAULT_COMMUNITY_TOPIC_EDITING_LIMIT_SECONDS + edit_limit_buffer if (timezone_now() - message.date_sent) > datetime.timedelta( seconds=deadline_seconds): raise JsonableError( _("The time limit for editing this message's topic has passed") ) rendering_result = None links_for_embed: Set[str] = set() prior_mention_user_ids: Set[int] = set() mention_data: Optional[MentionData] = None if content is not None: if content.rstrip() == "": content = "(deleted)" content = normalize_body(content) mention_backend = MentionBackend(user_profile.realm_id) mention_data = MentionData( mention_backend=mention_backend, content=content, ) prior_mention_user_ids = get_mentions_for_message_updates(message.id) # We render the message using the current user's realm; since # the cross-realm bots never edit messages, this should be # always correct. # Note: If rendering fails, the called code will raise a JsonableError. rendering_result = render_incoming_message( message, content, user_profile.realm, mention_data=mention_data, ) links_for_embed |= rendering_result.links_for_preview if message.is_stream_message() and rendering_result.mentions_wildcard: stream = access_stream_by_id(user_profile, message.recipient.type_id)[0] if not wildcard_mention_allowed(message.sender, stream): raise JsonableError( _("You do not have permission to use wildcard mentions in this stream." )) new_stream = None number_changed = 0 if stream_id is not None: assert message.is_stream_message() if not user_profile.can_move_messages_between_streams(): raise JsonableError( _("You don't have permission to move this message")) try: access_stream_by_id(user_profile, message.recipient.type_id) except JsonableError: raise JsonableError( _("You don't have permission to move this message due to missing access to its stream" )) new_stream = access_stream_by_id(user_profile, stream_id, require_active=True)[0] check_stream_access_based_on_stream_post_policy( user_profile, new_stream) number_changed = do_update_message( user_profile, message, new_stream, topic_name, propagate_mode, send_notification_to_old_thread, send_notification_to_new_thread, content, rendering_result, prior_mention_user_ids, mention_data, ) if links_for_embed: event_data = { "message_id": message.id, "message_content": message.content, # The choice of `user_profile.realm_id` rather than # `sender.realm_id` must match the decision made in the # `render_incoming_message` call earlier in this function. "message_realm_id": user_profile.realm_id, "urls": list(links_for_embed), } queue_json_publish("embed_links", event_data) return number_changed
def send_messages( data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int] ) -> int: (tot_messages, personals_pairs, options, output, random_seed) = data random.seed(random_seed) with open("var/test_messages.json", "r") as infile: dialog = ujson.load(infile) random.shuffle(dialog) texts = itertools.cycle(dialog) recipient_streams = [ klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] # type: List[int] recipient_huddles = [ h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE) ] # type: List[int] huddle_members = {} # type: Dict[int, List[int]] for h in recipient_huddles: huddle_members[h] = [ s.user_profile.id for s in Subscription.objects.filter(recipient_id=h) ] num_messages = 0 random_max = 1000000 recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]] while num_messages < tot_messages: saved_data = {} # type: Dict[str, Any] message = Message() message.sending_client = get_client('populate_db') message.content = next(texts) randkey = random.randint(1, random_max) if (num_messages > 0 and random.randint(1, random_max) * 100. / random_max < options["stickyness"]): # Use an old recipient message_type, recipient_id, saved_data = recipients[num_messages - 1] if message_type == Recipient.PERSONAL: personals_pair = saved_data['personals_pair'] random.shuffle(personals_pair) elif message_type == Recipient.STREAM: message.subject = saved_data['subject'] message.recipient = get_recipient_by_id(recipient_id) elif message_type == Recipient.HUDDLE: message.recipient = get_recipient_by_id(recipient_id) elif (randkey <= random_max * options["percent_huddles"] / 100.): message_type = Recipient.HUDDLE message.recipient = get_recipient_by_id( random.choice(recipient_huddles)) elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.): message_type = Recipient.PERSONAL personals_pair = random.choice(personals_pairs) random.shuffle(personals_pair) elif (randkey <= random_max * 1.0): message_type = Recipient.STREAM message.recipient = get_recipient_by_id( random.choice(recipient_streams)) if message_type == Recipient.HUDDLE: sender_id = random.choice(huddle_members[message.recipient.id]) message.sender = get_user_profile_by_id(sender_id) elif message_type == Recipient.PERSONAL: message.recipient = Recipient.objects.get( type=Recipient.PERSONAL, type_id=personals_pair[0]) message.sender = get_user_profile_by_id(personals_pair[1]) saved_data['personals_pair'] = personals_pair elif message_type == Recipient.STREAM: stream = Stream.objects.get(id=message.recipient.type_id) # Pick a random subscriber to the stream message.sender = random.choice( Subscription.objects.filter( recipient=message.recipient)).user_profile message.subject = stream.name + Text(random.randint(1, 3)) saved_data['subject'] = message.subject # Spoofing time not supported with threading if options['threads'] != 1: message.pub_date = timezone_now() else: # Distrubutes 80% of messages starting from 5 days ago, over a period # of 3 days. Then, distributes remaining messages over past 24 hours. spoofed_date = timezone_now() - timezone_timedelta(days=5) if (num_messages < tot_messages * 0.8): # Maximum of 3 days ahead, convert to minutes time_ahead = 3 * 24 * 60 time_ahead //= int(tot_messages * 0.8) else: time_ahead = 24 * 60 time_ahead //= int(tot_messages * 0.2) spoofed_minute = random.randint(time_ahead * num_messages, time_ahead * (num_messages + 1)) spoofed_date += timezone_timedelta(minutes=spoofed_minute) message.pub_date = spoofed_date # We disable USING_RABBITMQ here, so that deferred work is # executed in do_send_message_messages, rather than being # queued. This is important, because otherwise, if run-dev.py # wasn't running when populate_db was run, a developer can end # up with queued events that reference objects from a previous # life of the database, which naturally throws exceptions. settings.USING_RABBITMQ = False do_send_messages([{'message': message}]) settings.USING_RABBITMQ = True recipients[num_messages] = (message_type, message.recipient.id, saved_data) num_messages += 1 return tot_messages
def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile, missed_messages: List[Dict[ str, Any]], message_count: int) -> None: """ Send a reminder email to a user if she's missed some PMs by being offline. The email will have its reply to address set to a limited used email address that will send a zulip message to the correct recipient. This allows the user to respond to missed PMs, huddles, and @-mentions directly from the email. `user_profile` is the user to send the reminder to `missed_messages` is a list of dictionaries to Message objects and other data for a group of messages that share a recipient (and topic) """ from zerver.context_processors import common_context # Disabled missedmessage emails internally if not user_profile.enable_offline_email_notifications: return recipients = set((msg['message'].recipient_id, msg['message'].topic_name()) for msg in missed_messages) if len(recipients) != 1: raise ValueError( 'All missed_messages must have the same recipient and topic %r' % (recipients, )) # This link is no longer a part of the email, but keeping the code in case # we find a clean way to add it back in the future unsubscribe_link = one_click_unsubscribe_link(user_profile, "missed_messages") context = common_context(user_profile) context.update({ 'name': user_profile.full_name, 'message_count': message_count, 'unsubscribe_link': unsubscribe_link, 'realm_name_in_notifications': user_profile.realm_name_in_notifications, 'show_message_content': message_content_allowed_in_missedmessage_emails(user_profile) }) triggers = list(message['trigger'] for message in missed_messages) unique_triggers = set(triggers) context.update({ 'mention': 'mentioned' in unique_triggers or 'wildcard_mentioned' in unique_triggers, 'stream_email_notify': 'stream_email_notify' in unique_triggers, 'mention_count': triggers.count('mentioned') + triggers.count("wildcard_mentioned"), }) # If this setting (email mirroring integration) is enabled, only then # can users reply to email to send message to Zulip. Thus, one must # ensure to display warning in the template. if settings.EMAIL_GATEWAY_PATTERN: context.update({ 'reply_to_zulip': True, }) else: context.update({ 'reply_to_zulip': False, }) from zerver.lib.email_mirror import create_missed_message_address reply_to_address = create_missed_message_address( user_profile, missed_messages[0]['message']) if reply_to_address == FromAddress.NOREPLY: reply_to_name = None else: reply_to_name = "Zulip" narrow_url = get_narrow_url(user_profile, missed_messages[0]['message']) context.update({ 'narrow_url': narrow_url, }) senders = list(set(m['message'].sender for m in missed_messages)) if (missed_messages[0]['message'].recipient.type == Recipient.HUDDLE): display_recipient = get_display_recipient( missed_messages[0]['message'].recipient) # Make sure that this is a list of strings, not a string. assert not isinstance(display_recipient, str) other_recipients = [ r['full_name'] for r in display_recipient if r['id'] != user_profile.id ] context.update({'group_pm': True}) if len(other_recipients) == 2: huddle_display_name = " and ".join(other_recipients) context.update({'huddle_display_name': huddle_display_name}) elif len(other_recipients) == 3: huddle_display_name = "%s, %s, and %s" % ( other_recipients[0], other_recipients[1], other_recipients[2]) context.update({'huddle_display_name': huddle_display_name}) else: huddle_display_name = "%s, and %s others" % (', '.join( other_recipients[:2]), len(other_recipients) - 2) context.update({'huddle_display_name': huddle_display_name}) elif (missed_messages[0]['message'].recipient.type == Recipient.PERSONAL): context.update({'private_message': True}) elif (context['mention'] or context['stream_email_notify']): # Keep only the senders who actually mentioned the user if context['mention']: senders = list( set(m['message'].sender for m in missed_messages if m['trigger'] == 'mentioned' or m['trigger'] == 'wildcard_mentioned')) message = missed_messages[0]['message'] stream = Stream.objects.only('id', 'name').get(id=message.recipient.type_id) stream_header = "%s > %s" % (stream.name, message.topic_name()) context.update({ 'stream_header': stream_header, }) else: raise AssertionError("Invalid messages!") # If message content is disabled, then flush all information we pass to email. if not message_content_allowed_in_missedmessage_emails(user_profile): context.update({ 'reply_to_zulip': False, 'messages': [], 'sender_str': "", 'realm_str': user_profile.realm.name, 'huddle_display_name': "", }) else: context.update({ 'messages': build_message_list(user_profile, list(m['message'] for m in missed_messages)), 'sender_str': ", ".join(sender.full_name for sender in senders), 'realm_str': user_profile.realm.name, }) from_name = "Zulip missed messages" # type: str from_address = FromAddress.NOREPLY if len(senders) == 1 and settings.SEND_MISSED_MESSAGE_EMAILS_AS_USER: # If this setting is enabled, you can reply to the Zulip # missed message emails directly back to the original sender. # However, one must ensure the Zulip server is in the SPF # record for the domain, or there will be spam/deliverability # problems. sender = senders[0] from_name, from_address = (sender.full_name, sender.email) context.update({ 'reply_to_zulip': False, }) email_dict = { 'template_prefix': 'zerver/emails/missed_message', 'to_user_ids': [user_profile.id], 'from_name': from_name, 'from_address': from_address, 'reply_to_email': formataddr((reply_to_name, reply_to_address)), 'context': context } queue_json_publish("email_senders", email_dict) user_profile.last_reminder = timezone_now() user_profile.save(update_fields=['last_reminder'])
def test_new_user_input(self, unused_mock: mock.Mock) -> None: """Mostly a test for UserActivityInterval""" user_profile = self.example_user("hamlet") self.login('hamlet') self.assertEqual( UserActivityInterval.objects.filter( user_profile=user_profile).count(), 0) time_zero = timezone_now().replace(microsecond=0) with mock.patch('zerver.views.presence.timezone_now', return_value=time_zero): result = self.client_post("/json/users/me/presence", { 'status': 'active', 'new_user_input': 'true' }) self.assert_json_success(result) self.assertEqual( UserActivityInterval.objects.filter( user_profile=user_profile).count(), 1) interval = UserActivityInterval.objects.get(user_profile=user_profile) self.assertEqual(interval.start, time_zero) self.assertEqual(interval.end, time_zero + UserActivityInterval.MIN_INTERVAL_LENGTH) second_time = time_zero + timedelta(seconds=600) # Extent the interval with mock.patch('zerver.views.presence.timezone_now', return_value=second_time): result = self.client_post("/json/users/me/presence", { 'status': 'active', 'new_user_input': 'true' }) self.assert_json_success(result) self.assertEqual( UserActivityInterval.objects.filter( user_profile=user_profile).count(), 1) interval = UserActivityInterval.objects.get(user_profile=user_profile) self.assertEqual(interval.start, time_zero) self.assertEqual( interval.end, second_time + UserActivityInterval.MIN_INTERVAL_LENGTH) third_time = time_zero + timedelta(seconds=6000) with mock.patch('zerver.views.presence.timezone_now', return_value=third_time): result = self.client_post("/json/users/me/presence", { 'status': 'active', 'new_user_input': 'true' }) self.assert_json_success(result) self.assertEqual( UserActivityInterval.objects.filter( user_profile=user_profile).count(), 2) interval = UserActivityInterval.objects.filter( user_profile=user_profile).order_by('start')[0] self.assertEqual(interval.start, time_zero) self.assertEqual( interval.end, second_time + UserActivityInterval.MIN_INTERVAL_LENGTH) interval = UserActivityInterval.objects.filter( user_profile=user_profile).order_by('start')[1] self.assertEqual(interval.start, third_time) self.assertEqual(interval.end, third_time + UserActivityInterval.MIN_INTERVAL_LENGTH) self.assertEqual( seconds_usage_between(user_profile, time_zero, third_time).total_seconds(), 1500) self.assertEqual( seconds_usage_between(user_profile, time_zero, third_time + timedelta(seconds=10)).total_seconds(), 1510) self.assertEqual( seconds_usage_between(user_profile, time_zero, third_time + timedelta(seconds=1000)).total_seconds(), 2400) self.assertEqual( seconds_usage_between(user_profile, time_zero, third_time - timedelta(seconds=100)).total_seconds(), 1500) self.assertEqual( seconds_usage_between( user_profile, time_zero + timedelta(seconds=100), third_time - timedelta(seconds=100)).total_seconds(), 1400) self.assertEqual( seconds_usage_between( user_profile, time_zero + timedelta(seconds=1200), third_time - timedelta(seconds=100)).total_seconds(), 300) # Now test /activity with actual data user_profile.is_staff = True user_profile.save() result = self.client_get('/activity') self.assertEqual(result.status_code, 200)
def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile, missed_messages: List[Dict[ str, Any]], message_count: int) -> None: """ Send a reminder email to a user if she's missed some PMs by being offline. The email will have its reply to address set to a limited used email address that will send a Zulip message to the correct recipient. This allows the user to respond to missed PMs, huddles, and @-mentions directly from the email. `user_profile` is the user to send the reminder to `missed_messages` is a list of dictionaries to Message objects and other data for a group of messages that share a recipient (and topic) """ from zerver.context_processors import common_context # Disabled missedmessage emails internally if not user_profile.enable_offline_email_notifications: return recipients = {(msg["message"].recipient_id, msg["message"].topic_name()) for msg in missed_messages} if len(recipients) != 1: raise ValueError( f"All missed_messages must have the same recipient and topic {recipients!r}", ) # This link is no longer a part of the email, but keeping the code in case # we find a clean way to add it back in the future unsubscribe_link = one_click_unsubscribe_link(user_profile, "missed_messages") context = common_context(user_profile) context.update( name=user_profile.full_name, message_count=message_count, unsubscribe_link=unsubscribe_link, realm_name_in_notifications=user_profile.realm_name_in_notifications, ) triggers = [message["trigger"] for message in missed_messages] unique_triggers = set(triggers) context.update( mention="mentioned" in unique_triggers or "wildcard_mentioned" in unique_triggers, stream_email_notify="stream_email_notify" in unique_triggers, mention_count=triggers.count("mentioned") + triggers.count("wildcard_mentioned"), ) # If this setting (email mirroring integration) is enabled, only then # can users reply to email to send message to Zulip. Thus, one must # ensure to display warning in the template. if settings.EMAIL_GATEWAY_PATTERN: context.update(reply_to_zulip=True, ) else: context.update(reply_to_zulip=False, ) from zerver.lib.email_mirror import create_missed_message_address reply_to_address = create_missed_message_address( user_profile, missed_messages[0]["message"]) if reply_to_address == FromAddress.NOREPLY: reply_to_name = "" else: reply_to_name = "Zulip" narrow_url = get_narrow_url(user_profile, missed_messages[0]["message"]) context.update(narrow_url=narrow_url, ) senders = list({m["message"].sender for m in missed_messages}) if missed_messages[0]["message"].recipient.type == Recipient.HUDDLE: display_recipient = get_display_recipient( missed_messages[0]["message"].recipient) # Make sure that this is a list of strings, not a string. assert not isinstance(display_recipient, str) other_recipients = [ r["full_name"] for r in display_recipient if r["id"] != user_profile.id ] context.update(group_pm=True) if len(other_recipients) == 2: huddle_display_name = " and ".join(other_recipients) context.update(huddle_display_name=huddle_display_name) elif len(other_recipients) == 3: huddle_display_name = ( f"{other_recipients[0]}, {other_recipients[1]}, and {other_recipients[2]}" ) context.update(huddle_display_name=huddle_display_name) else: huddle_display_name = "{}, and {} others".format( ", ".join(other_recipients[:2]), len(other_recipients) - 2) context.update(huddle_display_name=huddle_display_name) elif missed_messages[0]["message"].recipient.type == Recipient.PERSONAL: context.update(private_message=True) elif context["mention"] or context["stream_email_notify"]: # Keep only the senders who actually mentioned the user if context["mention"]: senders = list({ m["message"].sender for m in missed_messages if m["trigger"] == "mentioned" or m["trigger"] == "wildcard_mentioned" }) message = missed_messages[0]["message"] stream = Stream.objects.only("id", "name").get(id=message.recipient.type_id) stream_header = f"{stream.name} > {message.topic_name()}" context.update(stream_header=stream_header, ) else: raise AssertionError("Invalid messages!") # If message content is disabled, then flush all information we pass to email. if not message_content_allowed_in_missedmessage_emails(user_profile): realm = user_profile.realm context.update( reply_to_zulip=False, messages=[], sender_str="", realm_str=realm.name, huddle_display_name="", show_message_content=False, message_content_disabled_by_user=not user_profile. message_content_in_email_notifications, message_content_disabled_by_realm=not realm. message_content_allowed_in_email_notifications, ) else: context.update( messages=build_message_list( user=user_profile, messages=[m["message"] for m in missed_messages], stream_map={}, ), sender_str=", ".join(sender.full_name for sender in senders), realm_str=user_profile.realm.name, show_message_content=True, ) with override_language(user_profile.default_language): from_name: str = _("Zulip missed messages") from_address = FromAddress.NOREPLY if len(senders) == 1 and settings.SEND_MISSED_MESSAGE_EMAILS_AS_USER: # If this setting is enabled, you can reply to the Zulip # missed message emails directly back to the original sender. # However, one must ensure the Zulip server is in the SPF # record for the domain, or there will be spam/deliverability # problems. # # Also, this setting is not really compatible with # EMAIL_ADDRESS_VISIBILITY_ADMINS. sender = senders[0] from_name, from_address = (sender.full_name, sender.email) context.update(reply_to_zulip=False, ) email_dict = { "template_prefix": "zerver/emails/missed_message", "to_user_ids": [user_profile.id], "from_name": from_name, "from_address": from_address, "reply_to_email": str(Address(display_name=reply_to_name, addr_spec=reply_to_address)), "context": context, } queue_json_publish("email_senders", email_dict) user_profile.last_reminder = timezone_now() user_profile.save(update_fields=["last_reminder"])
def validate(self, value: [datetime.date, datetime.datetime]): now = timezone_now() if now > value: raise ValueError("Date/Time is in the past.")
def handle(self, **options: Any) -> None: if options["percent_huddles"] + options["percent_personals"] > 100: self.stderr.write( "Error! More than 100% of messages allocated.\n") return # Get consistent data for backend tests. if options["test_suite"]: random.seed(0) # If max_topics is not set, we set it proportional to the # number of messages. if options["max_topics"] is None: options["max_topics"] = 1 + options["num_messages"] // 100 if options["delete"]: # Start by clearing all the data in our database clear_database() # Create our three default realms # Could in theory be done via zerver.lib.actions.do_create_realm, but # welcome-bot (needed for do_create_realm) hasn't been created yet create_internal_realm() zulip_realm = Realm.objects.create( string_id="zulip", name="Zulip Dev", emails_restricted_to_domains=False, email_address_visibility=Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS, description= "The Zulip development environment default organization." " It's great for testing!", invite_required=False, org_type=Realm.CORPORATE) RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com") if options["test_suite"]: mit_realm = Realm.objects.create( string_id="zephyr", name="MIT", emails_restricted_to_domains=True, invite_required=False, org_type=Realm.CORPORATE) RealmDomain.objects.create(realm=mit_realm, domain="mit.edu") lear_realm = Realm.objects.create( string_id="lear", name="Lear & Co.", emails_restricted_to_domains=False, invite_required=False, org_type=Realm.CORPORATE) # Create test Users (UserProfiles are automatically created, # as are subscriptions to the ability to receive personals). names = [ ("Zoe", "*****@*****.**"), ("Othello, the Moor of Venice", "*****@*****.**"), ("Iago", "*****@*****.**"), ("Prospero from The Tempest", "*****@*****.**"), ("Cordelia Lear", "*****@*****.**"), ("King Hamlet", "*****@*****.**"), ("aaron", "*****@*****.**"), ("Polonius", "*****@*****.**"), ("Desdemona", "*****@*****.**"), ] # For testing really large batches: # Create extra users with semi realistic names to make search # functions somewhat realistic. We'll still create 1000 users # like Extra222 User for some predicability. num_names = options['extra_users'] num_boring_names = 300 for i in range(min(num_names, num_boring_names)): full_name = 'Extra%03d User' % (i, ) names.append((full_name, '*****@*****.**' % (i, ))) if num_names > num_boring_names: fnames = [ 'Amber', 'Arpita', 'Bob', 'Cindy', 'Daniela', 'Dan', 'Dinesh', 'Faye', 'François', 'George', 'Hank', 'Irene', 'James', 'Janice', 'Jenny', 'Jill', 'John', 'Kate', 'Katelyn', 'Kobe', 'Lexi', 'Manish', 'Mark', 'Matt', 'Mayna', 'Michael', 'Pete', 'Peter', 'Phil', 'Phillipa', 'Preston', 'Sally', 'Scott', 'Sandra', 'Steve', 'Stephanie', 'Vera' ] mnames = ['de', 'van', 'von', 'Shaw', 'T.'] lnames = [ 'Adams', 'Agarwal', 'Beal', 'Benson', 'Bonita', 'Davis', 'George', 'Harden', 'James', 'Jones', 'Johnson', 'Jordan', 'Lee', 'Leonard', 'Singh', 'Smith', 'Patel', 'Towns', 'Wall' ] for i in range(num_boring_names, num_names): fname = random.choice(fnames) + str(i) full_name = fname if random.random() < 0.7: if random.random() < 0.5: full_name += ' ' + random.choice(mnames) full_name += ' ' + random.choice(lnames) email = fname.lower() + '@zulip.com' names.append((full_name, email)) create_users(zulip_realm, names, tos_version=settings.TOS_VERSION) iago = get_user_by_delivery_email("*****@*****.**", zulip_realm) do_change_user_role(iago, UserProfile.ROLE_REALM_ADMINISTRATOR) iago.is_staff = True iago.save(update_fields=['is_staff']) desdemona = get_user_by_delivery_email("*****@*****.**", zulip_realm) do_change_user_role(desdemona, UserProfile.ROLE_REALM_OWNER) guest_user = get_user_by_delivery_email("*****@*****.**", zulip_realm) guest_user.role = UserProfile.ROLE_GUEST guest_user.save(update_fields=['role']) # These bots are directly referenced from code and thus # are needed for the test suite. zulip_realm_bots = [ ("Zulip Error Bot", "*****@*****.**"), ("Zulip Default Bot", "*****@*****.**"), ] for i in range(options["extra_bots"]): zulip_realm_bots.append( ('Extra Bot %d' % (i, ), '*****@*****.**' % (i, ))) create_users(zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT) zoe = get_user_by_delivery_email("*****@*****.**", zulip_realm) zulip_webhook_bots = [ ("Zulip Webhook Bot", "*****@*****.**"), ] # If a stream is not supplied in the webhook URL, the webhook # will (in some cases) send the notification as a PM to the # owner of the webhook bot, so bot_owner can't be None create_users(zulip_realm, zulip_webhook_bots, bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe) aaron = get_user_by_delivery_email("*****@*****.**", zulip_realm) zulip_outgoing_bots = [ ("Outgoing Webhook", "*****@*****.**"), ] create_users(zulip_realm, zulip_outgoing_bots, bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, bot_owner=aaron) outgoing_webhook = get_user("*****@*****.**", zulip_realm) add_service("outgoing-webhook", user_profile=outgoing_webhook, interface=Service.GENERIC, base_url="http://127.0.0.1:5002", token=generate_api_key()) # Add the realm internal bots to each realm. create_if_missing_realm_internal_bots() # Create public streams. stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"] stream_dict: Dict[str, Dict[str, Any]] = { "Verona": { "description": "A city in Italy" }, "Denmark": { "description": "A Scandinavian country" }, "Scotland": { "description": "Located in the United Kingdom" }, "Venice": { "description": "A northeastern Italian city" }, "Rome": { "description": "Yet another Italian city", "is_web_public": True }, } bulk_create_streams(zulip_realm, stream_dict) recipient_streams: List[int] = [ Stream.objects.get(name=name, realm=zulip_realm).id for name in stream_list ] # Create subscriptions to streams. The following # algorithm will give each of the users a different but # deterministic subset of the streams (given a fixed list # of users). For the test suite, we have a fixed list of # subscriptions to make sure test data is consistent # across platforms. subscriptions_list: List[Tuple[UserProfile, Recipient]] = [] profiles: Sequence[ UserProfile] = UserProfile.objects.select_related().filter( is_bot=False).order_by("email") if options["test_suite"]: subscriptions_map = { '*****@*****.**': ['Verona'], '*****@*****.**': ['Verona'], '*****@*****.**': ['Verona', 'Denmark'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice', 'Rome'], '*****@*****.**': ['Verona'], '*****@*****.**': ['Verona', 'Denmark', 'Venice'], } for profile in profiles: email = profile.delivery_email if email not in subscriptions_map: raise Exception( f'Subscriptions not listed for user {email}') for stream_name in subscriptions_map[email]: stream = Stream.objects.get(name=stream_name) r = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id) subscriptions_list.append((profile, r)) else: num_streams = len(recipient_streams) num_users = len(profiles) for i, profile in enumerate(profiles): # Subscribe to some streams. fraction = float(i) / num_users num_recips = int(num_streams * fraction) + 1 for type_id in recipient_streams[:num_recips]: r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id) subscriptions_list.append((profile, r)) subscriptions_to_add: List[Subscription] = [] event_time = timezone_now() all_subscription_logs: (List[RealmAuditLog]) = [] i = 0 for profile, recipient in subscriptions_list: i += 1 color = STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)] s = Subscription(recipient=recipient, user_profile=profile, color=color) subscriptions_to_add.append(s) log = RealmAuditLog( realm=profile.realm, modified_user=profile, modified_stream_id=recipient.type_id, event_last_message_id=0, event_type=RealmAuditLog.SUBSCRIPTION_CREATED, event_time=event_time) all_subscription_logs.append(log) Subscription.objects.bulk_create(subscriptions_to_add) RealmAuditLog.objects.bulk_create(all_subscription_logs) # Create custom profile field data phone_number = try_add_realm_custom_profile_field( zulip_realm, "Phone number", CustomProfileField.SHORT_TEXT, hint='') biography = try_add_realm_custom_profile_field( zulip_realm, "Biography", CustomProfileField.LONG_TEXT, hint='What are you known for?') favorite_food = try_add_realm_custom_profile_field( zulip_realm, "Favorite food", CustomProfileField.SHORT_TEXT, hint="Or drink, if you'd prefer") field_data: ProfileFieldData = { 'vim': { 'text': 'Vim', 'order': '1' }, 'emacs': { 'text': 'Emacs', 'order': '2' }, } favorite_editor = try_add_realm_custom_profile_field( zulip_realm, "Favorite editor", CustomProfileField.CHOICE, field_data=field_data) birthday = try_add_realm_custom_profile_field( zulip_realm, "Birthday", CustomProfileField.DATE) favorite_website = try_add_realm_custom_profile_field( zulip_realm, "Favorite website", CustomProfileField.URL, hint="Or your personal blog's URL") mentor = try_add_realm_custom_profile_field( zulip_realm, "Mentor", CustomProfileField.USER) github_profile = try_add_realm_default_custom_profile_field( zulip_realm, "github") # Fill in values for Iago and Hamlet hamlet = get_user_by_delivery_email("*****@*****.**", zulip_realm) do_update_user_custom_profile_data_if_changed( iago, [ { "id": phone_number.id, "value": "+1-234-567-8901" }, { "id": biography.id, "value": "Betrayer of Othello." }, { "id": favorite_food.id, "value": "Apples" }, { "id": favorite_editor.id, "value": "emacs" }, { "id": birthday.id, "value": "2000-1-1" }, { "id": favorite_website.id, "value": "https://zulip.readthedocs.io/en/latest/" }, { "id": mentor.id, "value": [hamlet.id] }, { "id": github_profile.id, "value": 'zulip' }, ]) do_update_user_custom_profile_data_if_changed( hamlet, [ { "id": phone_number.id, "value": "+0-11-23-456-7890" }, { "id": biography.id, "value": "I am:\n* The prince of Denmark\n* Nephew to the usurping Claudius", }, { "id": favorite_food.id, "value": "Dark chocolate" }, { "id": favorite_editor.id, "value": "vim" }, { "id": birthday.id, "value": "1900-1-1" }, { "id": favorite_website.id, "value": "https://blog.zulig.org" }, { "id": mentor.id, "value": [iago.id] }, { "id": github_profile.id, "value": 'zulipbot' }, ]) else: zulip_realm = get_realm("zulip") recipient_streams = [ klass.type_id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] # Extract a list of all users user_profiles: List[UserProfile] = list( UserProfile.objects.filter(is_bot=False)) # Create a test realm emoji. IMAGE_FILE_PATH = static_path('images/test-images/checkbox.png') with open(IMAGE_FILE_PATH, 'rb') as fp: check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp) if not options["test_suite"]: # Populate users with some bar data for user in user_profiles: status: int = UserPresence.ACTIVE date = timezone_now() client = get_client("website") if user.full_name[0] <= 'H': client = get_client("ZulipAndroid") UserPresence.objects.get_or_create(user_profile=user, realm_id=user.realm_id, client=client, timestamp=date, status=status) user_profiles_ids = [user_profile.id for user_profile in user_profiles] # Create several initial huddles for i in range(options["num_huddles"]): get_huddle(random.sample(user_profiles_ids, random.randint(3, 4))) # Create several initial pairs for personals personals_pairs = [ random.sample(user_profiles_ids, 2) for i in range(options["num_personals"]) ] # Generate a new set of test data. create_test_data() # prepopulate the URL preview/embed data for the links present # in the config.generate_data.json data set. This makes it # possible for populate_db to run happily without Internet # access. with open("zerver/tests/fixtures/docs_url_preview_data.json") as f: urls_with_preview_data = ujson.load(f) for url in urls_with_preview_data: cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME) if options["delete"]: if options["test_suite"]: # Create test users; the MIT ones are needed to test # the Zephyr mirroring codepaths. testsuite_mit_users = [ ("Fred Sipb (MIT)", "*****@*****.**"), ("Athena Consulting Exchange User (MIT)", "*****@*****.**"), ("Esp Classroom (MIT)", "*****@*****.**"), ] create_users(mit_realm, testsuite_mit_users, tos_version=settings.TOS_VERSION) testsuite_lear_users = [ ("King Lear", "*****@*****.**"), ("Cordelia Lear", "*****@*****.**"), ] create_users(lear_realm, testsuite_lear_users, tos_version=settings.TOS_VERSION) if not options["test_suite"]: # To keep the messages.json fixtures file for the test # suite fast, don't add these users and subscriptions # when running populate_db for the test suite zulip_stream_dict: Dict[str, Dict[str, Any]] = { "devel": { "description": "For developing" }, "all": { "description": "For **everything**" }, "announce": { "description": "For announcements", 'stream_post_policy': Stream.STREAM_POST_POLICY_ADMINS }, "design": { "description": "For design" }, "support": { "description": "For support" }, "social": { "description": "For socializing" }, "test": { "description": "For testing `code`" }, "errors": { "description": "For errors" }, "sales": { "description": "For sales discussion" }, } # Calculate the maximum number of digits in any extra stream's # number, since a stream with name "Extra Stream 3" could show # up after "Extra Stream 29". (Used later to pad numbers with # 0s). maximum_digits = len(str(options['extra_streams'] - 1)) for i in range(options['extra_streams']): # Pad the number with 0s based on `maximum_digits`. number_str = str(i).zfill(maximum_digits) extra_stream_name = 'Extra Stream ' + number_str zulip_stream_dict[extra_stream_name] = { "description": "Auto-generated extra stream.", } bulk_create_streams(zulip_realm, zulip_stream_dict) # Now that we've created the notifications stream, configure it properly. zulip_realm.notifications_stream = get_stream( "announce", zulip_realm) zulip_realm.save(update_fields=['notifications_stream']) # Add a few default streams for default_stream_name in [ "design", "devel", "social", "support" ]: DefaultStream.objects.create(realm=zulip_realm, stream=get_stream( default_stream_name, zulip_realm)) # Now subscribe everyone to these streams subscribe_users_to_streams(zulip_realm, zulip_stream_dict) if not options["test_suite"]: # Update pointer of each user to point to the last message in their # UserMessage rows with sender_id=user_profile_id. users = list( UserMessage.objects.filter(message__sender_id=F( 'user_profile_id')).values('user_profile_id').annotate( pointer=Max('message_id'))) for user in users: UserProfile.objects.filter( id=user['user_profile_id']).update( pointer=user['pointer']) create_user_groups() if not options["test_suite"]: # We populate the analytics database here for # development purpose only call_command('populate_analytics_db') threads = options["threads"] jobs: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]] = [] for i in range(threads): count = options["num_messages"] // threads if i < options["num_messages"] % threads: count += 1 jobs.append((count, personals_pairs, options, self.stdout.write, random.randint(0, 10**10))) for job in jobs: generate_and_send_messages(job) if options["delete"]: if not options['test_suite']: # These bots are not needed by the test suite # Also, we don't want interacting with each other # in dev setup. internal_zulip_users_nosubs = [ ("Zulip Commit Bot", "*****@*****.**"), ("Zulip Trac Bot", "*****@*****.**"), ("Zulip Nagios Bot", "*****@*****.**"), ] create_users(zulip_realm, internal_zulip_users_nosubs, bot_type=UserProfile.DEFAULT_BOT) # Mark all messages as read UserMessage.objects.all().update(flags=UserMessage.flags.read) self.stdout.write("Successfully populated test database.\n")
def test_users_to_zerver_userprofile( self, mock_get_data_file: mock.Mock) -> None: custom_profile_field_user1 = { "Xf06054BBB": { "value": "random1" }, "Xf023DSCdd": { "value": "employee" } } custom_profile_field_user2 = { "Xf06054BBB": { "value": "random2" }, "Xf023DSCdd": { "value": "employer" } } user_data = [{ "id": "U08RGD1RD", "team_id": "T5YFFM2QY", "name": "john", "deleted": False, "is_mirror_dummy": False, "real_name": "John Doe", "profile": { "image_32": "", "email": "*****@*****.**", "avatar_hash": "hash", "phone": "+1-123-456-77-868", "fields": custom_profile_field_user1 } }, { "id": "U0CBK5KAT", "team_id": "T5YFFM2QY", "is_admin": True, "is_bot": False, "is_owner": True, "is_primary_owner": True, 'name': 'Jane', "real_name": "Jane Doe", "deleted": False, "is_mirror_dummy": False, "profile": { "image_32": "https://secure.gravatar.com/avatar/random.png", "fields": custom_profile_field_user2, "email": "*****@*****.**", "avatar_hash": "hash" } }, { "id": "U09TYF5Sk", "team_id": "T5YFFM2QY", "name": "Bot", "real_name": "Bot", "is_bot": True, "deleted": False, "is_mirror_dummy": False, "profile": { "image_32": "https://secure.gravatar.com/avatar/random1.png", "skype": "test_skype_name", "email": "*****@*****.**", "avatar_hash": "hash" } }, { "id": "UHSG7OPQN", "team_id": "T6LARQE2Z", 'name': 'matt.perry', "color": '9d8eee', "is_bot": False, "is_app_user": False, "is_mirror_dummy": True, "team_domain": "foreignteam", "profile": { "image_32": "https://secure.gravatar.com/avatar/random6.png", "avatar_hash": "hash", "first_name": "Matt", "last_name": "Perry", "real_name": "Matt Perry", "display_name": "matt.perry", "team": "T6LARQE2Z" } }] mock_get_data_file.return_value = user_data # As user with slack_id 'U0CBK5KAT' is the primary owner, that user should be imported first # and hence has zulip_id = 1 test_slack_user_id_to_zulip_user_id = { 'U08RGD1RD': 1, 'U0CBK5KAT': 0, 'U09TYF5Sk': 2, 'UHSG7OPQN': 3 } slack_data_dir = './random_path' timestamp = int(timezone_now().timestamp()) mock_get_data_file.return_value = user_data zerver_userprofile, avatar_list, slack_user_id_to_zulip_user_id, customprofilefield, \ customprofilefield_value = users_to_zerver_userprofile(slack_data_dir, user_data, 1, timestamp, 'test_domain') # Test custom profile fields self.assertEqual(customprofilefield[0]['field_type'], 1) self.assertEqual(customprofilefield[3]['name'], 'skype') cpf_name = {cpf['name'] for cpf in customprofilefield} self.assertIn('phone', cpf_name) self.assertIn('skype', cpf_name) cpf_name.remove('phone') cpf_name.remove('skype') for name in cpf_name: self.assertTrue(name.startswith('slack custom field ')) self.assertEqual(len(customprofilefield_value), 6) self.assertEqual(customprofilefield_value[0]['field'], 0) self.assertEqual(customprofilefield_value[0]['user_profile'], 1) self.assertEqual(customprofilefield_value[3]['user_profile'], 0) self.assertEqual(customprofilefield_value[5]['value'], 'test_skype_name') # test that the primary owner should always be imported first self.assertDictEqual(slack_user_id_to_zulip_user_id, test_slack_user_id_to_zulip_user_id) self.assertEqual(len(avatar_list), 4) self.assertEqual(len(zerver_userprofile), 4) self.assertEqual(zerver_userprofile[0]['is_staff'], False) self.assertEqual(zerver_userprofile[0]['is_bot'], False) self.assertEqual(zerver_userprofile[0]['is_active'], True) self.assertEqual(zerver_userprofile[0]['is_mirror_dummy'], False) self.assertEqual(zerver_userprofile[0]['role'], UserProfile.ROLE_MEMBER) self.assertEqual(zerver_userprofile[0]['enable_desktop_notifications'], True) self.assertEqual(zerver_userprofile[0]['email'], '*****@*****.**') self.assertEqual(zerver_userprofile[0]['full_name'], 'John Doe') self.assertEqual(zerver_userprofile[1]['id'], test_slack_user_id_to_zulip_user_id['U0CBK5KAT']) self.assertEqual(zerver_userprofile[1]['role'], UserProfile.ROLE_REALM_ADMINISTRATOR) self.assertEqual(zerver_userprofile[1]['is_staff'], False) self.assertEqual(zerver_userprofile[1]['is_active'], True) self.assertEqual(zerver_userprofile[0]['is_mirror_dummy'], False) self.assertEqual(zerver_userprofile[2]['id'], test_slack_user_id_to_zulip_user_id['U09TYF5Sk']) self.assertEqual(zerver_userprofile[2]['is_bot'], True) self.assertEqual(zerver_userprofile[2]['is_active'], True) self.assertEqual(zerver_userprofile[2]['is_mirror_dummy'], False) self.assertEqual(zerver_userprofile[2]['email'], '*****@*****.**') self.assertEqual(zerver_userprofile[2]['bot_type'], 1) self.assertEqual(zerver_userprofile[2]['avatar_source'], 'U') self.assertEqual(zerver_userprofile[3]['id'], test_slack_user_id_to_zulip_user_id['UHSG7OPQN']) self.assertEqual(zerver_userprofile[3]['role'], UserProfile.ROLE_MEMBER) self.assertEqual(zerver_userprofile[3]['is_staff'], False) self.assertEqual(zerver_userprofile[3]['is_active'], False) self.assertEqual(zerver_userprofile[3]['email'], '*****@*****.**') self.assertEqual(zerver_userprofile[3]['realm'], 1) self.assertEqual(zerver_userprofile[3]['full_name'], 'Matt Perry') self.assertEqual(zerver_userprofile[3]['short_name'], 'matt.perry') self.assertEqual(zerver_userprofile[3]['is_mirror_dummy'], True) self.assertEqual(zerver_userprofile[3]['is_api_super_user'], False)
def upload_to(instance, filename): now = timezone_now() base, extension = os.path.splitext(filename) extension = extension.lower() return f"ideas/{now:%Y/%m}/{instance.pk}{extension}"
def realm_summary_table(realm_minutes: Dict[str, float]) -> str: now = timezone_now() query = SQL(""" SELECT realm.string_id, realm.date_created, realm.plan_type, realm.org_type, coalesce(wau_table.value, 0) wau_count, coalesce(dau_table.value, 0) dau_count, coalesce(user_count_table.value, 0) user_profile_count, coalesce(bot_count_table.value, 0) bot_count FROM zerver_realm as realm LEFT OUTER JOIN ( SELECT value _14day_active_humans, realm_id from analytics_realmcount WHERE property = 'realm_active_humans::day' AND end_time = %(realm_active_humans_end_time)s ) as _14day_active_humans_table ON realm.id = _14day_active_humans_table.realm_id LEFT OUTER JOIN ( SELECT value, realm_id from analytics_realmcount WHERE property = '7day_actives::day' AND end_time = %(seven_day_actives_end_time)s ) as wau_table ON realm.id = wau_table.realm_id LEFT OUTER JOIN ( SELECT value, realm_id from analytics_realmcount WHERE property = '1day_actives::day' AND end_time = %(one_day_actives_end_time)s ) as dau_table ON realm.id = dau_table.realm_id LEFT OUTER JOIN ( SELECT value, realm_id from analytics_realmcount WHERE property = 'active_users_audit:is_bot:day' AND subgroup = 'false' AND end_time = %(active_users_audit_end_time)s ) as user_count_table ON realm.id = user_count_table.realm_id LEFT OUTER JOIN ( SELECT value, realm_id from analytics_realmcount WHERE property = 'active_users_audit:is_bot:day' AND subgroup = 'true' AND end_time = %(active_users_audit_end_time)s ) as bot_count_table ON realm.id = bot_count_table.realm_id WHERE _14day_active_humans IS NOT NULL or realm.plan_type = 3 ORDER BY dau_count DESC, string_id ASC """) cursor = connection.cursor() cursor.execute( query, { "realm_active_humans_end_time": COUNT_STATS["realm_active_humans::day"].last_successful_fill(), "seven_day_actives_end_time": COUNT_STATS["7day_actives::day"].last_successful_fill(), "one_day_actives_end_time": COUNT_STATS["1day_actives::day"].last_successful_fill(), "active_users_audit_end_time": COUNT_STATS["active_users_audit:is_bot:day"].last_successful_fill( ), }, ) rows = dictfetchall(cursor) cursor.close() # Fetch all the realm administrator users realm_owners: Dict[str, List[str]] = defaultdict(list) for up in UserProfile.objects.select_related("realm").filter( role=UserProfile.ROLE_REALM_OWNER, is_active=True, ): realm_owners[up.realm.string_id].append(up.delivery_email) for row in rows: row["date_created_day"] = row["date_created"].strftime("%Y-%m-%d") row["age_days"] = int( (now - row["date_created"]).total_seconds() / 86400) row["is_new"] = row["age_days"] < 12 * 7 row["realm_owner_emails"] = ", ".join(realm_owners[row["string_id"]]) # get messages sent per day counts = get_realm_day_counts() for row in rows: try: row["history"] = counts[row["string_id"]]["cnts"] except Exception: row["history"] = "" # estimate annual subscription revenue total_arr = 0 if settings.BILLING_ENABLED: estimated_arrs = estimate_annual_recurring_revenue_by_realm() realms_to_default_discount = get_realms_to_default_discount_dict() for row in rows: row["plan_type_string"] = get_plan_name(row["plan_type"]) string_id = row["string_id"] if string_id in estimated_arrs: row["arr"] = estimated_arrs[string_id] if row["plan_type"] == Realm.STANDARD: row["effective_rate"] = 100 - int( realms_to_default_discount.get(string_id, 0)) elif row["plan_type"] == Realm.STANDARD_FREE: row["effective_rate"] = 0 elif row[ "plan_type"] == Realm.LIMITED and string_id in realms_to_default_discount: row["effective_rate"] = 100 - int( realms_to_default_discount[string_id]) else: row["effective_rate"] = "" total_arr += sum(estimated_arrs.values()) for row in rows: row["org_type_string"] = get_org_type_display_name(row["org_type"]) # augment data with realm_minutes total_hours = 0.0 for row in rows: string_id = row["string_id"] minutes = realm_minutes.get(string_id, 0.0) hours = minutes / 60.0 total_hours += hours row["hours"] = str(int(hours)) try: row["hours_per_user"] = "******".format(hours / row["dau_count"]) except Exception: pass # formatting for row in rows: row["stats_link"] = realm_stats_link(row["string_id"]) row["string_id"] = realm_activity_link(row["string_id"]) # Count active sites def meets_goal(row: Dict[str, int]) -> bool: return row["dau_count"] >= 5 num_active_sites = len(list(filter(meets_goal, rows))) # create totals total_dau_count = 0 total_user_profile_count = 0 total_bot_count = 0 total_wau_count = 0 for row in rows: total_dau_count += int(row["dau_count"]) total_user_profile_count += int(row["user_profile_count"]) total_bot_count += int(row["bot_count"]) total_wau_count += int(row["wau_count"]) total_row = dict( string_id="Total", plan_type_string="", org_type_string="", effective_rate="", arr=total_arr, stats_link="", date_created_day="", realm_owner_emails="", dau_count=total_dau_count, user_profile_count=total_user_profile_count, bot_count=total_bot_count, hours=int(total_hours), wau_count=total_wau_count, ) rows.insert(0, total_row) content = loader.render_to_string( "analytics/realm_summary_table.html", dict( rows=rows, num_active_sites=num_active_sites, utctime=now.strftime("%Y-%m-%d %H:%MZ"), billing_enabled=settings.BILLING_ENABLED, ), ) return content
def test_do_auto_soft_deactivate_users(self) -> None: users = [ self.example_user('iago'), self.example_user('cordelia'), self.example_user('ZOE'), self.example_user('othello'), self.example_user('prospero'), self.example_user('aaron'), self.example_user('polonius'), ] sender = self.example_user('hamlet') realm = get_realm('zulip') stream_name = 'announce' for user in users + [sender]: self.subscribe(user, stream_name) client, _ = Client.objects.get_or_create(name='website') query = '/json/users/me/pointer' last_visit = timezone_now() count = 150 for user_profile in users: UserActivity.objects.get_or_create( user_profile=user_profile, client=client, query=query, count=count, last_visit=last_visit ) with mock.patch('logging.info'): users_deactivated = do_auto_soft_deactivate_users(-1, realm) self.assert_length(users_deactivated, len(users)) for user in users: self.assertTrue(user in users_deactivated) # Verify that deactivated users are caught up automatically message_id = self.send_stream_message(sender.email, stream_name) received_count = UserMessage.objects.filter(user_profile__in=users, message_id=message_id).count() self.assertEqual(0, received_count) with mock.patch('logging.info'): users_deactivated = do_auto_soft_deactivate_users(-1, realm) self.assert_length(users_deactivated, 0) # all users are already deactivated received_count = UserMessage.objects.filter(user_profile__in=users, message_id=message_id).count() self.assertEqual(len(users), received_count) # Verify that deactivated users are NOT caught up if # AUTO_CATCH_UP_SOFT_DEACTIVATED_USERS is off message_id = self.send_stream_message(sender.email, stream_name) received_count = UserMessage.objects.filter(user_profile__in=users, message_id=message_id).count() self.assertEqual(0, received_count) with self.settings(AUTO_CATCH_UP_SOFT_DEACTIVATED_USERS=False): with mock.patch('logging.info'): users_deactivated = do_auto_soft_deactivate_users(-1, realm) self.assert_length(users_deactivated, 0) # all users are already deactivated received_count = UserMessage.objects.filter(user_profile__in=users, message_id=message_id).count() self.assertEqual(0, received_count)
def test_edit_message(self) -> None: """ Verify if the time limit imposed on message editing is working correctly. """ iago = self.example_user("iago") self.login("iago") # Set limit to edit message content. MESSAGE_CONTENT_EDIT_LIMIT = 5 * 60 # 5 minutes result = self.client_patch( "/json/realm", { "allow_message_editing": "true", "message_content_edit_limit_seconds": MESSAGE_CONTENT_EDIT_LIMIT, }, ) self.assert_json_success(result) sent_message_id = self.send_stream_message( iago, "Scotland", topic_name="lunch", content="I want pizza!", ) message_sent_time = timezone_now() # Verify message sent. message = most_recent_message(iago) self.assertEqual(message.id, sent_message_id) self.assertEqual(message.content, "I want pizza!") # Edit message content now. This should work as we're editing # it immediately after sending i.e., before the limit exceeds. result = self.client_patch( f"/json/messages/{sent_message_id}", {"content": "I want burger!"} ) self.assert_json_success(result) message = most_recent_message(iago) self.assertEqual(message.id, sent_message_id) self.assertEqual(message.content, "I want burger!") # Now that we tested message editing works within the limit, # we want to verify it doesn't work beyond the limit. # # To do that we'll have to wait for the time limit to pass which is # 5 minutes here. Easy, use time.sleep() but mind that it slows down the # test to a great extent which isn't good. This is when mocking comes to rescue. # We can check what the original code does to determine whether the time limit # exceeded and mock that here such that the code runs as if the time limit # exceeded without actually waiting for that long! # # In this case, it is timezone_now, an alias to django.utils.timezone.now, # to which the difference with message-sent-time is checked. So, we want # that timezone_now() call to return `datetime` object representing time # that is beyond the limit. # # Notice how mock.patch() is used here to do exactly the above mentioned. # mock.patch() here makes any calls to `timezone_now` in `zerver.lib.actions` # to return the value passed to `return_value` in the its context. # You can also use mock.patch() as a decorator depending on the # requirements. Read more at the documentation link provided above. time_beyond_edit_limit = message_sent_time + datetime.timedelta( seconds=MESSAGE_CONTENT_EDIT_LIMIT + 100 ) # There's a buffer time applied to the limit, hence the extra 100s. with mock.patch( "zerver.lib.actions.timezone_now", return_value=time_beyond_edit_limit, ): result = self.client_patch( f"/json/messages/{sent_message_id}", {"content": "I actually want pizza."} ) self.assert_json_error(result, msg="The time limit for editing this message has passed") message = most_recent_message(iago) self.assertEqual(message.id, sent_message_id) self.assertEqual(message.content, "I want burger!")
def upload_to(instance, filename): now = timezone_now() filename_base, filename_ext = os.path.splitext(filename) return 'uploads/{}_{}{}'.format(now.strftime("%Y/%m/%d/%Y%m%d%H%M%S"), create_random_string(), filename_ext.lower())
def test_do_auto_soft_deactivate_users(self) -> None: users = [ self.example_user('iago'), self.example_user('cordelia'), self.example_user('ZOE'), self.example_user('othello'), self.example_user('prospero'), self.example_user('aaron'), self.example_user('polonius'), self.example_user('desdemona'), ] sender = self.example_user('hamlet') realm = get_realm('zulip') stream_name = 'announce' for user in [*users, sender]: self.subscribe(user, stream_name) client, _ = Client.objects.get_or_create(name='website') query = '/some/random/endpoint' last_visit = timezone_now() count = 150 for user_profile in users: UserActivity.objects.get_or_create( user_profile=user_profile, client=client, query=query, count=count, last_visit=last_visit, ) with self.assertLogs(logger_string, level="INFO") as m: users_deactivated = do_auto_soft_deactivate_users(-1, realm) log_output = [] for user in users: log_output.append( f"INFO:{logger_string}:Soft Deactivated user {user.id}") log_output.append( f"INFO:{logger_string}:Soft-deactivated batch of {len(users[:100])} users; {len(users[100:])} remain to process" ) log_output.append( f"INFO:{logger_string}:Caught up {len(users)} soft-deactivated users" ) self.assertEqual(set(m.output), set(log_output)) self.assert_length(users_deactivated, len(users)) for user in users: self.assertTrue(user in users_deactivated) # Verify that deactivated users are caught up automatically message_id = self.send_stream_message(sender, stream_name) received_count = UserMessage.objects.filter( user_profile__in=users, message_id=message_id).count() self.assertEqual(0, received_count) with self.assertLogs(logger_string, level="INFO") as m: users_deactivated = do_auto_soft_deactivate_users(-1, realm) log_output = [] log_output.append( f"INFO:{logger_string}:Caught up {len(users)} soft-deactivated users" ) self.assertEqual(set(m.output), set(log_output)) self.assert_length(users_deactivated, 0) # all users are already deactivated received_count = UserMessage.objects.filter( user_profile__in=users, message_id=message_id).count() self.assertEqual(len(users), received_count) # Verify that deactivated users are NOT caught up if # AUTO_CATCH_UP_SOFT_DEACTIVATED_USERS is off message_id = self.send_stream_message(sender, stream_name) received_count = UserMessage.objects.filter( user_profile__in=users, message_id=message_id).count() self.assertEqual(0, received_count) with self.settings(AUTO_CATCH_UP_SOFT_DEACTIVATED_USERS=False): with self.assertLogs(logger_string, level="INFO") as m: users_deactivated = do_auto_soft_deactivate_users(-1, realm) self.assertEqual(m.output, [ f"INFO:{logger_string}:Not catching up users since AUTO_CATCH_UP_SOFT_DEACTIVATED_USERS is off" ]) self.assert_length(users_deactivated, 0) # all users are already deactivated received_count = UserMessage.objects.filter( user_profile__in=users, message_id=message_id).count() self.assertEqual(0, received_count)
def test_convert_slack_workspace_messages( self, mock_get_messages_iterator: mock.Mock, mock_message: mock.Mock) -> None: output_dir = os.path.join(settings.TEST_WORKER_DIR, "test-slack-import") os.makedirs(output_dir, exist_ok=True) added_channels: Dict[str, Tuple[str, int]] = { "random": ("c5", 1), "general": ("c6", 2) } time = float(timezone_now().timestamp()) zerver_message = [{"id": 1, "ts": time}, {"id": 5, "ts": time}] def fake_get_messages_iter( slack_data_dir: str, added_channels: AddedChannelsT, added_mpims: AddedMPIMsT, dm_members: DMMembersT, ) -> Iterator[ZerverFieldsT]: import copy return iter(copy.deepcopy(zerver_message)) realm: Dict[str, Any] = {"zerver_subscription": []} user_list: List[Dict[str, Any]] = [] reactions = [{"name": "grinning", "users": ["U061A5N1G"], "count": 1}] attachments: List[Dict[str, Any]] = [] uploads: List[Dict[str, Any]] = [] zerver_usermessage = [{"id": 3}, {"id": 5}, {"id": 6}, {"id": 9}] mock_get_messages_iterator.side_effect = fake_get_messages_iter mock_message.side_effect = [ [ zerver_message[:1], zerver_usermessage[:2], attachments, uploads, reactions[:1] ], [ zerver_message[1:2], zerver_usermessage[2:5], attachments, uploads, reactions[1:1] ], ] with self.assertLogs(level="INFO"): # Hacky: We should include a zerver_userprofile, not the empty [] test_reactions, uploads, zerver_attachment = convert_slack_workspace_messages( "./random_path", user_list, 2, {}, {}, added_channels, {}, {}, realm, [], [], "domain", output_dir=output_dir, chunk_size=1, ) messages_file_1 = os.path.join(output_dir, "messages-000001.json") self.assertTrue(os.path.exists(messages_file_1)) messages_file_2 = os.path.join(output_dir, "messages-000002.json") self.assertTrue(os.path.exists(messages_file_2)) with open(messages_file_1, "rb") as f: message_json = orjson.loads(f.read()) self.assertEqual(message_json["zerver_message"], zerver_message[:1]) self.assertEqual(message_json["zerver_usermessage"], zerver_usermessage[:2]) with open(messages_file_2, "rb") as f: message_json = orjson.loads(f.read()) self.assertEqual(message_json["zerver_message"], zerver_message[1:2]) self.assertEqual(message_json["zerver_usermessage"], zerver_usermessage[2:5]) self.assertEqual(test_reactions, reactions)
def do_update_message( user_profile: UserProfile, target_message: Message, new_stream: Optional[Stream], topic_name: Optional[str], propagate_mode: str, send_notification_to_old_thread: bool, send_notification_to_new_thread: bool, content: Optional[str], rendering_result: Optional[MessageRenderingResult], prior_mention_user_ids: Set[int], mention_data: Optional[MentionData] = None, ) -> int: """ The main function for message editing. A message edit event can modify: * the message's content (in which case the caller will have set both content and rendered_content), * the topic, in which case the caller will have set topic_name * or both message's content and the topic * or stream and/or topic, in which case the caller will have set new_stream and/or topic_name. With topic edits, propagate_mode determines whether other message also have their topics edited. """ timestamp = timezone_now() target_message.last_edit_time = timestamp event: Dict[str, Any] = { "type": "update_message", "user_id": user_profile.id, "edit_timestamp": datetime_to_timestamp(timestamp), "message_id": target_message.id, "rendering_only": False, } edit_history_event: EditHistoryEvent = { "user_id": user_profile.id, "timestamp": event["edit_timestamp"], } changed_messages = [target_message] realm = user_profile.realm stream_being_edited = None if target_message.is_stream_message(): stream_id = target_message.recipient.type_id stream_being_edited = get_stream_by_id_in_realm(stream_id, realm) event["stream_name"] = stream_being_edited.name event["stream_id"] = stream_being_edited.id ums = UserMessage.objects.filter(message=target_message.id) if content is not None: assert rendering_result is not None # mention_data is required if there's a content edit. assert mention_data is not None # add data from group mentions to mentions_user_ids. for group_id in rendering_result.mentions_user_group_ids: members = mention_data.get_group_members(group_id) rendering_result.mentions_user_ids.update(members) update_user_message_flags(rendering_result, ums) # One could imagine checking realm.allow_edit_history here and # modifying the events based on that setting, but doing so # doesn't really make sense. We need to send the edit event # to clients regardless, and a client already had access to # the original/pre-edit content of the message anyway. That # setting must be enforced on the client side, and making a # change here simply complicates the logic for clients parsing # edit history events. event["orig_content"] = target_message.content event["orig_rendered_content"] = target_message.rendered_content edit_history_event["prev_content"] = target_message.content edit_history_event[ "prev_rendered_content"] = target_message.rendered_content edit_history_event[ "prev_rendered_content_version"] = target_message.rendered_content_version target_message.content = content target_message.rendered_content = rendering_result.rendered_content target_message.rendered_content_version = markdown_version event["content"] = content event["rendered_content"] = rendering_result.rendered_content event[ "prev_rendered_content_version"] = target_message.rendered_content_version event["is_me_message"] = Message.is_status_message( content, rendering_result.rendered_content) # target_message.has_image and target_message.has_link will have been # already updated by Markdown rendering in the caller. target_message.has_attachment = check_attachment_reference_change( target_message, rendering_result) if target_message.is_stream_message(): if topic_name is not None: new_topic_name = topic_name else: new_topic_name = target_message.topic_name() stream_topic: Optional[StreamTopicTarget] = StreamTopicTarget( stream_id=stream_id, topic_name=new_topic_name, ) else: stream_topic = None info = get_recipient_info( realm_id=realm.id, recipient=target_message.recipient, sender_id=target_message.sender_id, stream_topic=stream_topic, possible_wildcard_mention=mention_data.message_has_wildcards(), ) event["online_push_user_ids"] = list(info["online_push_user_ids"]) event["pm_mention_push_disabled_user_ids"] = list( info["pm_mention_push_disabled_user_ids"]) event["pm_mention_email_disabled_user_ids"] = list( info["pm_mention_email_disabled_user_ids"]) event["stream_push_user_ids"] = list(info["stream_push_user_ids"]) event["stream_email_user_ids"] = list(info["stream_email_user_ids"]) event["muted_sender_user_ids"] = list(info["muted_sender_user_ids"]) event["prior_mention_user_ids"] = list(prior_mention_user_ids) event["presence_idle_user_ids"] = filter_presence_idle_user_ids( info["active_user_ids"]) event["all_bot_user_ids"] = list(info["all_bot_user_ids"]) if rendering_result.mentions_wildcard: event["wildcard_mention_user_ids"] = list( info["wildcard_mention_user_ids"]) else: event["wildcard_mention_user_ids"] = [] do_update_mobile_push_notification( target_message, prior_mention_user_ids, rendering_result.mentions_user_ids, info["stream_push_user_ids"], ) if topic_name is not None or new_stream is not None: orig_topic_name = target_message.topic_name() event["propagate_mode"] = propagate_mode if new_stream is not None: assert content is None assert target_message.is_stream_message() assert stream_being_edited is not None edit_history_event["prev_stream"] = stream_being_edited.id edit_history_event["stream"] = new_stream.id event[ORIG_TOPIC] = orig_topic_name target_message.recipient_id = new_stream.recipient_id event["new_stream_id"] = new_stream.id event["propagate_mode"] = propagate_mode # When messages are moved from one stream to another, some # users may lose access to those messages, including guest # users and users not subscribed to the new stream (if it is a # private stream). For those users, their experience is as # though the messages were deleted, and we should send a # delete_message event to them instead. subs_to_old_stream = get_active_subscriptions_for_stream_id( stream_id, include_deactivated_users=True).select_related("user_profile") subs_to_new_stream = list( get_active_subscriptions_for_stream_id( new_stream.id, include_deactivated_users=True).select_related("user_profile")) old_stream_sub_ids = [ user.user_profile_id for user in subs_to_old_stream ] new_stream_sub_ids = [ user.user_profile_id for user in subs_to_new_stream ] # Get users who aren't subscribed to the new_stream. subs_losing_usermessages = [ sub for sub in subs_to_old_stream if sub.user_profile_id not in new_stream_sub_ids ] # Users who can longer access the message without some action # from administrators. subs_losing_access = [ sub for sub in subs_losing_usermessages if sub.user_profile.is_guest or not new_stream.is_public() ] ums = ums.exclude(user_profile_id__in=[ sub.user_profile_id for sub in subs_losing_usermessages ]) subs_gaining_usermessages = [] if not new_stream.is_history_public_to_subscribers(): # For private streams, with history not public to subscribers, # We find out users who are not present in the msgs' old stream # and create new UserMessage for these users so that they can # access this message. subs_gaining_usermessages += [ user_id for user_id in new_stream_sub_ids if user_id not in old_stream_sub_ids ] if topic_name is not None: topic_name = truncate_topic(topic_name) target_message.set_topic_name(topic_name) # These fields have legacy field names. event[ORIG_TOPIC] = orig_topic_name event[TOPIC_NAME] = topic_name event[TOPIC_LINKS] = topic_links(target_message.sender.realm_id, topic_name) edit_history_event["prev_topic"] = orig_topic_name edit_history_event["topic"] = topic_name update_edit_history(target_message, timestamp, edit_history_event) delete_event_notify_user_ids: List[int] = [] if propagate_mode in ["change_later", "change_all"]: assert topic_name is not None or new_stream is not None assert stream_being_edited is not None # Other messages should only get topic/stream fields in their edit history. topic_only_edit_history_event: EditHistoryEvent = { "user_id": edit_history_event["user_id"], "timestamp": edit_history_event["timestamp"], } if topic_name is not None: topic_only_edit_history_event["prev_topic"] = edit_history_event[ "prev_topic"] topic_only_edit_history_event["topic"] = edit_history_event[ "topic"] if new_stream is not None: topic_only_edit_history_event["prev_stream"] = edit_history_event[ "prev_stream"] topic_only_edit_history_event["stream"] = edit_history_event[ "stream"] messages_list = update_messages_for_topic_edit( acting_user=user_profile, edited_message=target_message, propagate_mode=propagate_mode, orig_topic_name=orig_topic_name, topic_name=topic_name, new_stream=new_stream, old_stream=stream_being_edited, edit_history_event=topic_only_edit_history_event, last_edit_time=timestamp, ) changed_messages += messages_list if new_stream is not None: assert stream_being_edited is not None changed_message_ids = [msg.id for msg in changed_messages] if subs_gaining_usermessages: ums_to_create = [] for message_id in changed_message_ids: for user_profile_id in subs_gaining_usermessages: # The fact that the user didn't have a UserMessage originally means we can infer that the user # was not mentioned in the original message (even if mention syntax was present, it would not # take effect for a user who was not subscribed). If we were editing the message's content, we # would rerender the message and then use the new stream's data to determine whether this is # a mention of a subscriber; but as we are not doing so, we choose to preserve the "was this # mention syntax an actual mention" decision made during the original rendering for implementation # simplicity. As a result, the only flag to consider applying here is read. um = UserMessageLite( user_profile_id=user_profile_id, message_id=message_id, flags=UserMessage.flags.read, ) ums_to_create.append(um) bulk_insert_ums(ums_to_create) # Delete UserMessage objects for users who will no # longer have access to these messages. Note: This could be # very expensive, since it's N guest users x M messages. UserMessage.objects.filter( user_profile_id__in=[ sub.user_profile_id for sub in subs_losing_usermessages ], message_id__in=changed_message_ids, ).delete() delete_event: DeleteMessagesEvent = { "type": "delete_message", "message_ids": changed_message_ids, "message_type": "stream", "stream_id": stream_being_edited.id, "topic": orig_topic_name, } delete_event_notify_user_ids = [ sub.user_profile_id for sub in subs_losing_access ] send_event(user_profile.realm, delete_event, delete_event_notify_user_ids) # Reset the Attachment.is_*_public caches for all messages # moved to another stream with different access permissions. if new_stream.invite_only != stream_being_edited.invite_only: Attachment.objects.filter( messages__in=changed_message_ids).update( is_realm_public=None, ) ArchivedAttachment.objects.filter( messages__in=changed_message_ids).update( is_realm_public=None, ) if new_stream.is_web_public != stream_being_edited.is_web_public: Attachment.objects.filter( messages__in=changed_message_ids).update( is_web_public=None, ) ArchivedAttachment.objects.filter( messages__in=changed_message_ids).update( is_web_public=None, ) # This does message.save(update_fields=[...]) save_message_for_edit_use_case(message=target_message) realm_id: Optional[int] = None if stream_being_edited is not None: realm_id = stream_being_edited.realm_id event["message_ids"] = update_to_dict_cache(changed_messages, realm_id) def user_info(um: UserMessage) -> Dict[str, Any]: return { "id": um.user_profile_id, "flags": um.flags_list(), } # The following blocks arranges that users who are subscribed to a # stream and can see history from before they subscribed get # live-update when old messages are edited (e.g. if the user does # a topic edit themself). # # We still don't send an update event to users who are not # subscribed to this stream and don't have a UserMessage row. This # means if a non-subscriber is viewing the narrow, they won't get # a real-time updates. This is a balance between sending # message-edit notifications for every public stream to every user # in the organization (too expansive, and also not what we do for # newly sent messages anyway) and having magical live-updates # where possible. users_to_be_notified = list(map(user_info, ums)) if stream_being_edited is not None: if stream_being_edited.is_history_public_to_subscribers: subscriptions = get_active_subscriptions_for_stream_id( stream_id, include_deactivated_users=False) # We exclude long-term idle users, since they by # definition have no active clients. subscriptions = subscriptions.exclude( user_profile__long_term_idle=True) # Remove duplicates by excluding the id of users already # in users_to_be_notified list. This is the case where a # user both has a UserMessage row and is a current # Subscriber subscriptions = subscriptions.exclude( user_profile_id__in=[um.user_profile_id for um in ums]) if new_stream is not None: assert delete_event_notify_user_ids is not None subscriptions = subscriptions.exclude( user_profile_id__in=delete_event_notify_user_ids) # All users that are subscribed to the stream must be # notified when a message is edited subscriber_ids = set( subscriptions.values_list("user_profile_id", flat=True)) if new_stream is not None: # TODO: Guest users don't see the new moved topic # unless breadcrumb message for new stream is # enabled. Excluding these users from receiving this # event helps us avoid a error traceback for our # clients. We should figure out a way to inform the # guest users of this new topic if sending a 'message' # event for these messages is not an option. # # Don't send this event to guest subs who are not # subscribers of the old stream but are subscribed to # the new stream; clients will be confused. old_stream_unsubbed_guests = [ sub for sub in subs_to_new_stream if sub.user_profile.is_guest and sub.user_profile_id not in subscriber_ids ] subscriptions = subscriptions.exclude(user_profile_id__in=[ sub.user_profile_id for sub in old_stream_unsubbed_guests ]) subscriber_ids = set( subscriptions.values_list("user_profile_id", flat=True)) users_to_be_notified += list( map(subscriber_info, sorted(list(subscriber_ids)))) # UserTopic updates and the content of notifications depend on # whether we've moved the entire topic, or just part of it. We # make that determination here. moved_all_visible_messages = False if topic_name is not None or new_stream is not None: assert stream_being_edited is not None if propagate_mode == "change_all": moved_all_visible_messages = True else: # With other propagate modes, if the user in fact moved # all messages in the stream, we want to explain it was a # full-topic move. # # For security model reasons, we don't want to allow a # user to take any action that would leak information # about older messages they cannot access (E.g. the only # remaining messages are in a stream without shared # history). The bulk_access_messages call below addresses # that concern. # # bulk_access_messages is inefficient for this task, since # we just want to do the exists() version of this # query. But it's nice to reuse code, and this bulk # operation is likely cheaper than a `GET /messages` # unless the topic has thousands of messages of history. unmoved_messages = messages_for_topic( stream_being_edited.recipient_id, orig_topic_name, ) visible_unmoved_messages = bulk_access_messages( user_profile, unmoved_messages, stream=stream_being_edited) moved_all_visible_messages = len(visible_unmoved_messages) == 0 # Migrate muted topic configuration in the following circumstances: # # * If propagate_mode is change_all, do so unconditionally. # # * If propagate_mode is change_later or change_one, do so when # the acting user has moved the entire topic (as visible to them). # # This rule corresponds to checking moved_all_visible_messages. # # We may want more complex behavior in cases where one appears to # be merging topics (E.g. there are existing messages in the # target topic). if moved_all_visible_messages: assert stream_being_edited is not None assert topic_name is not None or new_stream is not None for muting_user in get_users_muting_topic(stream_being_edited.id, orig_topic_name): # TODO: Ideally, this would be a bulk update operation, # because we are doing database operations in a loop here. # # This loop is only acceptable in production because it is # rare for more than a few users to have muted an # individual topic that is being moved; as of this # writing, no individual topic in Zulip Cloud had been # muted by more than 100 users. if new_stream is not None and muting_user.id in delete_event_notify_user_ids: # If the messages are being moved to a stream the user # cannot access, then we treat this as the # messages/topic being deleted for this user. This is # important for security reasons; we don't want to # give users a UserTopic row in a stream they cannot # access. Unmute the topic for such users. do_unmute_topic(muting_user, stream_being_edited, orig_topic_name) else: # Otherwise, we move the muted topic record for the user. # We call remove_topic_mute rather than do_unmute_topic to # avoid sending two events with new muted topics in # immediate succession; this is correct only because # muted_topics events always send the full set of topics. remove_topic_mute(muting_user, stream_being_edited.id, orig_topic_name) do_mute_topic( muting_user, new_stream if new_stream is not None else stream_being_edited, topic_name if topic_name is not None else orig_topic_name, ignore_duplicate=True, ) send_event(user_profile.realm, event, users_to_be_notified) if len( changed_messages ) > 0 and new_stream is not None and stream_being_edited is not None: # Notify users that the topic was moved. changed_messages_count = len(changed_messages) old_thread_notification_string = None if send_notification_to_old_thread: if moved_all_visible_messages: old_thread_notification_string = gettext_lazy( "This topic was moved to {new_location} by {user}.") elif changed_messages_count == 1: old_thread_notification_string = gettext_lazy( "A message was moved from this topic to {new_location} by {user}." ) else: old_thread_notification_string = gettext_lazy( "{changed_messages_count} messages were moved from this topic to {new_location} by {user}." ) new_thread_notification_string = None if send_notification_to_new_thread: if moved_all_visible_messages: new_thread_notification_string = gettext_lazy( "This topic was moved here from {old_location} by {user}.") elif changed_messages_count == 1: new_thread_notification_string = gettext_lazy( "A message was moved here from {old_location} by {user}.") else: new_thread_notification_string = gettext_lazy( "{changed_messages_count} messages were moved here from {old_location} by {user}." ) send_message_moved_breadcrumbs( user_profile, stream_being_edited, orig_topic_name, old_thread_notification_string, new_stream, topic_name, new_thread_notification_string, changed_messages_count, ) if (topic_name is not None and new_stream is None and content is None and len(changed_messages) > 0): assert stream_being_edited is not None maybe_send_resolve_topic_notifications( user_profile=user_profile, stream=stream_being_edited, old_topic=orig_topic_name, new_topic=topic_name, changed_messages=changed_messages, ) return len(changed_messages)
def handle(self, **options: Any) -> None: if options["percent_huddles"] + options["percent_personals"] > 100: self.stderr.write( "Error! More than 100% of messages allocated.\n") return if options["delete"]: # Start by clearing all the data in our database clear_database() # Create our two default realms # Could in theory be done via zerver.lib.actions.do_create_realm, but # welcome-bot (needed for do_create_realm) hasn't been created yet zulip_realm = Realm.objects.create( string_id="zulip", name="Zulip Dev", restricted_to_domain=True, description= "The Zulip development environment default organization." " It's great for testing!", invite_required=False, org_type=Realm.CORPORATE) RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com") if options["test_suite"]: mit_realm = Realm.objects.create(string_id="zephyr", name="MIT", restricted_to_domain=True, invite_required=False, org_type=Realm.CORPORATE) RealmDomain.objects.create(realm=mit_realm, domain="mit.edu") lear_realm = Realm.objects.create(string_id="lear", name="Lear & Co.", restricted_to_domain=False, invite_required=False, org_type=Realm.CORPORATE) # Create test Users (UserProfiles are automatically created, # as are subscriptions to the ability to receive personals). names = [ ("Zoe", "*****@*****.**"), ("Othello, the Moor of Venice", "*****@*****.**"), ("Iago", "*****@*****.**"), ("Prospero from The Tempest", "*****@*****.**"), ("Cordelia Lear", "*****@*****.**"), ("King Hamlet", "*****@*****.**"), ("aaron", "*****@*****.**"), ("Polonius", "*****@*****.**"), ] for i in range(options["extra_users"]): names.append( ('Extra User %d' % (i, ), '*****@*****.**' % (i, ))) create_users(zulip_realm, names) iago = get_user("*****@*****.**", zulip_realm) do_change_is_admin(iago, True) iago.is_staff = True iago.save(update_fields=['is_staff']) guest_user = get_user("*****@*****.**", zulip_realm) guest_user.is_guest = True guest_user.save(update_fields=['is_guest']) # These bots are directly referenced from code and thus # are needed for the test suite. all_realm_bots = [ (bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN, )) for bot in settings.INTERNAL_BOTS ] zulip_realm_bots = [ ("Zulip New User Bot", "*****@*****.**"), ("Zulip Error Bot", "*****@*****.**"), ("Zulip Default Bot", "*****@*****.**"), ("Welcome Bot", "*****@*****.**"), ] for i in range(options["extra_bots"]): zulip_realm_bots.append( ('Extra Bot %d' % (i, ), '*****@*****.**' % (i, ))) zulip_realm_bots.extend(all_realm_bots) create_users(zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT) zoe = get_user("*****@*****.**", zulip_realm) zulip_webhook_bots = [ ("Zulip Webhook Bot", "*****@*****.**"), ] # If a stream is not supplied in the webhook URL, the webhook # will (in some cases) send the notification as a PM to the # owner of the webhook bot, so bot_owner can't be None create_users(zulip_realm, zulip_webhook_bots, bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe) aaron = get_user("*****@*****.**", zulip_realm) zulip_outgoing_bots = [("Outgoing Webhook", "*****@*****.**")] create_users(zulip_realm, zulip_outgoing_bots, bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, bot_owner=aaron) # TODO: Clean up this initial bot creation code Service.objects.create( name="test", user_profile=get_user("*****@*****.**", zulip_realm), base_url="http://127.0.0.1:5002/bots/followup", token="abcd1234", interface=1) # Create public streams. stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"] stream_dict = { "Verona": { "description": "A city in Italy", "invite_only": False }, "Denmark": { "description": "A Scandinavian country", "invite_only": False }, "Scotland": { "description": "Located in the United Kingdom", "invite_only": False }, "Venice": { "description": "A northeastern Italian city", "invite_only": False }, "Rome": { "description": "Yet another Italian city", "invite_only": False } } # type: Dict[Text, Dict[Text, Any]] bulk_create_streams(zulip_realm, stream_dict) recipient_streams = [ Stream.objects.get(name=name, realm=zulip_realm).id for name in stream_list ] # type: List[int] # Create subscriptions to streams. The following # algorithm will give each of the users a different but # deterministic subset of the streams (given a fixed list # of users). subscriptions_to_add = [] # type: List[Subscription] event_time = timezone_now() all_subscription_logs = [] # type: (List[RealmAuditLog]) profiles = UserProfile.objects.select_related().filter( is_bot=False, is_guest=False).order_by( "email") # type: Sequence[UserProfile] for i, profile in enumerate(profiles): # Subscribe to some streams. for type_id in recipient_streams[:int( len(recipient_streams) * float(i) / len(profiles)) + 1]: r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id) s = Subscription(recipient=r, user_profile=profile, color=STREAM_ASSIGNMENT_COLORS[ i % len(STREAM_ASSIGNMENT_COLORS)]) subscriptions_to_add.append(s) log = RealmAuditLog(realm=profile.realm, modified_user=profile, modified_stream_id=type_id, event_last_message_id=0, event_type='subscription_created', event_time=event_time) all_subscription_logs.append(log) Subscription.objects.bulk_create(subscriptions_to_add) RealmAuditLog.objects.bulk_create(all_subscription_logs) # Create custom profile field data phone_number = try_add_realm_custom_profile_field( zulip_realm, "Phone number", CustomProfileField.SHORT_TEXT, hint='') biography = try_add_realm_custom_profile_field( zulip_realm, "Biography", CustomProfileField.LONG_TEXT, hint='What are you known for?') favorite_food = try_add_realm_custom_profile_field( zulip_realm, "Favorite food", CustomProfileField.SHORT_TEXT, hint="Or drink, if you'd prefer") # Fill in values for Iago and Hamlet hamlet = get_user("*****@*****.**", zulip_realm) do_update_user_custom_profile_data(iago, [ { "id": phone_number.id, "value": "+1-234-567-8901" }, { "id": biography.id, "value": "Betrayer of Othello." }, { "id": favorite_food.id, "value": "Apples" }, ]) do_update_user_custom_profile_data(hamlet, [ { "id": phone_number.id, "value": "+0-11-23-456-7890" }, { "id": biography.id, "value": "Prince of Denmark, and other things!" }, { "id": favorite_food.id, "value": "Dark chocolate" }, ]) else: zulip_realm = get_realm("zulip") recipient_streams = [ klass.type_id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] # Extract a list of all users user_profiles = list(UserProfile.objects.filter( is_bot=False)) # type: List[UserProfile] # Create a test realm emoji. IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images', 'test-images', 'checkbox.png') with open(IMAGE_FILE_PATH, 'rb') as fp: check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp) if not options["test_suite"]: # Populate users with some bar data for user in user_profiles: status = UserPresence.ACTIVE # type: int date = timezone_now() client = get_client("website") if user.full_name[0] <= 'H': client = get_client("ZulipAndroid") UserPresence.objects.get_or_create(user_profile=user, client=client, timestamp=date, status=status) user_profiles_ids = [user_profile.id for user_profile in user_profiles] # Create several initial huddles for i in range(options["num_huddles"]): get_huddle(random.sample(user_profiles_ids, random.randint(3, 4))) # Create several initial pairs for personals personals_pairs = [ random.sample(user_profiles_ids, 2) for i in range(options["num_personals"]) ] # Generate a new set of test data. create_test_data() # prepopulate the URL preview/embed data for the links present # in the config.generate_data.json data set. This makes it # possible for populate_db to run happily without Internet # access. with open("zerver/tests/fixtures/docs_url_preview_data.json", "r") as f: urls_with_preview_data = ujson.load(f) for url in urls_with_preview_data: cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME) threads = options["threads"] jobs = [ ] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]] for i in range(threads): count = options["num_messages"] // threads if i < options["num_messages"] % threads: count += 1 jobs.append((count, personals_pairs, options, self.stdout.write, random.randint(0, 10**10))) for job in jobs: send_messages(job) if options["delete"]: # Create the "website" and "API" clients; if we don't, the # default values in zerver/decorators.py will not work # with the Django test suite. get_client("website") get_client("API") if options["test_suite"]: # Create test users; the MIT ones are needed to test # the Zephyr mirroring codepaths. testsuite_mit_users = [ ("Fred Sipb (MIT)", "*****@*****.**"), ("Athena Consulting Exchange User (MIT)", "*****@*****.**"), ("Esp Classroom (MIT)", "*****@*****.**"), ] create_users(mit_realm, testsuite_mit_users) testsuite_lear_users = [ ("King Lear", "*****@*****.**"), ("Cordelia Lear", "*****@*****.**"), ] create_users(lear_realm, testsuite_lear_users) if not options["test_suite"]: # Initialize the email gateway bot as an API Super User email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT) email_gateway_bot.is_api_super_user = True email_gateway_bot.save() # To keep the messages.json fixtures file for the test # suite fast, don't add these users and subscriptions # when running populate_db for the test suite zulip_stream_dict = { "devel": { "description": "For developing", "invite_only": False }, "all": { "description": "For everything", "invite_only": False }, "announce": { "description": "For announcements", "invite_only": False }, "design": { "description": "For design", "invite_only": False }, "support": { "description": "For support", "invite_only": False }, "social": { "description": "For socializing", "invite_only": False }, "test": { "description": "For testing", "invite_only": False }, "errors": { "description": "For errors", "invite_only": False }, "sales": { "description": "For sales discussion", "invite_only": False } } # type: Dict[Text, Dict[Text, Any]] # Calculate the maximum number of digits in any extra stream's # number, since a stream with name "Extra Stream 3" could show # up after "Extra Stream 29". (Used later to pad numbers with # 0s). maximum_digits = len(str(options['extra_streams'] - 1)) for i in range(options['extra_streams']): # Pad the number with 0s based on `maximum_digits`. number_str = str(i).zfill(maximum_digits) extra_stream_name = 'Extra Stream ' + number_str zulip_stream_dict[extra_stream_name] = { "description": "Auto-generated extra stream.", "invite_only": False, } bulk_create_streams(zulip_realm, zulip_stream_dict) # Now that we've created the notifications stream, configure it properly. zulip_realm.notifications_stream = get_stream( "announce", zulip_realm) zulip_realm.save(update_fields=['notifications_stream']) # Add a few default streams for default_stream_name in [ "design", "devel", "social", "support" ]: DefaultStream.objects.create(realm=zulip_realm, stream=get_stream( default_stream_name, zulip_realm)) # Now subscribe everyone to these streams subscriptions_to_add = [] event_time = timezone_now() all_subscription_logs = [] profiles = UserProfile.objects.select_related().filter( realm=zulip_realm) for i, stream_name in enumerate(zulip_stream_dict): stream = Stream.objects.get(name=stream_name, realm=zulip_realm) recipient = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id) for profile in profiles: # Subscribe to some streams. s = Subscription( recipient=recipient, user_profile=profile, color=STREAM_ASSIGNMENT_COLORS[ i % len(STREAM_ASSIGNMENT_COLORS)]) subscriptions_to_add.append(s) log = RealmAuditLog(realm=profile.realm, modified_user=profile, modified_stream=stream, event_last_message_id=0, event_type='subscription_created', event_time=event_time) all_subscription_logs.append(log) Subscription.objects.bulk_create(subscriptions_to_add) RealmAuditLog.objects.bulk_create(all_subscription_logs) # These bots are not needed by the test suite internal_zulip_users_nosubs = [ ("Zulip Commit Bot", "*****@*****.**"), ("Zulip Trac Bot", "*****@*****.**"), ("Zulip Nagios Bot", "*****@*****.**"), ] create_users(zulip_realm, internal_zulip_users_nosubs, bot_type=UserProfile.DEFAULT_BOT) zulip_cross_realm_bots = [ ("Zulip Feedback Bot", "*****@*****.**"), ] create_users(zulip_realm, zulip_cross_realm_bots, bot_type=UserProfile.DEFAULT_BOT) # Mark all messages as read UserMessage.objects.all().update(flags=UserMessage.flags.read) if not options["test_suite"]: # Update pointer of each user to point to the last message in their # UserMessage rows with sender_id=user_profile_id. users = list( UserMessage.objects.filter(message__sender_id=F( 'user_profile_id')).values('user_profile_id').annotate( pointer=Max('message_id'))) for user in users: UserProfile.objects.filter( id=user['user_profile_id']).update( pointer=user['pointer']) create_user_groups() self.stdout.write("Successfully populated test database.\n")
def test_users_to_zerver_userprofile( self, mock_get_data_file: mock.Mock) -> None: custom_profile_field_user1 = { "Xf06054BBB": { "value": "random1" }, "Xf023DSCdd": { "value": "employee" }, } custom_profile_field_user2 = { "Xf06054BBB": { "value": "random2" }, "Xf023DSCdd": { "value": "employer" }, } user_data = [ { "id": "U08RGD1RD", "team_id": "T5YFFM2QY", "name": "john", "deleted": False, "is_mirror_dummy": False, "real_name": "John Doe", "profile": { "image_32": "", "email": "*****@*****.**", "avatar_hash": "hash", "phone": "+1-123-456-77-868", "fields": custom_profile_field_user1, }, }, { "id": "U0CBK5KAT", "team_id": "T5YFFM2QY", "is_admin": True, "is_bot": False, "is_owner": True, "is_primary_owner": True, "name": "Jane", "real_name": "Jane Doe", "deleted": False, "is_mirror_dummy": False, "profile": { "image_32": "https://secure.gravatar.com/avatar/random.png", "fields": custom_profile_field_user2, "email": "*****@*****.**", "avatar_hash": "hash", }, }, { "id": "U09TYF5Sk", "team_id": "T5YFFM2QY", "name": "Bot", "real_name": "Bot", "is_bot": True, "deleted": False, "is_mirror_dummy": False, "profile": { "image_32": "https://secure.gravatar.com/avatar/random1.png", "skype": "test_skype_name", "email": "*****@*****.**", "avatar_hash": "hash", }, }, { "id": "UHSG7OPQN", "team_id": "T6LARQE2Z", "name": "matt.perry", "color": "9d8eee", "is_bot": False, "is_app_user": False, "is_mirror_dummy": True, "team_domain": "foreignteam", "profile": { "image_32": "https://secure.gravatar.com/avatar/random6.png", "avatar_hash": "hash", "first_name": "Matt", "last_name": "Perry", "real_name": "Matt Perry", "display_name": "matt.perry", "team": "T6LARQE2Z", }, }, { "id": "U8VAHEVUY", "team_id": "T5YFFM2QY", "name": "steviejacob34", "real_name": "Steve Jacob", "is_admin": False, "is_owner": False, "is_primary_owner": False, "is_restricted": True, "is_ultra_restricted": False, "is_bot": False, "is_mirror_dummy": False, "profile": { "email": "*****@*****.**", "avatar_hash": "hash", "image_32": "https://secure.gravatar.com/avatar/random6.png", }, }, { "id": "U8X25EBAB", "team_id": "T5YFFM2QY", "name": "pratikweb_0", "real_name": "Pratik", "is_admin": False, "is_owner": False, "is_primary_owner": False, "is_restricted": True, "is_ultra_restricted": True, "is_bot": False, "is_mirror_dummy": False, "profile": { "email": "*****@*****.**", "avatar_hash": "hash", "image_32": "https://secure.gravatar.com/avatar/random.png", }, }, { "id": "U015J7JSE", "team_id": "T5YFFM2QY", "name": "georgesm27", "real_name": "George", "is_admin": True, "is_owner": True, "is_primary_owner": False, "is_restricted": False, "is_ultra_restricted": False, "is_bot": False, "is_mirror_dummy": False, "profile": { "email": "*****@*****.**", "avatar_hash": "hash", "image_32": "https://secure.gravatar.com/avatar/random5.png", }, }, { "id": "U1RDFEC80", "team_id": "T5YFFM2QY", "name": "daniel.smith", "real_name": "Daniel Smith", "is_admin": True, "is_owner": False, "is_primary_owner": False, "is_restricted": False, "is_ultra_restricted": False, "is_bot": False, "is_mirror_dummy": False, "profile": { "email": "*****@*****.**", "avatar_hash": "hash", "image_32": "https://secure.gravatar.com/avatar/random7.png", }, }, ] mock_get_data_file.return_value = user_data # As user with slack_id 'U0CBK5KAT' is the primary owner, that user should be imported first # and hence has zulip_id = 1 test_slack_user_id_to_zulip_user_id = { "U08RGD1RD": 1, "U0CBK5KAT": 0, "U09TYF5Sk": 2, "UHSG7OPQN": 3, "U8VAHEVUY": 4, "U8X25EBAB": 5, "U015J7JSE": 6, "U1RDFEC80": 7, } slack_data_dir = "./random_path" timestamp = int(timezone_now().timestamp()) mock_get_data_file.return_value = user_data with self.assertLogs(level="INFO"): ( zerver_userprofile, avatar_list, slack_user_id_to_zulip_user_id, customprofilefield, customprofilefield_value, ) = users_to_zerver_userprofile(slack_data_dir, user_data, 1, timestamp, "test_domain") # Test custom profile fields self.assertEqual(customprofilefield[0]["field_type"], 1) self.assertEqual(customprofilefield[3]["name"], "skype") cpf_name = {cpf["name"] for cpf in customprofilefield} self.assertIn("phone", cpf_name) self.assertIn("skype", cpf_name) cpf_name.remove("phone") cpf_name.remove("skype") for name in cpf_name: self.assertTrue(name.startswith("Slack custom field ")) self.assertEqual(len(customprofilefield_value), 6) self.assertEqual(customprofilefield_value[0]["field"], 0) self.assertEqual(customprofilefield_value[0]["user_profile"], 1) self.assertEqual(customprofilefield_value[3]["user_profile"], 0) self.assertEqual(customprofilefield_value[5]["value"], "test_skype_name") # test that the primary owner should always be imported first self.assertDictEqual(slack_user_id_to_zulip_user_id, test_slack_user_id_to_zulip_user_id) self.assertEqual(len(avatar_list), 8) self.assertEqual(len(zerver_userprofile), 8) self.assertEqual(zerver_userprofile[0]["is_staff"], False) self.assertEqual(zerver_userprofile[0]["is_bot"], False) self.assertEqual(zerver_userprofile[0]["is_active"], True) self.assertEqual(zerver_userprofile[0]["is_mirror_dummy"], False) self.assertEqual(zerver_userprofile[0]["role"], UserProfile.ROLE_MEMBER) self.assertEqual(zerver_userprofile[0]["enable_desktop_notifications"], True) self.assertEqual(zerver_userprofile[0]["email"], "*****@*****.**") self.assertEqual(zerver_userprofile[0]["full_name"], "John Doe") self.assertEqual(zerver_userprofile[1]["id"], test_slack_user_id_to_zulip_user_id["U0CBK5KAT"]) self.assertEqual(zerver_userprofile[1]["role"], UserProfile.ROLE_REALM_OWNER) self.assertEqual(zerver_userprofile[1]["is_staff"], False) self.assertEqual(zerver_userprofile[1]["is_active"], True) self.assertEqual(zerver_userprofile[0]["is_mirror_dummy"], False) self.assertEqual(zerver_userprofile[2]["id"], test_slack_user_id_to_zulip_user_id["U09TYF5Sk"]) self.assertEqual(zerver_userprofile[2]["is_bot"], True) self.assertEqual(zerver_userprofile[2]["is_active"], True) self.assertEqual(zerver_userprofile[2]["is_mirror_dummy"], False) self.assertEqual(zerver_userprofile[2]["email"], "*****@*****.**") self.assertEqual(zerver_userprofile[2]["bot_type"], 1) self.assertEqual(zerver_userprofile[2]["avatar_source"], "U") self.assertEqual(zerver_userprofile[3]["id"], test_slack_user_id_to_zulip_user_id["UHSG7OPQN"]) self.assertEqual(zerver_userprofile[3]["role"], UserProfile.ROLE_MEMBER) self.assertEqual(zerver_userprofile[3]["is_staff"], False) self.assertEqual(zerver_userprofile[3]["is_active"], False) self.assertEqual(zerver_userprofile[3]["email"], "*****@*****.**") self.assertEqual(zerver_userprofile[3]["realm"], 1) self.assertEqual(zerver_userprofile[3]["full_name"], "Matt Perry") self.assertEqual(zerver_userprofile[3]["is_mirror_dummy"], True) self.assertEqual(zerver_userprofile[3]["can_forge_sender"], False) self.assertEqual(zerver_userprofile[4]["id"], test_slack_user_id_to_zulip_user_id["U8VAHEVUY"]) self.assertEqual(zerver_userprofile[4]["role"], UserProfile.ROLE_GUEST) self.assertEqual(zerver_userprofile[4]["is_staff"], False) self.assertEqual(zerver_userprofile[4]["is_active"], True) self.assertEqual(zerver_userprofile[4]["is_mirror_dummy"], False) self.assertEqual(zerver_userprofile[5]["id"], test_slack_user_id_to_zulip_user_id["U8X25EBAB"]) self.assertEqual(zerver_userprofile[5]["role"], UserProfile.ROLE_GUEST) self.assertEqual(zerver_userprofile[5]["is_staff"], False) self.assertEqual(zerver_userprofile[5]["is_active"], True) self.assertEqual(zerver_userprofile[5]["is_mirror_dummy"], False) self.assertEqual(zerver_userprofile[6]["id"], test_slack_user_id_to_zulip_user_id["U015J7JSE"]) self.assertEqual(zerver_userprofile[6]["role"], UserProfile.ROLE_REALM_OWNER) self.assertEqual(zerver_userprofile[6]["is_staff"], False) self.assertEqual(zerver_userprofile[6]["is_active"], True) self.assertEqual(zerver_userprofile[6]["is_mirror_dummy"], False) self.assertEqual(zerver_userprofile[7]["id"], test_slack_user_id_to_zulip_user_id["U1RDFEC80"]) self.assertEqual(zerver_userprofile[7]["role"], UserProfile.ROLE_REALM_ADMINISTRATOR) self.assertEqual(zerver_userprofile[7]["is_staff"], False) self.assertEqual(zerver_userprofile[7]["is_active"], True) self.assertEqual(zerver_userprofile[7]["is_mirror_dummy"], False)
def test_search(self) -> None: reset_emails_in_zulip_realm() def check_hamlet_user_query_result(result: HttpResponse) -> None: self.assert_in_success_response(['<span class="label">user</span>\n', '<h3>King Hamlet</h3>', '<b>Email</b>: [email protected]', '<b>Is active</b>: True<br>', '<b>Admins</b>: [email protected], [email protected]\n', 'class="copy-button" data-copytext="[email protected], [email protected]"', ], result) def check_zulip_realm_query_result(result: HttpResponse) -> None: zulip_realm = get_realm("zulip") self.assert_in_success_response([f'<input type="hidden" name="realm_id" value="{zulip_realm.id}"', 'Zulip Dev</h3>', '<option value="1" selected>Self Hosted</option>', '<option value="2" >Limited</option>', 'input type="number" name="discount" value="None"', '<option value="active" selected>Active</option>', '<option value="deactivated" >Deactivated</option>', 'scrub-realm-button">', 'data-string-id="zulip"'], result) def check_lear_realm_query_result(result: HttpResponse) -> None: lear_realm = get_realm("lear") self.assert_in_success_response([f'<input type="hidden" name="realm_id" value="{lear_realm.id}"', 'Lear & Co.</h3>', '<option value="1" selected>Self Hosted</option>', '<option value="2" >Limited</option>', 'input type="number" name="discount" value="None"', '<option value="active" selected>Active</option>', '<option value="deactivated" >Deactivated</option>', 'scrub-realm-button">', 'data-string-id="lear"', '<b>Name</b>: Zulip Standard', '<b>Status</b>: Active', '<b>Billing schedule</b>: Annual', '<b>Licenses</b>: 2/10 (Manual)', '<b>Price per license</b>: $80.0', '<b>Payment method</b>: Send invoice', '<b>Next invoice date</b>: 02 January 2017', ], result) def check_preregistration_user_query_result(result: HttpResponse, email: str, invite: bool=False) -> None: self.assert_in_success_response(['<span class="label">preregistration user</span>\n', f'<b>Email</b>: {email}', ], result) if invite: self.assert_in_success_response(['<span class="label">invite</span>'], result) self.assert_in_success_response(['<b>Expires in</b>: 1\xa0week, 3', '<b>Status</b>: Link has never been clicked'], result) self.assert_in_success_response([], result) else: self.assert_not_in_success_response(['<span class="label">invite</span>'], result) self.assert_in_success_response(['<b>Expires in</b>: 1\xa0day', '<b>Status</b>: Link has never been clicked'], result) def check_realm_creation_query_result(result: HttpResponse, email: str) -> None: self.assert_in_success_response(['<span class="label">preregistration user</span>\n', '<span class="label">realm creation</span>\n', '<b>Link</b>: http://testserver/accounts/do_confirm/', '<b>Expires in</b>: 1\xa0day<br>\n', ], result) def check_multiuse_invite_link_query_result(result: HttpResponse) -> None: self.assert_in_success_response(['<span class="label">multiuse invite</span>\n', '<b>Link</b>: http://zulip.testserver/join/', '<b>Expires in</b>: 1\xa0week, 3', ], result) def check_realm_reactivation_link_query_result(result: HttpResponse) -> None: self.assert_in_success_response(['<span class="label">realm reactivation</span>\n', '<b>Link</b>: http://zulip.testserver/reactivate/', '<b>Expires in</b>: 1\xa0day', ], result) self.login('cordelia') result = self.client_get("/activity/support") self.assertEqual(result.status_code, 302) self.assertEqual(result["Location"], "/login/") self.login('iago') customer = Customer.objects.create(realm=get_realm("lear"), stripe_customer_id='cus_123') now = datetime(2016, 1, 2, tzinfo=timezone.utc) plan = CustomerPlan.objects.create(customer=customer, billing_cycle_anchor=now, billing_schedule=CustomerPlan.ANNUAL, tier=CustomerPlan.STANDARD, price_per_license=8000, next_invoice_date=add_months(now, 12)) LicenseLedger.objects.create(licenses=10, licenses_at_next_renewal=10, event_time=timezone_now(), is_renewal=True, plan=plan) result = self.client_get("/activity/support") self.assert_in_success_response(['<input type="text" name="q" class="input-xxlarge search-query"'], result) result = self.client_get("/activity/support", {"q": "*****@*****.**"}) check_hamlet_user_query_result(result) check_zulip_realm_query_result(result) result = self.client_get("/activity/support", {"q": "lear"}) check_lear_realm_query_result(result) result = self.client_get("/activity/support", {"q": "http://lear.testserver"}) check_lear_realm_query_result(result) with self.settings(REALM_HOSTS={'zulip': 'localhost'}): result = self.client_get("/activity/support", {"q": "http://localhost"}) check_zulip_realm_query_result(result) result = self.client_get("/activity/support", {"q": "[email protected], lear"}) check_hamlet_user_query_result(result) check_zulip_realm_query_result(result) check_lear_realm_query_result(result) result = self.client_get("/activity/support", {"q": "lear, Hamlet <*****@*****.**>"}) check_hamlet_user_query_result(result) check_zulip_realm_query_result(result) check_lear_realm_query_result(result) self.client_post('/accounts/home/', {'email': self.nonreg_email("test")}) self.login('iago') result = self.client_get("/activity/support", {"q": self.nonreg_email("test")}) check_preregistration_user_query_result(result, self.nonreg_email("test")) check_zulip_realm_query_result(result) stream_ids = [self.get_stream_id("Denmark")] invitee_emails = [self.nonreg_email("test1")] self.client_post("/json/invites", {"invitee_emails": invitee_emails, "stream_ids": orjson.dumps(stream_ids).decode(), "invite_as": PreregistrationUser.INVITE_AS['MEMBER']}) result = self.client_get("/activity/support", {"q": self.nonreg_email("test1")}) check_preregistration_user_query_result(result, self.nonreg_email("test1"), invite=True) check_zulip_realm_query_result(result) email = self.nonreg_email('alice') self.client_post('/new/', {'email': email}) result = self.client_get("/activity/support", {"q": email}) check_realm_creation_query_result(result, email) do_create_multiuse_invite_link(self.example_user("hamlet"), invited_as=1) result = self.client_get("/activity/support", {"q": "zulip"}) check_multiuse_invite_link_query_result(result) check_zulip_realm_query_result(result) MultiuseInvite.objects.all().delete() do_send_realm_reactivation_email(get_realm("zulip")) result = self.client_get("/activity/support", {"q": "zulip"}) check_realm_reactivation_link_query_result(result) check_zulip_realm_query_result(result)
def handle(self, *args: Any, **options: Any) -> None: # TODO: This should arguably only delete the objects # associated with the "analytics" realm. do_drop_all_analytics_tables() # This also deletes any objects with this realm as a foreign key Realm.objects.filter(string_id="analytics").delete() # Because we just deleted a bunch of objects in the database # directly (rather than deleting individual objects in Django, # in which case our post_save hooks would have flushed the # individual objects from memcached for us), we need to flush # memcached in order to ensure deleted objects aren't still # present in the memcached cache. from zerver.apps import flush_cache flush_cache(None) installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA) last_end_time = floor_to_day(timezone_now()) realm = do_create_realm(string_id="analytics", name="Analytics", date_created=installation_time) with mock.patch("zerver.lib.create_user.timezone_now", return_value=installation_time): shylock = create_user( "*****@*****.**", "Shylock", realm, full_name="Shylock", role=UserProfile.ROLE_REALM_OWNER, ) do_change_user_role(shylock, UserProfile.ROLE_REALM_OWNER, acting_user=None) stream = Stream.objects.create(name="all", realm=realm, date_created=installation_time) recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM) stream.recipient = recipient stream.save(update_fields=["recipient"]) # Subscribe shylock to the stream to avoid invariant failures. # TODO: This should use subscribe_users_to_streams from populate_db. subs = [ Subscription( recipient=recipient, user_profile=shylock, is_user_active=shylock.is_active, color=STREAM_ASSIGNMENT_COLORS[0], ), ] Subscription.objects.bulk_create(subs) FixtureData = Mapping[Union[str, int, None], List[int]] def insert_fixture_data( stat: CountStat, fixture_data: FixtureData, table: Type[BaseCount], ) -> None: end_times = time_range(last_end_time, last_end_time, stat.frequency, len(list(fixture_data.values())[0])) if table == InstallationCount: id_args: Dict[str, Any] = {} if table == RealmCount: id_args = {"realm": realm} if table == UserCount: id_args = {"realm": realm, "user": shylock} if table == StreamCount: id_args = {"stream": stream, "realm": realm} for subgroup, values in fixture_data.items(): table.objects.bulk_create( table( property=stat.property, subgroup=subgroup, end_time=end_time, value=value, **id_args, ) for end_time, value in zip(end_times, values) if value != 0) stat = COUNT_STATS["1day_actives::day"] realm_data: FixtureData = { None: self.generate_fixture_data(stat, 0.08, 0.02, 3, 0.3, 6, partial_sum=True), } insert_fixture_data(stat, realm_data, RealmCount) installation_data: FixtureData = { None: self.generate_fixture_data(stat, 0.8, 0.2, 4, 0.3, 6, partial_sum=True), } insert_fixture_data(stat, installation_data, InstallationCount) FillState.objects.create(property=stat.property, end_time=last_end_time, state=FillState.DONE) stat = COUNT_STATS["7day_actives::day"] realm_data = { None: self.generate_fixture_data(stat, 0.2, 0.07, 3, 0.3, 6, partial_sum=True), } insert_fixture_data(stat, realm_data, RealmCount) installation_data = { None: self.generate_fixture_data(stat, 2, 0.7, 4, 0.3, 6, partial_sum=True), } insert_fixture_data(stat, installation_data, InstallationCount) FillState.objects.create(property=stat.property, end_time=last_end_time, state=FillState.DONE) stat = COUNT_STATS["realm_active_humans::day"] realm_data = { None: self.generate_fixture_data(stat, 0.8, 0.08, 3, 0.5, 3, partial_sum=True), } insert_fixture_data(stat, realm_data, RealmCount) installation_data = { None: self.generate_fixture_data(stat, 1, 0.3, 4, 0.5, 3, partial_sum=True), } insert_fixture_data(stat, installation_data, InstallationCount) FillState.objects.create(property=stat.property, end_time=last_end_time, state=FillState.DONE) stat = COUNT_STATS["active_users_audit:is_bot:day"] realm_data = { "false": self.generate_fixture_data(stat, 1, 0.2, 3.5, 0.8, 2, partial_sum=True), "true": self.generate_fixture_data(stat, 0.3, 0.05, 3, 0.3, 2, partial_sum=True), } insert_fixture_data(stat, realm_data, RealmCount) installation_data = { "false": self.generate_fixture_data(stat, 3, 1, 4, 0.8, 2, partial_sum=True), "true": self.generate_fixture_data(stat, 1, 0.4, 4, 0.8, 2, partial_sum=True), } insert_fixture_data(stat, installation_data, InstallationCount) FillState.objects.create(property=stat.property, end_time=last_end_time, state=FillState.DONE) stat = COUNT_STATS["messages_sent:is_bot:hour"] user_data: FixtureData = { "false": self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8, holiday_rate=0.1), } insert_fixture_data(stat, user_data, UserCount) realm_data = { "false": self.generate_fixture_data(stat, 35, 15, 6, 0.6, 4), "true": self.generate_fixture_data(stat, 15, 15, 3, 0.4, 2), } insert_fixture_data(stat, realm_data, RealmCount) installation_data = { "false": self.generate_fixture_data(stat, 350, 150, 6, 0.6, 4), "true": self.generate_fixture_data(stat, 150, 150, 3, 0.4, 2), } insert_fixture_data(stat, installation_data, InstallationCount) FillState.objects.create(property=stat.property, end_time=last_end_time, state=FillState.DONE) stat = COUNT_STATS["messages_sent:message_type:day"] user_data = { "public_stream": self.generate_fixture_data(stat, 1.5, 1, 3, 0.6, 8), "private_message": self.generate_fixture_data(stat, 0.5, 0.3, 1, 0.6, 8), "huddle_message": self.generate_fixture_data(stat, 0.2, 0.2, 2, 0.6, 8), } insert_fixture_data(stat, user_data, UserCount) realm_data = { "public_stream": self.generate_fixture_data(stat, 30, 8, 5, 0.6, 4), "private_stream": self.generate_fixture_data(stat, 7, 7, 5, 0.6, 4), "private_message": self.generate_fixture_data(stat, 13, 5, 5, 0.6, 4), "huddle_message": self.generate_fixture_data(stat, 6, 3, 3, 0.6, 4), } insert_fixture_data(stat, realm_data, RealmCount) installation_data = { "public_stream": self.generate_fixture_data(stat, 300, 80, 5, 0.6, 4), "private_stream": self.generate_fixture_data(stat, 70, 70, 5, 0.6, 4), "private_message": self.generate_fixture_data(stat, 130, 50, 5, 0.6, 4), "huddle_message": self.generate_fixture_data(stat, 60, 30, 3, 0.6, 4), } insert_fixture_data(stat, installation_data, InstallationCount) FillState.objects.create(property=stat.property, end_time=last_end_time, state=FillState.DONE) website, created = Client.objects.get_or_create(name="website") old_desktop, created = Client.objects.get_or_create( name="desktop app Linux 0.3.7") android, created = Client.objects.get_or_create(name="ZulipAndroid") iOS, created = Client.objects.get_or_create(name="ZulipiOS") react_native, created = Client.objects.get_or_create( name="ZulipMobile") API, created = Client.objects.get_or_create(name="API: Python") zephyr_mirror, created = Client.objects.get_or_create( name="zephyr_mirror") unused, created = Client.objects.get_or_create(name="unused") long_webhook, created = Client.objects.get_or_create( name="ZulipLooooooooooongNameWebhook") stat = COUNT_STATS["messages_sent:client:day"] user_data = { website.id: self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8), zephyr_mirror.id: self.generate_fixture_data(stat, 0, 0.3, 1.5, 0.6, 8), } insert_fixture_data(stat, user_data, UserCount) realm_data = { website.id: self.generate_fixture_data(stat, 30, 20, 5, 0.6, 3), old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, 0.6, 3), android.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3), iOS.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3), react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3), API.id: self.generate_fixture_data(stat, 5, 5, 5, 0.6, 3), zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, 0.6, 3), unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0), long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3), } insert_fixture_data(stat, realm_data, RealmCount) installation_data = { website.id: self.generate_fixture_data(stat, 300, 200, 5, 0.6, 3), old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, 0.6, 3), android.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3), iOS.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3), react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3), API.id: self.generate_fixture_data(stat, 50, 50, 5, 0.6, 3), zephyr_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, 0.6, 3), unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0), long_webhook.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3), } insert_fixture_data(stat, installation_data, InstallationCount) FillState.objects.create(property=stat.property, end_time=last_end_time, state=FillState.DONE) stat = COUNT_STATS["messages_in_stream:is_bot:day"] realm_data = { "false": self.generate_fixture_data(stat, 30, 5, 6, 0.6, 4), "true": self.generate_fixture_data(stat, 20, 2, 3, 0.2, 3), } insert_fixture_data(stat, realm_data, RealmCount) stream_data: Mapping[Union[int, str, None], List[int]] = { "false": self.generate_fixture_data(stat, 10, 7, 5, 0.6, 4), "true": self.generate_fixture_data(stat, 5, 3, 2, 0.4, 2), } insert_fixture_data(stat, stream_data, StreamCount) FillState.objects.create(property=stat.property, end_time=last_end_time, state=FillState.DONE) stat = COUNT_STATS["messages_read::hour"] user_data = { None: self.generate_fixture_data(stat, 7, 3, 2, 0.6, 8, holiday_rate=0.1), } insert_fixture_data(stat, user_data, UserCount) realm_data = { None: self.generate_fixture_data(stat, 50, 35, 6, 0.6, 4) } insert_fixture_data(stat, realm_data, RealmCount) FillState.objects.create(property=stat.property, end_time=last_end_time, state=FillState.DONE)
def is_analytics_ready(realm: Realm) -> bool: return (timezone_now() - realm.date_created) > MAX_TIME_FOR_FULL_ANALYTICS_GENERATION
def consume(self, event: Dict[str, Any]) -> None: start = time.time() if event["type"] == "mark_stream_messages_as_read": user_profile = get_user_profile_by_id(event["user_profile_id"]) for recipient_id in event["stream_recipient_ids"]: do_mark_stream_messages_as_read(user_profile, recipient_id) elif event["type"] == "mark_stream_messages_as_read_for_everyone": # This event is generated by the stream deactivation code path. batch_size = 100 offset = 0 while True: messages = Message.objects.filter( recipient_id=event["stream_recipient_id"] ).order_by("id")[offset : offset + batch_size] UserMessage.objects.filter(message__in=messages).extra( where=[UserMessage.where_unread()] ).update(flags=F("flags").bitor(UserMessage.flags.read)) offset += len(messages) if len(messages) < batch_size: break elif event["type"] == "clear_push_device_tokens": try: clear_push_device_tokens(event["user_profile_id"]) except PushNotificationBouncerRetryLaterError: def failure_processor(event: Dict[str, Any]) -> None: logger.warning( "Maximum retries exceeded for trigger:%s event:clear_push_device_tokens", event["user_profile_id"], ) retry_event(self.queue_name, event, failure_processor) elif event["type"] == "realm_export": realm = Realm.objects.get(id=event["realm_id"]) output_dir = tempfile.mkdtemp(prefix="zulip-export-") export_event = RealmAuditLog.objects.get(id=event["id"]) user_profile = get_user_profile_by_id(event["user_profile_id"]) try: public_url = export_realm_wrapper( realm=realm, output_dir=output_dir, threads=6, upload=True, public_only=True, delete_after_upload=True, ) except Exception: export_event.extra_data = orjson.dumps( dict( failed_timestamp=timezone_now().timestamp(), ) ).decode() export_event.save(update_fields=["extra_data"]) logging.error( "Data export for %s failed after %s", user_profile.realm.string_id, time.time() - start, ) notify_realm_export(user_profile) return assert public_url is not None # Update the extra_data field now that the export is complete. export_event.extra_data = orjson.dumps( dict( export_path=urllib.parse.urlparse(public_url).path, ) ).decode() export_event.save(update_fields=["extra_data"]) # Send a private message notification letting the user who # triggered the export know the export finished. with override_language(user_profile.default_language): content = _( "Your data export is complete and has been uploaded here:\n\n{public_url}" ).format(public_url=public_url) internal_send_private_message( sender=get_system_bot(settings.NOTIFICATION_BOT), recipient_user=user_profile, content=content, ) # For future frontend use, also notify administrator # clients that the export happened. notify_realm_export(user_profile) logging.info( "Completed data export for %s in %s", user_profile.realm.string_id, time.time() - start, ) end = time.time() logger.info("deferred_work processed %s event (%dms)", event["type"], (end - start) * 1000)
def handle(self, *args: Any, **options: Any) -> None: # Suppress spammy output from the push notifications logger push_notifications_logger.disabled = True if options["percent_huddles"] + options["percent_personals"] > 100: self.stderr.write("Error! More than 100% of messages allocated.\n") return # Get consistent data for backend tests. if options["test_suite"]: random.seed(0) with connection.cursor() as cursor: # Sometimes bugs relating to confusing recipient.id for recipient.type_id # or <object>.id for <object>.recipient_id remain undiscovered by the test suite # due to these numbers happening to coincide in such a way that it makes tests # accidentally pass. By bumping the Recipient.id sequence by a large enough number, # we can have those ids in a completely different range of values than object ids, # eliminating the possibility of such coincidences. cursor.execute("SELECT setval('zerver_recipient_id_seq', 100)") if options["max_topics"] is None: # If max_topics is not set, we use a default that's big # enough "more topics" should appear, and scales slowly # with the number of messages. options["max_topics"] = 8 + options["num_messages"] // 1000 if options["delete"]: # Start by clearing all the data in our database clear_database() # Create our three default realms # Could in theory be done via zerver.actions.create_realm.do_create_realm, but # welcome-bot (needed for do_create_realm) hasn't been created yet create_internal_realm() zulip_realm = do_create_realm( string_id="zulip", name="Zulip Dev", emails_restricted_to_domains=False, email_address_visibility=Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS, description="The Zulip development environment default organization." " It's great for testing!", invite_required=False, plan_type=Realm.PLAN_TYPE_SELF_HOSTED, org_type=Realm.ORG_TYPES["business"]["id"], enable_spectator_access=True, ) RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com") assert zulip_realm.notifications_stream is not None zulip_realm.notifications_stream.name = "Verona" zulip_realm.notifications_stream.description = "A city in Italy" zulip_realm.notifications_stream.save(update_fields=["name", "description"]) realm_user_default = RealmUserDefault.objects.get(realm=zulip_realm) realm_user_default.enter_sends = True realm_user_default.save() if options["test_suite"]: mit_realm = do_create_realm( string_id="zephyr", name="MIT", emails_restricted_to_domains=True, invite_required=False, plan_type=Realm.PLAN_TYPE_SELF_HOSTED, org_type=Realm.ORG_TYPES["business"]["id"], ) RealmDomain.objects.create(realm=mit_realm, domain="mit.edu") lear_realm = do_create_realm( string_id="lear", name="Lear & Co.", emails_restricted_to_domains=False, invite_required=False, plan_type=Realm.PLAN_TYPE_SELF_HOSTED, org_type=Realm.ORG_TYPES["business"]["id"], ) # Default to allowing all members to send mentions in # large streams for the test suite to keep # mention-related tests simple. zulip_realm.wildcard_mention_policy = Realm.WILDCARD_MENTION_POLICY_MEMBERS zulip_realm.save(update_fields=["wildcard_mention_policy"]) # Create test Users (UserProfiles are automatically created, # as are subscriptions to the ability to receive personals). names = [ ("Zoe", "*****@*****.**"), ("Othello, the Moor of Venice", "*****@*****.**"), ("Iago", "*****@*****.**"), ("Prospero from The Tempest", "*****@*****.**"), ("Cordelia, Lear's daughter", "*****@*****.**"), ("King Hamlet", "*****@*****.**"), ("aaron", "*****@*****.**"), ("Polonius", "*****@*****.**"), ("Desdemona", "*****@*****.**"), ("शिव", "*****@*****.**"), ] # For testing really large batches: # Create extra users with semi realistic names to make search # functions somewhat realistic. We'll still create 1000 users # like Extra222 User for some predictability. num_names = options["extra_users"] num_boring_names = 300 for i in range(min(num_names, num_boring_names)): full_name = f"Extra{i:03} User" names.append((full_name, f"extrauser{i}@zulip.com")) if num_names > num_boring_names: fnames = [ "Amber", "Arpita", "Bob", "Cindy", "Daniela", "Dan", "Dinesh", "Faye", "François", "George", "Hank", "Irene", "James", "Janice", "Jenny", "Jill", "John", "Kate", "Katelyn", "Kobe", "Lexi", "Manish", "Mark", "Matt", "Mayna", "Michael", "Pete", "Peter", "Phil", "Phillipa", "Preston", "Sally", "Scott", "Sandra", "Steve", "Stephanie", "Vera", ] mnames = ["de", "van", "von", "Shaw", "T."] lnames = [ "Adams", "Agarwal", "Beal", "Benson", "Bonita", "Davis", "George", "Harden", "James", "Jones", "Johnson", "Jordan", "Lee", "Leonard", "Singh", "Smith", "Patel", "Towns", "Wall", ] non_ascii_names = [ "Günter", "أحمد", "Magnús", "आशी", "イツキ", "语嫣", "அருண்", "Александр", "José", ] # to imitate emoji insertions in usernames raw_emojis = ["😎", "😂", "🐱👤"] for i in range(num_boring_names, num_names): fname = random.choice(fnames) + str(i) full_name = fname if random.random() < 0.7: if random.random() < 0.3: full_name += " " + random.choice(non_ascii_names) else: full_name += " " + random.choice(mnames) if random.random() < 0.1: full_name += f" {random.choice(raw_emojis)} " else: full_name += " " + random.choice(lnames) email = fname.lower() + "@zulip.com" names.append((full_name, email)) create_users(zulip_realm, names, tos_version=settings.TERMS_OF_SERVICE_VERSION) # Add time zones to some users. Ideally, this would be # done in the initial create_users calls, but the # tuple-based interface for that function doesn't support # doing so. def assign_time_zone_by_delivery_email(delivery_email: str, new_time_zone: str) -> None: u = get_user_by_delivery_email(delivery_email, zulip_realm) u.timezone = new_time_zone u.save(update_fields=["timezone"]) # Note: Hamlet keeps default time zone of "". assign_time_zone_by_delivery_email("*****@*****.**", "US/Pacific") assign_time_zone_by_delivery_email("*****@*****.**", "US/Pacific") assign_time_zone_by_delivery_email("*****@*****.**", "US/Eastern") assign_time_zone_by_delivery_email("*****@*****.**", "US/Eastern") assign_time_zone_by_delivery_email("*****@*****.**", "Canada/Newfoundland") assign_time_zone_by_delivery_email("*****@*****.**", "Asia/Shanghai") # China assign_time_zone_by_delivery_email("*****@*****.**", "Asia/Kolkata") # India assign_time_zone_by_delivery_email("*****@*****.**", "UTC") users = UserProfile.objects.filter(realm=zulip_realm) # All users in development environment are full members initially because # waiting period threshold is 0. Groups of Iago, Dedemona, Shiva and # Polonius will be updated according to their role in do_change_user_role. full_members_user_group = UserGroup.objects.get( realm=zulip_realm, name="@role:fullmembers", is_system_group=True ) members_user_group = UserGroup.objects.get( realm=zulip_realm, name="@role:members", is_system_group=True ) user_group_memberships = [] for user_profile in list(users): for group in [full_members_user_group, members_user_group]: user_group_membership = UserGroupMembership( user_group=group, user_profile=user_profile ) user_group_memberships.append(user_group_membership) UserGroupMembership.objects.bulk_create(user_group_memberships) iago = get_user_by_delivery_email("*****@*****.**", zulip_realm) do_change_user_role(iago, UserProfile.ROLE_REALM_ADMINISTRATOR, acting_user=None) iago.is_staff = True iago.save(update_fields=["is_staff"]) # We need to create at least two test draft for Iago for the sake # of the cURL tests. Two since one will be deleted. Draft.objects.create( user_profile=iago, recipient=None, topic="Release Notes", content="Release 4.0 will contain ...", last_edit_time=timezone_now(), ) Draft.objects.create( user_profile=iago, recipient=None, topic="Release Notes", content="Release 4.0 will contain many new features such as ... ", last_edit_time=timezone_now(), ) desdemona = get_user_by_delivery_email("*****@*****.**", zulip_realm) do_change_user_role(desdemona, UserProfile.ROLE_REALM_OWNER, acting_user=None) shiva = get_user_by_delivery_email("*****@*****.**", zulip_realm) do_change_user_role(shiva, UserProfile.ROLE_MODERATOR, acting_user=None) polonius = get_user_by_delivery_email("*****@*****.**", zulip_realm) do_change_user_role(polonius, UserProfile.ROLE_GUEST, acting_user=None) # These bots are directly referenced from code and thus # are needed for the test suite. zulip_realm_bots = [ ("Zulip Error Bot", "*****@*****.**"), ("Zulip Default Bot", "*****@*****.**"), ] for i in range(options["extra_bots"]): zulip_realm_bots.append((f"Extra Bot {i}", f"extrabot{i}@zulip.com")) create_users( zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT, bot_owner=desdemona ) zoe = get_user_by_delivery_email("*****@*****.**", zulip_realm) zulip_webhook_bots = [ ("Zulip Webhook Bot", "*****@*****.**"), ] # If a stream is not supplied in the webhook URL, the webhook # will (in some cases) send the notification as a PM to the # owner of the webhook bot, so bot_owner can't be None create_users( zulip_realm, zulip_webhook_bots, bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe, ) aaron = get_user_by_delivery_email("*****@*****.**", zulip_realm) zulip_outgoing_bots = [ ("Outgoing Webhook", "*****@*****.**"), ] create_users( zulip_realm, zulip_outgoing_bots, bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, bot_owner=aaron, ) outgoing_webhook = get_user("*****@*****.**", zulip_realm) add_service( "outgoing-webhook", user_profile=outgoing_webhook, interface=Service.GENERIC, base_url="http://127.0.0.1:5002", token=generate_api_key(), ) # Add the realm internal bots to each realm. create_if_missing_realm_internal_bots() # Create public streams. signups_stream = Realm.INITIAL_PRIVATE_STREAM_NAME stream_list = [ "Verona", "Denmark", "Scotland", "Venice", "Rome", signups_stream, ] stream_dict: Dict[str, Dict[str, Any]] = { "Denmark": {"description": "A Scandinavian country"}, "Scotland": {"description": "Located in the United Kingdom"}, "Venice": {"description": "A northeastern Italian city"}, "Rome": {"description": "Yet another Italian city", "is_web_public": True}, } bulk_create_streams(zulip_realm, stream_dict) recipient_streams: List[int] = [ Stream.objects.get(name=name, realm=zulip_realm).id for name in stream_list ] # Create subscriptions to streams. The following # algorithm will give each of the users a different but # deterministic subset of the streams (given a fixed list # of users). For the test suite, we have a fixed list of # subscriptions to make sure test data is consistent # across platforms. subscriptions_list: List[Tuple[UserProfile, Recipient]] = [] profiles: Sequence[UserProfile] = list( UserProfile.objects.select_related().filter(is_bot=False).order_by("email") ) if options["test_suite"]: subscriptions_map = { "*****@*****.**": ["Verona"], "*****@*****.**": ["Verona"], "*****@*****.**": ["Verona", "Denmark", signups_stream], "*****@*****.**": [ "Verona", "Denmark", "Scotland", signups_stream, ], "*****@*****.**": ["Verona", "Denmark", "Scotland"], "*****@*****.**": ["Verona", "Denmark", "Scotland", "Venice"], "*****@*****.**": ["Verona", "Denmark", "Scotland", "Venice", "Rome"], "*****@*****.**": ["Verona"], "*****@*****.**": [ "Verona", "Denmark", "Venice", signups_stream, ], "*****@*****.**": ["Verona", "Denmark", "Scotland"], } for profile in profiles: email = profile.delivery_email if email not in subscriptions_map: raise Exception(f"Subscriptions not listed for user {email}") for stream_name in subscriptions_map[email]: stream = Stream.objects.get(name=stream_name, realm=zulip_realm) r = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id) subscriptions_list.append((profile, r)) else: num_streams = len(recipient_streams) num_users = len(profiles) for i, profile in enumerate(profiles): # Subscribe to some streams. fraction = float(i) / num_users num_recips = int(num_streams * fraction) + 1 for type_id in recipient_streams[:num_recips]: r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id) subscriptions_list.append((profile, r)) subscriptions_to_add: List[Subscription] = [] event_time = timezone_now() all_subscription_logs: (List[RealmAuditLog]) = [] i = 0 for profile, recipient in subscriptions_list: i += 1 color = STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)] s = Subscription( recipient=recipient, user_profile=profile, is_user_active=profile.is_active, color=color, ) subscriptions_to_add.append(s) log = RealmAuditLog( realm=profile.realm, modified_user=profile, modified_stream_id=recipient.type_id, event_last_message_id=0, event_type=RealmAuditLog.SUBSCRIPTION_CREATED, event_time=event_time, ) all_subscription_logs.append(log) Subscription.objects.bulk_create(subscriptions_to_add) RealmAuditLog.objects.bulk_create(all_subscription_logs) # Create custom profile field data phone_number = try_add_realm_custom_profile_field( zulip_realm, "Phone number", CustomProfileField.SHORT_TEXT, hint="" ) biography = try_add_realm_custom_profile_field( zulip_realm, "Biography", CustomProfileField.LONG_TEXT, hint="What are you known for?", ) favorite_food = try_add_realm_custom_profile_field( zulip_realm, "Favorite food", CustomProfileField.SHORT_TEXT, hint="Or drink, if you'd prefer", ) field_data: ProfileFieldData = { "vim": {"text": "Vim", "order": "1"}, "emacs": {"text": "Emacs", "order": "2"}, } favorite_editor = try_add_realm_custom_profile_field( zulip_realm, "Favorite editor", CustomProfileField.SELECT, field_data=field_data ) birthday = try_add_realm_custom_profile_field( zulip_realm, "Birthday", CustomProfileField.DATE ) favorite_website = try_add_realm_custom_profile_field( zulip_realm, "Favorite website", CustomProfileField.URL, hint="Or your personal blog's URL", ) mentor = try_add_realm_custom_profile_field( zulip_realm, "Mentor", CustomProfileField.USER ) github_profile = try_add_realm_default_custom_profile_field(zulip_realm, "github") # Fill in values for Iago and Hamlet hamlet = get_user_by_delivery_email("*****@*****.**", zulip_realm) do_update_user_custom_profile_data_if_changed( iago, [ {"id": phone_number.id, "value": "+1-234-567-8901"}, {"id": biography.id, "value": "Betrayer of Othello."}, {"id": favorite_food.id, "value": "Apples"}, {"id": favorite_editor.id, "value": "emacs"}, {"id": birthday.id, "value": "2000-01-01"}, {"id": favorite_website.id, "value": "https://zulip.readthedocs.io/en/latest/"}, {"id": mentor.id, "value": [hamlet.id]}, {"id": github_profile.id, "value": "zulip"}, ], ) do_update_user_custom_profile_data_if_changed( hamlet, [ {"id": phone_number.id, "value": "+0-11-23-456-7890"}, { "id": biography.id, "value": "I am:\n* The prince of Denmark\n* Nephew to the usurping Claudius", }, {"id": favorite_food.id, "value": "Dark chocolate"}, {"id": favorite_editor.id, "value": "vim"}, {"id": birthday.id, "value": "1900-01-01"}, {"id": favorite_website.id, "value": "https://blog.zulig.org"}, {"id": mentor.id, "value": [iago.id]}, {"id": github_profile.id, "value": "zulipbot"}, ], ) else: zulip_realm = get_realm("zulip") recipient_streams = [ klass.type_id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] # Extract a list of all users user_profiles: List[UserProfile] = list(UserProfile.objects.filter(is_bot=False)) # Create a test realm emoji. IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png") with open(IMAGE_FILE_PATH, "rb") as fp: check_add_realm_emoji(zulip_realm, "green_tick", iago, File(fp)) if not options["test_suite"]: # Populate users with some bar data for user in user_profiles: status: int = UserPresence.ACTIVE date = timezone_now() client = get_client("website") if user.full_name[0] <= "H": client = get_client("ZulipAndroid") UserPresence.objects.get_or_create( user_profile=user, realm_id=user.realm_id, client=client, timestamp=date, status=status, ) user_profiles_ids = [user_profile.id for user_profile in user_profiles] # Create several initial huddles for i in range(options["num_huddles"]): get_huddle(random.sample(user_profiles_ids, random.randint(3, 4))) # Create several initial pairs for personals personals_pairs = [ random.sample(user_profiles_ids, 2) for i in range(options["num_personals"]) ] create_alert_words(zulip_realm.id) # Generate a new set of test data. create_test_data() if options["delete"]: if options["test_suite"]: # Create test users; the MIT ones are needed to test # the Zephyr mirroring codepaths. event_time = timezone_now() testsuite_mit_users = [ ("Fred Sipb (MIT)", "*****@*****.**"), ("Athena Consulting Exchange User (MIT)", "*****@*****.**"), ("Esp Classroom (MIT)", "*****@*****.**"), ] create_users( mit_realm, testsuite_mit_users, tos_version=settings.TERMS_OF_SERVICE_VERSION ) mit_user = get_user_by_delivery_email("*****@*****.**", mit_realm) mit_signup_stream = Stream.objects.get( name=Realm.INITIAL_PRIVATE_STREAM_NAME, realm=mit_realm ) bulk_add_subscriptions(mit_realm, [mit_signup_stream], [mit_user], acting_user=None) testsuite_lear_users = [ ("King Lear", "*****@*****.**"), ("Cordelia, Lear's daughter", "*****@*****.**"), ] create_users( lear_realm, testsuite_lear_users, tos_version=settings.TERMS_OF_SERVICE_VERSION ) lear_user = get_user_by_delivery_email("*****@*****.**", lear_realm) lear_signup_stream = Stream.objects.get( name=Realm.INITIAL_PRIVATE_STREAM_NAME, realm=lear_realm ) bulk_add_subscriptions( lear_realm, [lear_signup_stream], [lear_user], acting_user=None ) if not options["test_suite"]: # To keep the messages.json fixtures file for the test # suite fast, don't add these users and subscriptions # when running populate_db for the test suite # to imitate emoji insertions in stream names raw_emojis = ["😎", "😂", "🐱👤"] zulip_stream_dict: Dict[str, Dict[str, Any]] = { "devel": {"description": "For developing"}, # ビデオゲーム - VideoGames (japanese) "ビデオゲーム": {"description": f"Share your favorite video games! {raw_emojis[2]}"}, "announce": { "description": "For announcements", "stream_post_policy": Stream.STREAM_POST_POLICY_ADMINS, }, "design": {"description": "For design"}, "support": {"description": "For support"}, "social": {"description": "For socializing"}, "test": {"description": "For testing `code`"}, "errors": {"description": "For errors"}, # 조리법 - Recipes (Korean) , Пельмени - Dumplings (Russian) "조리법 " + raw_emojis[0]: {"description": "Everything cooking, from pasta to Пельмени"}, } extra_stream_names = [ "802.11a", "Ad Hoc Network", "Augmented Reality", "Cycling", "DPI", "FAQ", "FiFo", "commits", "Control panel", "desktop", "компьютеры", "Data security", "desktop", "काम", "discussions", "Cloud storage", "GCI", "Vaporware", "Recent Trends", "issues", "live", "Health", "mobile", "空間", "provision", "hidrógeno", "HR", "アニメ", ] # Add stream names and stream descriptions for i in range(options["extra_streams"]): extra_stream_name = random.choice(extra_stream_names) + " " + str(i) # to imitate emoji insertions in stream names if random.random() <= 0.15: extra_stream_name += random.choice(raw_emojis) zulip_stream_dict[extra_stream_name] = { "description": "Auto-generated extra stream.", } bulk_create_streams(zulip_realm, zulip_stream_dict) # Now that we've created the notifications stream, configure it properly. zulip_realm.notifications_stream = get_stream("announce", zulip_realm) zulip_realm.save(update_fields=["notifications_stream"]) # Add a few default streams for default_stream_name in ["design", "devel", "social", "support"]: DefaultStream.objects.create( realm=zulip_realm, stream=get_stream(default_stream_name, zulip_realm) ) # Now subscribe everyone to these streams subscribe_users_to_streams(zulip_realm, zulip_stream_dict) create_user_groups() if not options["test_suite"]: # We populate the analytics database here for # development purpose only call_command("populate_analytics_db") threads = options["threads"] jobs: List[Tuple[int, List[List[int]], Dict[str, Any], int]] = [] for i in range(threads): count = options["num_messages"] // threads if i < options["num_messages"] % threads: count += 1 jobs.append((count, personals_pairs, options, random.randint(0, 10**10))) for job in jobs: generate_and_send_messages(job) if options["delete"]: if not options["test_suite"]: # These bots are not needed by the test suite # Also, we don't want interacting with each other # in dev setup. internal_zulip_users_nosubs = [ ("Zulip Commit Bot", "*****@*****.**"), ("Zulip Trac Bot", "*****@*****.**"), ("Zulip Nagios Bot", "*****@*****.**"), ] create_users( zulip_realm, internal_zulip_users_nosubs, bot_type=UserProfile.DEFAULT_BOT, bot_owner=desdemona, ) mark_all_messages_as_read() self.stdout.write("Successfully populated test database.\n") push_notifications_logger.disabled = False
def back_date(num_weeks: int) -> None: user_presence = UserPresence.objects.filter( user_profile=user_profile)[0] user_presence.timestamp = timezone_now() - datetime.timedelta( weeks=num_weeks) user_presence.save()
def convert_channel_data(channel_data: List[ZerverFieldsT], user_data_map: Dict[str, Dict[str, Any]], subscriber_handler: SubscriberHandler, stream_id_mapper: IdMapper, user_id_mapper: IdMapper, realm_id: int, team_name: str) -> List[ZerverFieldsT]: channel_data_list = [ d for d in channel_data if d['team'] == team_name ] channel_members_map = {} # type: Dict[str, List[str]] channel_admins_map = {} # type: Dict[str, List[str]] def initialize_stream_membership_dicts() -> None: for channel in channel_data: channel_name = channel["name"] channel_members_map[channel_name] = [] channel_admins_map[channel_name] = [] for username in user_data_map: user_dict = user_data_map[username] teams = user_dict["teams"] for team in teams: if team["name"] != team_name: continue for channel in team["channels"]: channel_roles = channel["roles"] channel_name = channel["name"] if "channel_admin" in channel_roles: channel_admins_map[channel_name].append(username) elif "channel_user" in channel_roles: channel_members_map[channel_name].append(username) def get_invite_only_value_from_channel_type(channel_type: str) -> bool: # Channel can have two types in Mattermost # "O" for a public channel. # "P" for a private channel. if channel_type == 'O': return False elif channel_type == 'P': return True else: # nocoverage raise Exception('unexpected value') streams = [] initialize_stream_membership_dicts() for channel_dict in channel_data_list: now = int(timezone_now().timestamp()) stream_id = stream_id_mapper.get(channel_dict['name']) stream_name = channel_dict["name"] invite_only = get_invite_only_value_from_channel_type(channel_dict['type']) stream = build_stream( date_created=now, realm_id=realm_id, name=channel_dict['display_name'], # Purpose describes how the channel should be used. It is similar to # stream description and is shown in channel list to help others decide # whether to join. # Header text always appears right next to channel name in channel header. # Can be used for advertising the purpose of stream, making announcements as # well as including frequently used links. So probably not a bad idea to use # this as description if the channel purpose is empty. description=channel_dict["purpose"] or channel_dict['header'], stream_id=stream_id, # Mattermost export don't include data of archived(~ deactivated) channels. deactivated=False, invite_only=invite_only, ) channel_users = set() for username in channel_admins_map[stream_name]: channel_users.add(user_id_mapper.get(username)) for username in channel_members_map[stream_name]: channel_users.add(user_id_mapper.get(username)) if channel_users: subscriber_handler.set_info( stream_id=stream_id, users=channel_users, ) streams.append(stream) return streams
def test_analytics_not_running(self) -> None: realm = get_realm("zulip") self.assertEqual(FillState.objects.count(), 0) realm.date_created = timezone_now() - timedelta(days=3) realm.save(update_fields=["date_created"]) with mock.patch('logging.warning'): result = self.client_get('/json/analytics/chart_data', {'chart_name': 'messages_sent_over_time'}) self.assert_json_error_contains(result, 'No analytics data available') realm.date_created = timezone_now() - timedelta(days=1, hours=2) realm.save(update_fields=["date_created"]) with mock.patch('logging.warning'): result = self.client_get('/json/analytics/chart_data', {'chart_name': 'messages_sent_over_time'}) self.assert_json_error_contains(result, 'No analytics data available') realm.date_created = timezone_now() - timedelta(days=1, minutes=10) realm.save(update_fields=["date_created"]) result = self.client_get('/json/analytics/chart_data', {'chart_name': 'messages_sent_over_time'}) self.assert_json_success(result) realm.date_created = timezone_now() - timedelta(hours=10) realm.save(update_fields=["date_created"]) result = self.client_get('/json/analytics/chart_data', {'chart_name': 'messages_sent_over_time'}) self.assert_json_success(result) end_time = timezone_now() - timedelta(days=5) fill_state = FillState.objects.create( property='messages_sent:is_bot:hour', end_time=end_time, state=FillState.DONE) realm.date_created = timezone_now() - timedelta(days=3) realm.save(update_fields=["date_created"]) with mock.patch('logging.warning'): result = self.client_get('/json/analytics/chart_data', {'chart_name': 'messages_sent_over_time'}) self.assert_json_error_contains(result, 'No analytics data available') realm.date_created = timezone_now() - timedelta(days=1, minutes=10) realm.save(update_fields=["date_created"]) result = self.client_get('/json/analytics/chart_data', {'chart_name': 'messages_sent_over_time'}) self.assert_json_success(result) end_time = timezone_now() - timedelta(days=2) fill_state.end_time = end_time fill_state.save(update_fields=["end_time"]) realm.date_created = timezone_now() - timedelta(days=3) realm.save(update_fields=["date_created"]) result = self.client_get('/json/analytics/chart_data', {'chart_name': 'messages_sent_over_time'}) self.assert_json_success(result) realm.date_created = timezone_now() - timedelta(days=1, hours=2) realm.save(update_fields=["date_created"]) with mock.patch('logging.warning'): result = self.client_get('/json/analytics/chart_data', {'chart_name': 'messages_sent_over_time'}) self.assert_json_error_contains(result, 'No analytics data available') realm.date_created = timezone_now() - timedelta(days=1, minutes=10) realm.save(update_fields=["date_created"]) result = self.client_get('/json/analytics/chart_data', {'chart_name': 'messages_sent_over_time'}) self.assert_json_success(result)
def maybe_send_batched_emails(self) -> None: with self.lock: # self.timer_event just triggered execution of this # function in a thread, so now that we hold the lock, we # clear the timer_event attribute to record that no Timer # is active. self.timer_event = None current_time = timezone_now() with transaction.atomic(): events_to_process = ScheduledMessageNotificationEmail.objects.filter( scheduled_timestamp__lte=current_time).select_related() # Batch the entries by user events_by_recipient: Dict[int, List[Dict[str, Any]]] = {} for event in events_to_process: entry = dict( user_profile_id=event.user_profile_id, message_id=event.message_id, trigger=event.trigger, mentioned_user_group_id=event.mentioned_user_group_id, ) if event.user_profile_id in events_by_recipient: events_by_recipient[event.user_profile_id].append( entry) else: events_by_recipient[event.user_profile_id] = [entry] for user_profile_id in events_by_recipient.keys(): events: List[Dict[ str, Any]] = events_by_recipient[user_profile_id] logging.info( "Batch-processing %s missedmessage_emails events for user %s", len(events), user_profile_id, ) try: # Because we process events in batches, an # escaped exception here would lead to # duplicate messages being sent for other # users in the same events_to_process batch, # and no guarantee of forward progress. handle_missedmessage_emails(user_profile_id, events) except Exception: logging.exception( "Failed to process %d missedmessage_emails for user %s", len(events), user_profile_id, stack_info=True, ) events_to_process.delete() # By only restarting the timer if there are actually events in # the queue, we ensure this queue processor is idle when there # are no missed-message emails to process. This avoids # constant CPU usage when there is no work to do. if ScheduledMessageNotificationEmail.objects.exists(): self.ensure_timer()
def estimate_recent_messages(realm: Realm, hours: int) -> int: stat = COUNT_STATS['messages_sent:is_bot:hour'] d = timezone_now() - datetime.timedelta(hours=hours) return RealmCount.objects.filter( property=stat.property, end_time__gt=d, realm=realm).aggregate( Sum('value'))['value__sum'] or 0
def test_convert_slack_workspace_messages( self, mock_get_messages_iterator: mock.Mock, mock_message: mock.Mock) -> None: output_dir = os.path.join(settings.TEST_WORKER_DIR, 'test-slack-import') os.makedirs(output_dir, exist_ok=True) added_channels = { 'random': ('c5', 1), 'general': ('c6', 2) } # type: Dict[str, Tuple[str, int]] time = float(timezone_now().timestamp()) zerver_message = [{'id': 1, 'ts': time}, {'id': 5, 'ts': time}] def fake_get_messages_iter( slack_data_dir: str, added_channels: AddedChannelsT, added_mpims: AddedMPIMsT, dm_members: DMMembersT) -> Iterator[ZerverFieldsT]: import copy return iter(copy.deepcopy(zerver_message)) realm = {'zerver_subscription': []} # type: Dict[str, Any] user_list = [] # type: List[Dict[str, Any]] reactions = [{"name": "grinning", "users": ["U061A5N1G"], "count": 1}] attachments = uploads = [] # type: List[Dict[str, Any]] zerver_usermessage = [{'id': 3}, {'id': 5}, {'id': 6}, {'id': 9}] mock_get_messages_iterator.side_effect = fake_get_messages_iter mock_message.side_effect = [[ zerver_message[:1], zerver_usermessage[:2], attachments, uploads, reactions[:1] ], [ zerver_message[1:2], zerver_usermessage[2:5], attachments, uploads, reactions[1:1] ]] # Hacky: We should include a zerver_userprofile, not the empty [] test_reactions, uploads, zerver_attachment = convert_slack_workspace_messages( './random_path', user_list, 2, {}, {}, added_channels, {}, {}, realm, [], [], 'domain', output_dir=output_dir, chunk_size=1) messages_file_1 = os.path.join(output_dir, 'messages-000001.json') self.assertTrue(os.path.exists(messages_file_1)) messages_file_2 = os.path.join(output_dir, 'messages-000002.json') self.assertTrue(os.path.exists(messages_file_2)) with open(messages_file_1) as f: message_json = ujson.load(f) self.assertEqual(message_json['zerver_message'], zerver_message[:1]) self.assertEqual(message_json['zerver_usermessage'], zerver_usermessage[:2]) with open(messages_file_2) as f: message_json = ujson.load(f) self.assertEqual(message_json['zerver_message'], zerver_message[1:2]) self.assertEqual(message_json['zerver_usermessage'], zerver_usermessage[2:5]) self.assertEqual(test_reactions, reactions)
def slack_workspace_to_realm(domain_name: str, realm_id: int, user_list: List[ZerverFieldsT], realm_subdomain: str, slack_data_dir: str, custom_emoji_list: ZerverFieldsT)-> Tuple[ZerverFieldsT, AddedUsersT, AddedRecipientsT, AddedChannelsT, List[ZerverFieldsT], ZerverFieldsT]: """ Returns: 1. realm, Converted Realm data 2. added_users, which is a dictionary to map from slack user id to zulip user id 3. added_recipient, which is a dictionary to map from channel name to zulip recipient_id 4. added_channels, which is a dictionary to map from channel name to channel id, zulip stream_id 5. avatars, which is list to map avatars to zulip avatar records.json 6. emoji_url_map, which is maps emoji name to its slack url """ NOW = float(timezone_now().timestamp()) zerver_realm = build_zerver_realm(realm_id, realm_subdomain, NOW) realm = dict(zerver_client=[{"name": "populate_db", "id": 1}, {"name": "website", "id": 2}, {"name": "API", "id": 3}], zerver_userpresence=[], # shows last logged in data, which is not available in slack zerver_userprofile_mirrordummy=[], zerver_realmdomain=[{"realm": realm_id, "allow_subdomains": False, "domain": domain_name, "id": realm_id}], zerver_useractivity=[], zerver_realm=zerver_realm, zerver_huddle=[], zerver_userprofile_crossrealm=[], zerver_useractivityinterval=[], zerver_realmfilter=[]) zerver_userprofile, avatars, added_users, zerver_customprofilefield, \ zerver_customprofilefield_value = users_to_zerver_userprofile(slack_data_dir, user_list, realm_id, int(NOW), domain_name) channels_to_zerver_stream_fields = channels_to_zerver_stream(slack_data_dir, realm_id, added_users, zerver_userprofile) zerver_realmemoji, emoji_url_map = build_realmemoji(custom_emoji_list, realm_id) realm['zerver_realmemoji'] = zerver_realmemoji # See https://zulipchat.com/help/set-default-streams-for-new-users # for documentation on zerver_defaultstream realm['zerver_userprofile'] = zerver_userprofile # Custom profile fields realm['zerver_customprofilefield'] = zerver_customprofilefield realm['zerver_customprofilefieldvalue'] = zerver_customprofilefield_value realm['zerver_defaultstream'] = channels_to_zerver_stream_fields[0] realm['zerver_stream'] = channels_to_zerver_stream_fields[1] realm['zerver_subscription'] = channels_to_zerver_stream_fields[3] realm['zerver_recipient'] = channels_to_zerver_stream_fields[4] added_channels = channels_to_zerver_stream_fields[2] added_recipient = channels_to_zerver_stream_fields[5] return realm, added_users, added_recipient, added_channels, avatars, emoji_url_map