示例#1
0
文件: presence.py 项目: gnprice/zulip
def update_active_status_backend(request: HttpRequest, user_profile: UserProfile,
                                 status: str=REQ(),
                                 ping_only: bool=REQ(validator=check_bool, default=False),
                                 new_user_input: bool=REQ(validator=check_bool, default=False)
                                 ) -> HttpResponse:
    status_val = UserPresence.status_from_string(status)
    if status_val is None:
        raise JsonableError(_("Invalid status: %s") % (status,))
    else:
        update_user_presence(user_profile, request.client, timezone_now(),
                             status_val, new_user_input)

    if ping_only:
        ret = {}  # type: Dict[str, Any]
    else:
        ret = get_status_list(user_profile)

    if user_profile.realm.is_zephyr_mirror_realm:
        # In zephyr mirroring realms, users can't see the presence of other
        # users, but each user **is** interested in whether their mirror bot
        # (running as their user) has been active.
        try:
            activity = UserActivity.objects.get(user_profile = user_profile,
                                                query="get_events_backend",
                                                client__name="zephyr_mirror")

            ret['zephyr_mirror_active'] = \
                (activity.last_visit > timezone_now() - datetime.timedelta(minutes=5))
        except UserActivity.DoesNotExist:
            ret['zephyr_mirror_active'] = False

    return json_success(ret)
示例#2
0
def update_messages_for_topic_edit(message: Message,
                                   propagate_mode: str,
                                   orig_topic_name: str,
                                   topic_name: str) -> List[Message]:
    propagate_query = Q(recipient = message.recipient, subject = orig_topic_name)
    # We only change messages up to 2 days in the past, to avoid hammering our
    # DB by changing an unbounded amount of messages
    if propagate_mode == 'change_all':
        before_bound = timezone_now() - datetime.timedelta(days=2)

        propagate_query = (propagate_query & ~Q(id = message.id) &
                           Q(pub_date__range=(before_bound, timezone_now())))
    if propagate_mode == 'change_later':
        propagate_query = propagate_query & Q(id__gt = message.id)

    messages = Message.objects.filter(propagate_query).select_related()

    # Evaluate the query before running the update
    messages_list = list(messages)
    messages.update(subject=topic_name)

    for m in messages_list:
        # The cached ORM object is not changed by messages.update()
        # and the remote cache update requires the new value
        m.set_topic_name(topic_name)

    return messages_list
    def handle(self, *args: Any, **options: Any) -> None:

        if settings.EMAIL_DELIVERER_DISABLED:
            # Here doing a check and sleeping indefinitely on this setting might
            # not sound right. Actually we do this check to avoid running this
            # process on every server that might be in service to a realm. See
            # the comment in zproject/settings.py file about renaming this setting.
            sleep_forever()

        with lockfile("/tmp/zulip_scheduled_message_deliverer.lockfile"):
            while True:
                messages_to_deliver = ScheduledMessage.objects.filter(
                    scheduled_timestamp__lte=timezone_now(),
                    delivered=False)
                if messages_to_deliver:
                    for message in messages_to_deliver:
                        with transaction.atomic():
                            do_send_messages([self.construct_message(message)])
                            message.delivered = True
                            message.save(update_fields=['delivered'])

                cur_time = timezone_now()
                time_next_min = (cur_time + timedelta(minutes=1)).replace(second=0, microsecond=0)
                sleep_time = (time_next_min - cur_time).total_seconds()
                time.sleep(sleep_time)
示例#4
0
 def test_expired_messages_in_each_realm(self):
     # type: () -> None
     # Check result realm messages order and result content
     # when all realm has expired messages.
     expired_mit_messages = self._make_mit_messages(3, timezone_now() - timedelta(days=101))
     self._make_mit_messages(4, timezone_now() - timedelta(days=50))
     zulip_messages_ids = Message.objects.order_by('id').filter(
         sender__realm=self.zulip_realm).values_list('id', flat=True)[3:10]
     expired_zulip_messages = self._change_messages_pub_date(zulip_messages_ids,
                                                             timezone_now() - timedelta(days=31))
     # Iterate by result
     expired_messages_result = [messages_list for messages_list in get_expired_messages()]
     self.assertEqual(len(expired_messages_result), 2)
     # Check mit.edu realm expired messages.
     self.assertEqual(len(expired_messages_result[0]['expired_messages']), 3)
     self.assertEqual(expired_messages_result[0]['realm_id'], self.mit_realm.id)
     # Check zulip.com realm expired messages.
     self.assertEqual(len(expired_messages_result[1]['expired_messages']), 7)
     self.assertEqual(expired_messages_result[1]['realm_id'], self.zulip_realm.id)
     # Compare expected messages ids with result messages ids.
     self.assertEqual(
         sorted([message.id for message in expired_mit_messages]),
         [message.id for message in expired_messages_result[0]['expired_messages']]
     )
     self.assertEqual(
         sorted([message.id for message in expired_zulip_messages]),
         [message.id for message in expired_messages_result[1]['expired_messages']]
     )
示例#5
0
    def filter(self, record: logging.LogRecord) -> bool:
        from django.conf import settings
        from django.core.cache import cache

        # Track duplicate errors
        duplicate = False
        rate = getattr(settings, '%s_LIMIT' % self.__class__.__name__.upper(),
                       600)  # seconds
        if rate > 0:
            # Test if the cache works
            try:
                cache.set('RLF_TEST_KEY', 1, 1)
                use_cache = cache.get('RLF_TEST_KEY') == 1
            except Exception:
                use_cache = False

            if use_cache:
                if record.exc_info is not None:
                    tb = '\n'.join(traceback.format_exception(*record.exc_info))
                else:
                    tb = str(record)
                key = self.__class__.__name__.upper() + hashlib.sha1(tb.encode()).hexdigest()
                duplicate = cache.get(key) == 1
                if not duplicate:
                    cache.set(key, 1, rate)
            else:
                min_date = timezone_now() - timedelta(seconds=rate)
                duplicate = (self.last_error >= min_date)
                if not duplicate:
                    self.last_error = timezone_now()

        return not duplicate
示例#6
0
文件: models.py 项目: cundi/BBS_
 def save(self, *args, **kwargs):
     if not self.pk:
         self.created = timezone_now()
     else:                              # 为了保证我们一直拥有创建数据,添加下面的条件语句
         if not self.created:
             self.created = timezone_now()
         self.modified = timezone_now()
     super(CreationModificationDateMixin, self).save(*args, **kwargs)
示例#7
0
    def save(self, *args, **kwargs):
        if not self.pk:
            self.created = timezone_now()
        else:
            # To ensure we have a creation date always we add this one
            if not self.created:
                self.created = timezone_now()

            self.modified = timezone_now()

        super(CreationModificationDateMixin, self).save(*args, **kwargs)
示例#8
0
    def test_multiple_stream_senders(self,
                                     mock_send_future_email: mock.MagicMock,
                                     mock_enough_traffic: mock.MagicMock) -> None:

        client = 'website'  # this makes `sent_by_human` return True

        othello = self.example_user('othello')
        self.subscribe(othello, 'Verona')

        one_day_ago = timezone_now() - datetime.timedelta(days=1)
        Message.objects.all().update(pub_date=one_day_ago)
        one_sec_ago = timezone_now() - datetime.timedelta(seconds=1)

        cutoff = time.mktime(one_sec_ago.timetuple())

        senders = ['hamlet', 'cordelia',  'iago', 'prospero', 'ZOE']
        for sender_name in senders:
            email = self.example_email(sender_name)
            self.login(email)
            content = 'some content for ' + email
            payload = dict(
                type='stream',
                client=client,
                to='Verona',
                topic='lunch',
                content=content,
            )
            result = self.client_post("/json/messages", payload)
            self.assert_json_success(result)

        flush_per_request_caches()
        with queries_captured() as queries:
            handle_digest_email(othello.id, cutoff)

        self.assertTrue(24 <= len(queries) <= 25)

        self.assertEqual(mock_send_future_email.call_count, 1)
        kwargs = mock_send_future_email.call_args[1]
        self.assertEqual(kwargs['to_user_ids'], [othello.id])

        hot_convo = kwargs['context']['hot_conversations'][0]

        expected_participants = {
            self.example_user(sender).full_name
            for sender in senders
        }

        self.assertEqual(set(hot_convo['participants']), expected_participants)
        self.assertEqual(hot_convo['count'], 5 - 2)  # 5 messages, but 2 shown
        teaser_messages = hot_convo['first_few_messages'][0]['senders']
        self.assertIn('some content', teaser_messages[0]['content'][0]['plain'])
        self.assertIn(teaser_messages[0]['sender'], expected_participants)
示例#9
0
文件: gitter.py 项目: gregmccoy/zulip
def gitter_workspace_to_realm(domain_name: str, gitter_data: GitterDataT,
                              realm_subdomain: str) -> Tuple[ZerverFieldsT,
                                                             List[ZerverFieldsT],
                                                             Dict[str, int]]:
    """
    Returns:
    1. realm, Converted Realm data
    2. avatars, which is list to map avatars to zulip avatar records.json
    3. user_map, which is a dictionary to map from gitter user id to zulip user id
    """
    NOW = float(timezone_now().timestamp())
    zerver_realm = build_zerver_realm(realm_id, realm_subdomain, NOW, 'Gitter')  # type: List[ZerverFieldsT]
    realm = build_realm(zerver_realm, realm_id, domain_name)

    zerver_userprofile, avatars, user_map = build_userprofile(int(NOW), domain_name, gitter_data)
    zerver_stream, zerver_defaultstream = build_stream_and_defaultstream(int(NOW))
    zerver_recipient, zerver_subscription = build_recipient_and_subscription(
        zerver_userprofile, zerver_stream)

    realm['zerver_userprofile'] = zerver_userprofile
    realm['zerver_stream'] = zerver_stream
    realm['zerver_defaultstream'] = zerver_defaultstream
    realm['zerver_recipient'] = zerver_recipient
    realm['zerver_subscription'] = zerver_subscription

    return realm, avatars, user_map
示例#10
0
 def test_disabled(self, mock_django_timezone: mock.MagicMock,
                   mock_queue_digest_recipient: mock.MagicMock) -> None:
     cutoff = timezone_now()
     # A Tuesday
     mock_django_timezone.return_value = datetime.datetime(year=2016, month=1, day=5)
     enqueue_emails(cutoff)
     mock_queue_digest_recipient.assert_not_called()
示例#11
0
    def compute_activity(self, user_activity_objects: QuerySet) -> None:
        # Report data from the past week.
        #
        # This is a rough report of client activity because we inconsistently
        # register activity from various clients; think of it as telling you
        # approximately how many people from a group have used a particular
        # client recently. For example, this might be useful to get a sense of
        # how popular different versions of a desktop client are.
        #
        # Importantly, this does NOT tell you anything about the relative
        # volumes of requests from clients.
        threshold = timezone_now() - datetime.timedelta(days=7)
        client_counts = user_activity_objects.filter(
            last_visit__gt=threshold).values("client__name").annotate(
            count=Count('client__name'))

        total = 0
        counts = []
        for client_type in client_counts:
            count = client_type["count"]
            client = client_type["client__name"]
            total += count
            counts.append((count, client))

        counts.sort()

        for count in counts:
            print("%25s %15d" % (count[1], count[0]))
        print("Total:", total)
示例#12
0
文件: presence.py 项目: gnprice/zulip
def get_presence_backend(request: HttpRequest, user_profile: UserProfile,
                         email: Text) -> HttpResponse:
    try:
        target = get_user(email, user_profile.realm)
    except UserProfile.DoesNotExist:
        return json_error(_('No such user'))
    if not target.is_active:
        return json_error(_('No such user'))
    if target.is_bot:
        return json_error(_('Presence is not supported for bot users.'))

    presence_dict = UserPresence.get_status_dict_by_user(target)
    if len(presence_dict) == 0:
        return json_error(_('No presence data for %s' % (target.email,)))

    # For initial version, we just include the status and timestamp keys
    result = dict(presence=presence_dict[target.email])
    aggregated_info = result['presence']['aggregated']
    aggr_status_duration = datetime_to_timestamp(timezone_now()) - aggregated_info['timestamp']
    if aggr_status_duration > settings.OFFLINE_THRESHOLD_SECS:
        aggregated_info['status'] = 'offline'
    for val in result['presence'].values():
        val.pop('client', None)
        val.pop('pushable', None)
    return json_success(result)
示例#13
0
def get_realm_expired_messages(realm: Any) -> Optional[Dict[str, Any]]:
    expired_date = timezone_now() - timedelta(days=realm.message_retention_days)
    expired_messages = Message.objects.order_by('id').filter(sender__realm=realm,
                                                             pub_date__lt=expired_date)
    if not expired_messages.exists():
        return None
    return {'realm_id': realm.id, 'expired_messages': expired_messages}
示例#14
0
def create_confirmation_link(obj: ContentType, host: str,
                             confirmation_type: int,
                             url_args: Optional[Dict[str, str]]=None) -> str:
    key = generate_key()
    Confirmation.objects.create(content_object=obj, date_sent=timezone_now(), confirmation_key=key,
                                realm=obj.realm, type=confirmation_type)
    return confirmation_url(key, host, confirmation_type, url_args)
示例#15
0
def delete_realm_user_sessions(realm):
    # type: (Realm) -> None
    realm_user_ids = [user_profile.id for user_profile in
                      UserProfile.objects.filter(realm=realm)]
    for session in Session.objects.filter(expire_date__gte=timezone_now()):
        if get_session_user(session) in realm_user_ids:
            delete_session(session)
示例#16
0
def check_key_is_valid(creation_key: Text) -> bool:
    if not RealmCreationKey.objects.filter(creation_key=creation_key).exists():
        return False
    time_elapsed = timezone_now() - RealmCreationKey.objects.get(creation_key=creation_key).date_created
    if time_elapsed.total_seconds() > settings.REALM_CREATION_LINK_VALIDITY_DAYS * 24 * 3600:
        return False
    return True
示例#17
0
def send_future_email(template_prefix: str, realm: Realm, to_user_id: Optional[int]=None,
                      to_email: Optional[str]=None, from_name: Optional[str]=None,
                      from_address: Optional[str]=None, context: Dict[str, Any]={},
                      delay: datetime.timedelta=datetime.timedelta(0)) -> None:
    template_name = template_prefix.split('/')[-1]
    email_fields = {'template_prefix': template_prefix, 'to_user_id': to_user_id, 'to_email': to_email,
                    'from_name': from_name, 'from_address': from_address, 'context': context}

    if settings.DEVELOPMENT and not settings.TEST_SUITE:
        send_email(template_prefix, to_user_id=to_user_id, to_email=to_email, from_name=from_name,
                   from_address=from_address, context=context)
        # For logging the email

    assert (to_user_id is None) ^ (to_email is None)
    if to_user_id is not None:
        # The realm is redundant if we have a to_user_id; this assert just
        # expresses that fact
        assert(UserProfile.objects.filter(id=to_user_id, realm=realm).exists())
        to_field = {'user_id': to_user_id}  # type: Dict[str, Any]
    else:
        to_field = {'address': parseaddr(to_email)[1]}

    ScheduledEmail.objects.create(
        type=EMAIL_TYPES[template_name],
        scheduled_timestamp=timezone_now() + delay,
        realm=realm,
        data=ujson.dumps(email_fields),
        **to_field)
示例#18
0
def generate_realm_creation_url() -> Text:
    key = generate_key()
    RealmCreationKey.objects.create(creation_key=key, date_created=timezone_now())
    return u'%s%s%s' % (settings.EXTERNAL_URI_SCHEME,
                        settings.EXTERNAL_HOST,
                        reverse('zerver.views.create_realm',
                                kwargs={'creation_key': key}))
示例#19
0
    def test_show_billing(self) -> None:
        customer = Customer.objects.create(realm=get_realm("zulip"), stripe_customer_id="cus_id")

        # realm admin, but no CustomerPlan -> no billing link
        user = self.example_user('iago')
        self.login(user.email)
        result_html = self._get_home_page().content.decode('utf-8')
        self.assertNotIn('Billing', result_html)

        # realm admin, with inactive CustomerPlan -> show billing link
        CustomerPlan.objects.create(customer=customer, billing_cycle_anchor=timezone_now(),
                                    billing_schedule=CustomerPlan.ANNUAL, next_invoice_date=timezone_now(),
                                    tier=CustomerPlan.STANDARD, status=CustomerPlan.ENDED)
        result_html = self._get_home_page().content.decode('utf-8')
        self.assertIn('Billing', result_html)

        # billing admin, with CustomerPlan -> show billing link
        user.is_realm_admin = False
        user.is_billing_admin = True
        user.save(update_fields=['is_realm_admin', 'is_billing_admin'])
        result_html = self._get_home_page().content.decode('utf-8')
        self.assertIn('Billing', result_html)

        # billing admin, but no CustomerPlan -> no billing link
        CustomerPlan.objects.all().delete()
        result_html = self._get_home_page().content.decode('utf-8')
        self.assertNotIn('Billing', result_html)

        # billing admin, no customer object -> make sure it doesn't crash
        customer.delete()
        result = self._get_home_page()
        self.assertEqual(result.status_code, 200)
示例#20
0
def create_user_profile(realm: Realm, email: str, password: Optional[str],
                        active: bool, bot_type: Optional[int], full_name: str,
                        short_name: str, bot_owner: Optional[UserProfile],
                        is_mirror_dummy: bool, tos_version: Optional[str],
                        timezone: Optional[str],
                        tutorial_status: Optional[str] = UserProfile.TUTORIAL_WAITING,
                        enter_sends: bool = False) -> UserProfile:
    now = timezone_now()
    email = UserManager.normalize_email(email)

    user_profile = UserProfile(is_staff=False, is_active=active,
                               full_name=full_name, short_name=short_name,
                               last_login=now, date_joined=now, realm=realm,
                               pointer=-1, is_bot=bool(bot_type), bot_type=bot_type,
                               bot_owner=bot_owner, is_mirror_dummy=is_mirror_dummy,
                               tos_version=tos_version, timezone=timezone,
                               tutorial_status=tutorial_status,
                               enter_sends=enter_sends,
                               onboarding_steps=ujson.dumps([]),
                               default_language=realm.default_language,
                               twenty_four_hour_time=realm.default_twenty_four_hour_time,
                               delivery_email=email)

    if bot_type or not active:
        password = None
    if realm.email_address_visibility == Realm.EMAIL_ADDRESS_VISIBILITY_EVERYONE:
        # If emails are visible to everyone, we can set this here and save a DB query
        user_profile.email = get_display_email_address(user_profile, realm)
    user_profile.set_password(password)
    user_profile.api_key = generate_api_key()
    return user_profile
示例#21
0
 def test_regenerate_api_key(self) -> None:
     now = timezone_now()
     user = self.example_user('hamlet')
     do_regenerate_api_key(user, user)
     self.assertEqual(RealmAuditLog.objects.filter(event_type=RealmAuditLog.USER_API_KEY_CHANGED,
                                                   event_time__gte=now).count(), 1)
     self.assertTrue(user.api_key)
示例#22
0
def receiver_is_idle(user_profile_id, realm_presences):
    # type: (int, Optional[Dict[int, Dict[Text, Dict[str, Any]]]]) -> bool
    # If a user has no message-receiving event queues, they've got no open zulip
    # session so we notify them
    all_client_descriptors = get_client_descriptors_for_user(user_profile_id)
    message_event_queues = [client for client in all_client_descriptors if client.accepts_messages()]
    off_zulip = len(message_event_queues) == 0

    # It's possible a recipient is not in the realm of a sender. We don't have
    # presence information in this case (and it's hard to get without an additional
    # db query) so we simply don't try to guess if this cross-realm recipient
    # has been idle for too long
    if realm_presences is None or user_profile_id not in realm_presences:
        return off_zulip

    # We want to find the newest "active" presence entity and compare that to the
    # activity expiry threshold.
    user_presence = realm_presences[user_profile_id]
    latest_active_timestamp = None
    idle = False

    for client, status in six.iteritems(user_presence):
        if (latest_active_timestamp is None or status['timestamp'] > latest_active_timestamp) and \
                status['status'] == 'active':
            latest_active_timestamp = status['timestamp']

    if latest_active_timestamp is None:
        idle = True
    else:
        active_datetime = timestamp_to_datetime(latest_active_timestamp)
        # 140 seconds is consistent with presence.js:OFFLINE_THRESHOLD_SECS
        idle = timezone_now() - active_datetime > datetime.timedelta(seconds=140)

    return off_zulip or idle
示例#23
0
    def save(self):
        user = self._user
        if user is None:
            raise RuntimeError("ResponseForm without user, can't be saved.")
        logger.debug("Saving response data to database and requesing doing update to submission_url")

        # Get the instance and update metadata
        instance = super().save(commit=False)
        instance.response_time = timezone_now()
        instance.response_by = user
        instance.response_notify = self.get_notify()

        # prepare for upload
        upload = async_response_upload(instance) if self.has_changed() else None

        # save to db and update form internal state
        fields = self._meta.fields + ('response_time', 'response_by', 'response_notify')
        instance.save(update_fields=fields)
        self.original_fields(instance, update=True)

        # if there is upload, do it now after instance is saved
        if upload:
            logger.debug("Instance has changes, requesting upload to aplus.")
            upload()
        else:
            logger.debug("No changes to response, so no aplus updated needed.")

        return instance
示例#24
0
def email_on_new_login(sender: Any, user: UserProfile, request: Any, **kwargs: Any) -> None:
    # We import here to minimize the dependencies of this module,
    # since it runs as part of `manage.py` initialization
    from zerver.context_processors import common_context

    if not settings.SEND_LOGIN_EMAILS:
        return

    if request:
        # If the user's account was just created, avoid sending an email.
        if getattr(user, "just_registered", False):
            return

        user_agent = request.META.get('HTTP_USER_AGENT', "").lower()

        context = common_context(user)
        context['user_email'] = user.email
        user_tz = user.timezone
        if user_tz == '':
            user_tz = timezone_get_current_timezone_name()
        local_time = timezone_now().astimezone(get_timezone(user_tz))
        context['login_time'] = local_time.strftime('%A, %B %d, %Y at %I:%M%p ') + user_tz
        context['device_ip'] = request.META.get('REMOTE_ADDR') or _("Unknown IP address")
        context['device_os'] = get_device_os(user_agent)
        context['device_browser'] = get_device_browser(user_agent)

        email_dict = {
            'template_prefix': 'zerver/emails/notify_new_login',
            'to_user_id': user.id,
            'from_name': 'Zulip Account Security',
            'from_address': FromAddress.NOREPLY,
            'context': context}
        queue_json_publish("email_senders", email_dict)
示例#25
0
    def test_delete_old_unclaimed_attachments(self) -> None:
        # Upload some files and make them older than a weeek
        self.login(self.example_email("hamlet"))
        d1 = StringIO("zulip!")
        d1.name = "dummy_1.txt"
        result = self.client_post("/json/user_uploads", {'file': d1})
        d1_path_id = re.sub('/user_uploads/', '', result.json()['uri'])

        d2 = StringIO("zulip!")
        d2.name = "dummy_2.txt"
        result = self.client_post("/json/user_uploads", {'file': d2})
        d2_path_id = re.sub('/user_uploads/', '', result.json()['uri'])

        two_week_ago = timezone_now() - datetime.timedelta(weeks=2)
        d1_attachment = Attachment.objects.get(path_id = d1_path_id)
        d1_attachment.create_time = two_week_ago
        d1_attachment.save()
        self.assertEqual(str(d1_attachment), u'<Attachment: dummy_1.txt>')
        d2_attachment = Attachment.objects.get(path_id = d2_path_id)
        d2_attachment.create_time = two_week_ago
        d2_attachment.save()

        # Send message referring only dummy_1
        self.subscribe(self.example_user("hamlet"), "Denmark")
        body = "Some files here ...[zulip.txt](http://localhost:9991/user_uploads/" + d1_path_id + ")"
        self.send_stream_message(self.example_email("hamlet"), "Denmark", body, "test")

        # dummy_2 should not exist in database or the uploads folder
        do_delete_old_unclaimed_attachments(2)
        self.assertTrue(not Attachment.objects.filter(path_id = d2_path_id).exists())
        self.assertTrue(not delete_message_image(d2_path_id))
示例#26
0
def create_user_profile(realm, email, password, active, bot_type, full_name,
                        short_name, bot_owner, is_mirror_dummy, tos_version,
                        timezone, tutorial_status=UserProfile.TUTORIAL_WAITING,
                        enter_sends=False):
    # type: (Realm, Text, Optional[Text], bool, Optional[int], Text, Text, Optional[UserProfile], bool, Text, Optional[Text], Optional[Text], bool) -> UserProfile
    now = timezone_now()
    email = UserManager.normalize_email(email)

    user_profile = UserProfile(email=email, is_staff=False, is_active=active,
                               full_name=full_name, short_name=short_name,
                               last_login=now, date_joined=now, realm=realm,
                               pointer=-1, is_bot=bool(bot_type), bot_type=bot_type,
                               bot_owner=bot_owner, is_mirror_dummy=is_mirror_dummy,
                               tos_version=tos_version, timezone=timezone,
                               tutorial_status=tutorial_status,
                               enter_sends=enter_sends,
                               onboarding_steps=ujson.dumps([]),
                               default_language=realm.default_language)

    if bot_type or not active:
        password = None

    user_profile.set_password(password)

    user_profile.api_key = random_api_key()
    return user_profile
示例#27
0
def create_user_profile(realm: Realm, email: str, password: Optional[str],
                        active: bool, bot_type: Optional[int], full_name: str,
                        short_name: str, bot_owner: Optional[UserProfile],
                        is_mirror_dummy: bool, tos_version: Optional[str],
                        timezone: Optional[str],
                        tutorial_status: Optional[str] = UserProfile.TUTORIAL_WAITING,
                        enter_sends: bool = False) -> UserProfile:
    now = timezone_now()
    email = UserManager.normalize_email(email)

    user_profile = UserProfile(email=email, is_staff=False, is_active=active,
                               full_name=full_name, short_name=short_name,
                               last_login=now, date_joined=now, realm=realm,
                               pointer=-1, is_bot=bool(bot_type), bot_type=bot_type,
                               bot_owner=bot_owner, is_mirror_dummy=is_mirror_dummy,
                               tos_version=tos_version, timezone=timezone,
                               tutorial_status=tutorial_status,
                               enter_sends=enter_sends,
                               onboarding_steps=ujson.dumps([]),
                               default_language=realm.default_language,
                               twenty_four_hour_time=realm.default_twenty_four_hour_time,
                               delivery_email=email)

    if bot_type or not active:
        password = None

    user_profile.set_password(password)

    user_profile.api_key = generate_api_key()
    return user_profile
示例#28
0
def subscribe_users_to_streams(realm: Realm, stream_dict: Dict[str, Dict[str, Any]]) -> None:
    subscriptions_to_add = []
    event_time = timezone_now()
    all_subscription_logs = []
    profiles = UserProfile.objects.select_related().filter(realm=realm)
    for i, stream_name in enumerate(stream_dict):
        stream = Stream.objects.get(name=stream_name, realm=realm)
        recipient = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id)
        for profile in profiles:
            # Subscribe to some streams.
            s = Subscription(
                recipient=recipient,
                user_profile=profile,
                color=STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)])
            subscriptions_to_add.append(s)

            log = RealmAuditLog(realm=profile.realm,
                                modified_user=profile,
                                modified_stream=stream,
                                event_last_message_id=0,
                                event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
                                event_time=event_time)
            all_subscription_logs.append(log)
    Subscription.objects.bulk_create(subscriptions_to_add)
    RealmAuditLog.objects.bulk_create(all_subscription_logs)
示例#29
0
    def test_get_users_for_soft_deactivation(self) -> None:
        users = [
            self.example_user('hamlet'),
            self.example_user('iago'),
            self.example_user('cordelia'),
            self.example_user('ZOE'),
            self.example_user('othello'),
            self.example_user('prospero'),
            self.example_user('aaron'),
        ]
        client, _ = Client.objects.get_or_create(name='website')
        query = '/json/users/me/pointer'
        last_visit = timezone_now()
        count = 150
        for user_profile in UserProfile.objects.all():
            UserActivity.objects.get_or_create(
                user_profile=user_profile,
                client=client,
                query=query,
                count=count,
                last_visit=last_visit
            )
        filter_kwargs = dict(user_profile__realm=get_realm('zulip'))
        users_to_deactivate = get_users_for_soft_deactivation(-1, filter_kwargs)

        self.assert_length(users_to_deactivate, 7)
        for user in users_to_deactivate:
            self.assertTrue(user in users)
示例#30
0
    def get_mirror_user(self,
                        realm_id: int,
                        name: str) -> Dict[str, Any]:
        if name in self.name_to_mirror_user_map:
            user = self.name_to_mirror_user_map[name]
            return user

        user_id = self._new_mirror_user_id()
        short_name = name
        full_name = name
        email = 'mirror-{user_id}@example.com'.format(user_id=user_id)
        delivery_email = email
        avatar_source = 'G'
        date_joined = int(timezone_now().timestamp())
        timezone = 'UTC'

        user = build_user_profile(
            avatar_source=avatar_source,
            date_joined=date_joined,
            delivery_email=delivery_email,
            email=email,
            full_name=full_name,
            id=user_id,
            is_active=False,
            is_realm_admin=False,
            is_guest=False,
            is_mirror_dummy=True,
            realm_id=realm_id,
            short_name=short_name,
            timezone=timezone,
        )

        self.name_to_mirror_user_map[name] = user
        return user
示例#31
0
def check_update_message(
    user_profile: UserProfile,
    message_id: int,
    stream_id: Optional[int] = None,
    topic_name: Optional[str] = None,
    propagate_mode: str = "change_one",
    send_notification_to_old_thread: bool = True,
    send_notification_to_new_thread: bool = True,
    content: Optional[str] = None,
) -> int:
    """This will update a message given the message id and user profile.
    It checks whether the user profile has the permission to edit the message
    and raises a JsonableError if otherwise.
    It returns the number changed.
    """
    message, ignored_user_message = access_message(user_profile, message_id)

    if not user_profile.realm.allow_message_editing:
        raise JsonableError(
            _("Your organization has turned off message editing"))

    # The zerver/views/message_edit.py call point already strips this
    # via REQ_topic; so we can delete this line if we arrange a
    # contract where future callers in the embedded bots system strip
    # use REQ_topic as well (or otherwise are guaranteed to strip input).
    if topic_name is not None:
        topic_name = topic_name.strip()
        if topic_name == message.topic_name():
            topic_name = None

    validate_message_edit_payload(message, stream_id, topic_name,
                                  propagate_mode, content)

    is_no_topic_msg = message.topic_name() == "(no topic)"

    if content is not None or topic_name is not None:
        if not can_edit_content_or_topic(message, user_profile,
                                         is_no_topic_msg, content, topic_name):
            raise JsonableError(
                _("You don't have permission to edit this message"))

    # If there is a change to the content, check that it hasn't been too long
    # Allow an extra 20 seconds since we potentially allow editing 15 seconds
    # past the limit, and in case there are network issues, etc. The 15 comes
    # from (min_seconds_to_edit + seconds_left_buffer) in message_edit.js; if
    # you change this value also change those two parameters in message_edit.js.
    edit_limit_buffer = 20
    if content is not None and user_profile.realm.message_content_edit_limit_seconds > 0:
        deadline_seconds = user_profile.realm.message_content_edit_limit_seconds + edit_limit_buffer
        if (timezone_now() - message.date_sent) > datetime.timedelta(
                seconds=deadline_seconds):
            raise JsonableError(
                _("The time limit for editing this message has passed"))

    # If there is a change to the topic, check that the user is allowed to
    # edit it and that it has not been too long. If this is not the user who
    # sent the message, they are not the admin, and the time limit for editing
    # topics is passed, raise an error.
    if (topic_name is not None and message.sender != user_profile
            and not user_profile.is_realm_admin
            and not user_profile.is_moderator and not is_no_topic_msg):
        deadline_seconds = Realm.DEFAULT_COMMUNITY_TOPIC_EDITING_LIMIT_SECONDS + edit_limit_buffer
        if (timezone_now() - message.date_sent) > datetime.timedelta(
                seconds=deadline_seconds):
            raise JsonableError(
                _("The time limit for editing this message's topic has passed")
            )

    rendering_result = None
    links_for_embed: Set[str] = set()
    prior_mention_user_ids: Set[int] = set()
    mention_data: Optional[MentionData] = None
    if content is not None:
        if content.rstrip() == "":
            content = "(deleted)"
        content = normalize_body(content)

        mention_backend = MentionBackend(user_profile.realm_id)
        mention_data = MentionData(
            mention_backend=mention_backend,
            content=content,
        )
        prior_mention_user_ids = get_mentions_for_message_updates(message.id)

        # We render the message using the current user's realm; since
        # the cross-realm bots never edit messages, this should be
        # always correct.
        # Note: If rendering fails, the called code will raise a JsonableError.
        rendering_result = render_incoming_message(
            message,
            content,
            user_profile.realm,
            mention_data=mention_data,
        )
        links_for_embed |= rendering_result.links_for_preview

        if message.is_stream_message() and rendering_result.mentions_wildcard:
            stream = access_stream_by_id(user_profile,
                                         message.recipient.type_id)[0]
            if not wildcard_mention_allowed(message.sender, stream):
                raise JsonableError(
                    _("You do not have permission to use wildcard mentions in this stream."
                      ))

    new_stream = None
    number_changed = 0

    if stream_id is not None:
        assert message.is_stream_message()
        if not user_profile.can_move_messages_between_streams():
            raise JsonableError(
                _("You don't have permission to move this message"))
        try:
            access_stream_by_id(user_profile, message.recipient.type_id)
        except JsonableError:
            raise JsonableError(
                _("You don't have permission to move this message due to missing access to its stream"
                  ))

        new_stream = access_stream_by_id(user_profile,
                                         stream_id,
                                         require_active=True)[0]
        check_stream_access_based_on_stream_post_policy(
            user_profile, new_stream)

    number_changed = do_update_message(
        user_profile,
        message,
        new_stream,
        topic_name,
        propagate_mode,
        send_notification_to_old_thread,
        send_notification_to_new_thread,
        content,
        rendering_result,
        prior_mention_user_ids,
        mention_data,
    )

    if links_for_embed:
        event_data = {
            "message_id": message.id,
            "message_content": message.content,
            # The choice of `user_profile.realm_id` rather than
            # `sender.realm_id` must match the decision made in the
            # `render_incoming_message` call earlier in this function.
            "message_realm_id": user_profile.realm_id,
            "urls": list(links_for_embed),
        }
        queue_json_publish("embed_links", event_data)

    return number_changed
示例#32
0
def send_messages(
    data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
                Callable[[str], Any], int]
) -> int:
    (tot_messages, personals_pairs, options, output, random_seed) = data
    random.seed(random_seed)

    with open("var/test_messages.json", "r") as infile:
        dialog = ujson.load(infile)
    random.shuffle(dialog)
    texts = itertools.cycle(dialog)

    recipient_streams = [
        klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
    ]  # type: List[int]
    recipient_huddles = [
        h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
    ]  # type: List[int]

    huddle_members = {}  # type: Dict[int, List[int]]
    for h in recipient_huddles:
        huddle_members[h] = [
            s.user_profile.id
            for s in Subscription.objects.filter(recipient_id=h)
        ]

    num_messages = 0
    random_max = 1000000
    recipients = {}  # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
    while num_messages < tot_messages:
        saved_data = {}  # type: Dict[str, Any]
        message = Message()
        message.sending_client = get_client('populate_db')

        message.content = next(texts)

        randkey = random.randint(1, random_max)
        if (num_messages > 0
                and random.randint(1, random_max) * 100. / random_max <
                options["stickyness"]):
            # Use an old recipient
            message_type, recipient_id, saved_data = recipients[num_messages -
                                                                1]
            if message_type == Recipient.PERSONAL:
                personals_pair = saved_data['personals_pair']
                random.shuffle(personals_pair)
            elif message_type == Recipient.STREAM:
                message.subject = saved_data['subject']
                message.recipient = get_recipient_by_id(recipient_id)
            elif message_type == Recipient.HUDDLE:
                message.recipient = get_recipient_by_id(recipient_id)
        elif (randkey <= random_max * options["percent_huddles"] / 100.):
            message_type = Recipient.HUDDLE
            message.recipient = get_recipient_by_id(
                random.choice(recipient_huddles))
        elif (randkey <= random_max *
              (options["percent_huddles"] + options["percent_personals"]) /
              100.):
            message_type = Recipient.PERSONAL
            personals_pair = random.choice(personals_pairs)
            random.shuffle(personals_pair)
        elif (randkey <= random_max * 1.0):
            message_type = Recipient.STREAM
            message.recipient = get_recipient_by_id(
                random.choice(recipient_streams))

        if message_type == Recipient.HUDDLE:
            sender_id = random.choice(huddle_members[message.recipient.id])
            message.sender = get_user_profile_by_id(sender_id)
        elif message_type == Recipient.PERSONAL:
            message.recipient = Recipient.objects.get(
                type=Recipient.PERSONAL, type_id=personals_pair[0])
            message.sender = get_user_profile_by_id(personals_pair[1])
            saved_data['personals_pair'] = personals_pair
        elif message_type == Recipient.STREAM:
            stream = Stream.objects.get(id=message.recipient.type_id)
            # Pick a random subscriber to the stream
            message.sender = random.choice(
                Subscription.objects.filter(
                    recipient=message.recipient)).user_profile
            message.subject = stream.name + Text(random.randint(1, 3))
            saved_data['subject'] = message.subject

        # Spoofing time not supported with threading
        if options['threads'] != 1:
            message.pub_date = timezone_now()
        else:
            # Distrubutes 80% of messages starting from 5 days ago, over a period
            # of 3 days. Then, distributes remaining messages over past 24 hours.
            spoofed_date = timezone_now() - timezone_timedelta(days=5)
            if (num_messages < tot_messages * 0.8):
                # Maximum of 3 days ahead, convert to minutes
                time_ahead = 3 * 24 * 60
                time_ahead //= int(tot_messages * 0.8)
            else:
                time_ahead = 24 * 60
                time_ahead //= int(tot_messages * 0.2)

            spoofed_minute = random.randint(time_ahead * num_messages,
                                            time_ahead * (num_messages + 1))
            spoofed_date += timezone_timedelta(minutes=spoofed_minute)
            message.pub_date = spoofed_date

        # We disable USING_RABBITMQ here, so that deferred work is
        # executed in do_send_message_messages, rather than being
        # queued.  This is important, because otherwise, if run-dev.py
        # wasn't running when populate_db was run, a developer can end
        # up with queued events that reference objects from a previous
        # life of the database, which naturally throws exceptions.
        settings.USING_RABBITMQ = False
        do_send_messages([{'message': message}])
        settings.USING_RABBITMQ = True

        recipients[num_messages] = (message_type, message.recipient.id,
                                    saved_data)
        num_messages += 1
    return tot_messages
示例#33
0
def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile,
                                                missed_messages: List[Dict[
                                                    str, Any]],
                                                message_count: int) -> None:
    """
    Send a reminder email to a user if she's missed some PMs by being offline.

    The email will have its reply to address set to a limited used email
    address that will send a zulip message to the correct recipient. This
    allows the user to respond to missed PMs, huddles, and @-mentions directly
    from the email.

    `user_profile` is the user to send the reminder to
    `missed_messages` is a list of dictionaries to Message objects and other data
                      for a group of messages that share a recipient (and topic)
    """
    from zerver.context_processors import common_context
    # Disabled missedmessage emails internally
    if not user_profile.enable_offline_email_notifications:
        return

    recipients = set((msg['message'].recipient_id, msg['message'].topic_name())
                     for msg in missed_messages)
    if len(recipients) != 1:
        raise ValueError(
            'All missed_messages must have the same recipient and topic %r' %
            (recipients, ))

    # This link is no longer a part of the email, but keeping the code in case
    # we find a clean way to add it back in the future
    unsubscribe_link = one_click_unsubscribe_link(user_profile,
                                                  "missed_messages")
    context = common_context(user_profile)
    context.update({
        'name':
        user_profile.full_name,
        'message_count':
        message_count,
        'unsubscribe_link':
        unsubscribe_link,
        'realm_name_in_notifications':
        user_profile.realm_name_in_notifications,
        'show_message_content':
        message_content_allowed_in_missedmessage_emails(user_profile)
    })

    triggers = list(message['trigger'] for message in missed_messages)
    unique_triggers = set(triggers)
    context.update({
        'mention':
        'mentioned' in unique_triggers
        or 'wildcard_mentioned' in unique_triggers,
        'stream_email_notify':
        'stream_email_notify' in unique_triggers,
        'mention_count':
        triggers.count('mentioned') + triggers.count("wildcard_mentioned"),
    })

    # If this setting (email mirroring integration) is enabled, only then
    # can users reply to email to send message to Zulip. Thus, one must
    # ensure to display warning in the template.
    if settings.EMAIL_GATEWAY_PATTERN:
        context.update({
            'reply_to_zulip': True,
        })
    else:
        context.update({
            'reply_to_zulip': False,
        })

    from zerver.lib.email_mirror import create_missed_message_address
    reply_to_address = create_missed_message_address(
        user_profile, missed_messages[0]['message'])
    if reply_to_address == FromAddress.NOREPLY:
        reply_to_name = None
    else:
        reply_to_name = "Zulip"

    narrow_url = get_narrow_url(user_profile, missed_messages[0]['message'])
    context.update({
        'narrow_url': narrow_url,
    })

    senders = list(set(m['message'].sender for m in missed_messages))
    if (missed_messages[0]['message'].recipient.type == Recipient.HUDDLE):
        display_recipient = get_display_recipient(
            missed_messages[0]['message'].recipient)
        # Make sure that this is a list of strings, not a string.
        assert not isinstance(display_recipient, str)
        other_recipients = [
            r['full_name'] for r in display_recipient
            if r['id'] != user_profile.id
        ]
        context.update({'group_pm': True})
        if len(other_recipients) == 2:
            huddle_display_name = " and ".join(other_recipients)
            context.update({'huddle_display_name': huddle_display_name})
        elif len(other_recipients) == 3:
            huddle_display_name = "%s, %s, and %s" % (
                other_recipients[0], other_recipients[1], other_recipients[2])
            context.update({'huddle_display_name': huddle_display_name})
        else:
            huddle_display_name = "%s, and %s others" % (', '.join(
                other_recipients[:2]), len(other_recipients) - 2)
            context.update({'huddle_display_name': huddle_display_name})
    elif (missed_messages[0]['message'].recipient.type == Recipient.PERSONAL):
        context.update({'private_message': True})
    elif (context['mention'] or context['stream_email_notify']):
        # Keep only the senders who actually mentioned the user
        if context['mention']:
            senders = list(
                set(m['message'].sender for m in missed_messages
                    if m['trigger'] == 'mentioned'
                    or m['trigger'] == 'wildcard_mentioned'))
        message = missed_messages[0]['message']
        stream = Stream.objects.only('id',
                                     'name').get(id=message.recipient.type_id)
        stream_header = "%s > %s" % (stream.name, message.topic_name())
        context.update({
            'stream_header': stream_header,
        })
    else:
        raise AssertionError("Invalid messages!")

    # If message content is disabled, then flush all information we pass to email.
    if not message_content_allowed_in_missedmessage_emails(user_profile):
        context.update({
            'reply_to_zulip': False,
            'messages': [],
            'sender_str': "",
            'realm_str': user_profile.realm.name,
            'huddle_display_name': "",
        })
    else:
        context.update({
            'messages':
            build_message_list(user_profile,
                               list(m['message'] for m in missed_messages)),
            'sender_str':
            ", ".join(sender.full_name for sender in senders),
            'realm_str':
            user_profile.realm.name,
        })

    from_name = "Zulip missed messages"  # type: str
    from_address = FromAddress.NOREPLY
    if len(senders) == 1 and settings.SEND_MISSED_MESSAGE_EMAILS_AS_USER:
        # If this setting is enabled, you can reply to the Zulip
        # missed message emails directly back to the original sender.
        # However, one must ensure the Zulip server is in the SPF
        # record for the domain, or there will be spam/deliverability
        # problems.
        sender = senders[0]
        from_name, from_address = (sender.full_name, sender.email)
        context.update({
            'reply_to_zulip': False,
        })

    email_dict = {
        'template_prefix': 'zerver/emails/missed_message',
        'to_user_ids': [user_profile.id],
        'from_name': from_name,
        'from_address': from_address,
        'reply_to_email': formataddr((reply_to_name, reply_to_address)),
        'context': context
    }
    queue_json_publish("email_senders", email_dict)

    user_profile.last_reminder = timezone_now()
    user_profile.save(update_fields=['last_reminder'])
示例#34
0
    def test_new_user_input(self, unused_mock: mock.Mock) -> None:
        """Mostly a test for UserActivityInterval"""
        user_profile = self.example_user("hamlet")
        self.login('hamlet')
        self.assertEqual(
            UserActivityInterval.objects.filter(
                user_profile=user_profile).count(), 0)
        time_zero = timezone_now().replace(microsecond=0)
        with mock.patch('zerver.views.presence.timezone_now',
                        return_value=time_zero):
            result = self.client_post("/json/users/me/presence", {
                'status': 'active',
                'new_user_input': 'true'
            })
        self.assert_json_success(result)
        self.assertEqual(
            UserActivityInterval.objects.filter(
                user_profile=user_profile).count(), 1)
        interval = UserActivityInterval.objects.get(user_profile=user_profile)
        self.assertEqual(interval.start, time_zero)
        self.assertEqual(interval.end,
                         time_zero + UserActivityInterval.MIN_INTERVAL_LENGTH)

        second_time = time_zero + timedelta(seconds=600)
        # Extent the interval
        with mock.patch('zerver.views.presence.timezone_now',
                        return_value=second_time):
            result = self.client_post("/json/users/me/presence", {
                'status': 'active',
                'new_user_input': 'true'
            })
        self.assert_json_success(result)
        self.assertEqual(
            UserActivityInterval.objects.filter(
                user_profile=user_profile).count(), 1)
        interval = UserActivityInterval.objects.get(user_profile=user_profile)
        self.assertEqual(interval.start, time_zero)
        self.assertEqual(
            interval.end,
            second_time + UserActivityInterval.MIN_INTERVAL_LENGTH)

        third_time = time_zero + timedelta(seconds=6000)
        with mock.patch('zerver.views.presence.timezone_now',
                        return_value=third_time):
            result = self.client_post("/json/users/me/presence", {
                'status': 'active',
                'new_user_input': 'true'
            })
        self.assert_json_success(result)
        self.assertEqual(
            UserActivityInterval.objects.filter(
                user_profile=user_profile).count(), 2)
        interval = UserActivityInterval.objects.filter(
            user_profile=user_profile).order_by('start')[0]
        self.assertEqual(interval.start, time_zero)
        self.assertEqual(
            interval.end,
            second_time + UserActivityInterval.MIN_INTERVAL_LENGTH)
        interval = UserActivityInterval.objects.filter(
            user_profile=user_profile).order_by('start')[1]
        self.assertEqual(interval.start, third_time)
        self.assertEqual(interval.end,
                         third_time + UserActivityInterval.MIN_INTERVAL_LENGTH)

        self.assertEqual(
            seconds_usage_between(user_profile, time_zero,
                                  third_time).total_seconds(), 1500)
        self.assertEqual(
            seconds_usage_between(user_profile, time_zero, third_time +
                                  timedelta(seconds=10)).total_seconds(), 1510)
        self.assertEqual(
            seconds_usage_between(user_profile, time_zero, third_time +
                                  timedelta(seconds=1000)).total_seconds(),
            2400)
        self.assertEqual(
            seconds_usage_between(user_profile, time_zero, third_time -
                                  timedelta(seconds=100)).total_seconds(),
            1500)
        self.assertEqual(
            seconds_usage_between(
                user_profile, time_zero + timedelta(seconds=100),
                third_time - timedelta(seconds=100)).total_seconds(), 1400)
        self.assertEqual(
            seconds_usage_between(
                user_profile, time_zero + timedelta(seconds=1200),
                third_time - timedelta(seconds=100)).total_seconds(), 300)

        # Now test /activity with actual data
        user_profile.is_staff = True
        user_profile.save()
        result = self.client_get('/activity')
        self.assertEqual(result.status_code, 200)
示例#35
0
def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile,
                                                missed_messages: List[Dict[
                                                    str, Any]],
                                                message_count: int) -> None:
    """
    Send a reminder email to a user if she's missed some PMs by being offline.

    The email will have its reply to address set to a limited used email
    address that will send a Zulip message to the correct recipient. This
    allows the user to respond to missed PMs, huddles, and @-mentions directly
    from the email.

    `user_profile` is the user to send the reminder to
    `missed_messages` is a list of dictionaries to Message objects and other data
                      for a group of messages that share a recipient (and topic)
    """
    from zerver.context_processors import common_context

    # Disabled missedmessage emails internally
    if not user_profile.enable_offline_email_notifications:
        return

    recipients = {(msg["message"].recipient_id, msg["message"].topic_name())
                  for msg in missed_messages}
    if len(recipients) != 1:
        raise ValueError(
            f"All missed_messages must have the same recipient and topic {recipients!r}",
        )

    # This link is no longer a part of the email, but keeping the code in case
    # we find a clean way to add it back in the future
    unsubscribe_link = one_click_unsubscribe_link(user_profile,
                                                  "missed_messages")
    context = common_context(user_profile)
    context.update(
        name=user_profile.full_name,
        message_count=message_count,
        unsubscribe_link=unsubscribe_link,
        realm_name_in_notifications=user_profile.realm_name_in_notifications,
    )

    triggers = [message["trigger"] for message in missed_messages]
    unique_triggers = set(triggers)
    context.update(
        mention="mentioned" in unique_triggers
        or "wildcard_mentioned" in unique_triggers,
        stream_email_notify="stream_email_notify" in unique_triggers,
        mention_count=triggers.count("mentioned") +
        triggers.count("wildcard_mentioned"),
    )

    # If this setting (email mirroring integration) is enabled, only then
    # can users reply to email to send message to Zulip. Thus, one must
    # ensure to display warning in the template.
    if settings.EMAIL_GATEWAY_PATTERN:
        context.update(reply_to_zulip=True, )
    else:
        context.update(reply_to_zulip=False, )

    from zerver.lib.email_mirror import create_missed_message_address

    reply_to_address = create_missed_message_address(
        user_profile, missed_messages[0]["message"])
    if reply_to_address == FromAddress.NOREPLY:
        reply_to_name = ""
    else:
        reply_to_name = "Zulip"

    narrow_url = get_narrow_url(user_profile, missed_messages[0]["message"])
    context.update(narrow_url=narrow_url, )

    senders = list({m["message"].sender for m in missed_messages})
    if missed_messages[0]["message"].recipient.type == Recipient.HUDDLE:
        display_recipient = get_display_recipient(
            missed_messages[0]["message"].recipient)
        # Make sure that this is a list of strings, not a string.
        assert not isinstance(display_recipient, str)
        other_recipients = [
            r["full_name"] for r in display_recipient
            if r["id"] != user_profile.id
        ]
        context.update(group_pm=True)
        if len(other_recipients) == 2:
            huddle_display_name = " and ".join(other_recipients)
            context.update(huddle_display_name=huddle_display_name)
        elif len(other_recipients) == 3:
            huddle_display_name = (
                f"{other_recipients[0]}, {other_recipients[1]}, and {other_recipients[2]}"
            )
            context.update(huddle_display_name=huddle_display_name)
        else:
            huddle_display_name = "{}, and {} others".format(
                ", ".join(other_recipients[:2]),
                len(other_recipients) - 2)
            context.update(huddle_display_name=huddle_display_name)
    elif missed_messages[0]["message"].recipient.type == Recipient.PERSONAL:
        context.update(private_message=True)
    elif context["mention"] or context["stream_email_notify"]:
        # Keep only the senders who actually mentioned the user
        if context["mention"]:
            senders = list({
                m["message"].sender
                for m in missed_messages if m["trigger"] == "mentioned"
                or m["trigger"] == "wildcard_mentioned"
            })
        message = missed_messages[0]["message"]
        stream = Stream.objects.only("id",
                                     "name").get(id=message.recipient.type_id)
        stream_header = f"{stream.name} > {message.topic_name()}"
        context.update(stream_header=stream_header, )
    else:
        raise AssertionError("Invalid messages!")

    # If message content is disabled, then flush all information we pass to email.
    if not message_content_allowed_in_missedmessage_emails(user_profile):
        realm = user_profile.realm
        context.update(
            reply_to_zulip=False,
            messages=[],
            sender_str="",
            realm_str=realm.name,
            huddle_display_name="",
            show_message_content=False,
            message_content_disabled_by_user=not user_profile.
            message_content_in_email_notifications,
            message_content_disabled_by_realm=not realm.
            message_content_allowed_in_email_notifications,
        )
    else:
        context.update(
            messages=build_message_list(
                user=user_profile,
                messages=[m["message"] for m in missed_messages],
                stream_map={},
            ),
            sender_str=", ".join(sender.full_name for sender in senders),
            realm_str=user_profile.realm.name,
            show_message_content=True,
        )

    with override_language(user_profile.default_language):
        from_name: str = _("Zulip missed messages")
    from_address = FromAddress.NOREPLY
    if len(senders) == 1 and settings.SEND_MISSED_MESSAGE_EMAILS_AS_USER:
        # If this setting is enabled, you can reply to the Zulip
        # missed message emails directly back to the original sender.
        # However, one must ensure the Zulip server is in the SPF
        # record for the domain, or there will be spam/deliverability
        # problems.
        #
        # Also, this setting is not really compatible with
        # EMAIL_ADDRESS_VISIBILITY_ADMINS.
        sender = senders[0]
        from_name, from_address = (sender.full_name, sender.email)
        context.update(reply_to_zulip=False, )

    email_dict = {
        "template_prefix":
        "zerver/emails/missed_message",
        "to_user_ids": [user_profile.id],
        "from_name":
        from_name,
        "from_address":
        from_address,
        "reply_to_email":
        str(Address(display_name=reply_to_name, addr_spec=reply_to_address)),
        "context":
        context,
    }
    queue_json_publish("email_senders", email_dict)

    user_profile.last_reminder = timezone_now()
    user_profile.save(update_fields=["last_reminder"])
示例#36
0
    def validate(self, value: [datetime.date, datetime.datetime]):

        now = timezone_now()
        if now > value:
            raise ValueError("Date/Time is in the past.")
示例#37
0
    def handle(self, **options: Any) -> None:
        if options["percent_huddles"] + options["percent_personals"] > 100:
            self.stderr.write(
                "Error!  More than 100% of messages allocated.\n")
            return

        # Get consistent data for backend tests.
        if options["test_suite"]:
            random.seed(0)

        # If max_topics is not set, we set it proportional to the
        # number of messages.
        if options["max_topics"] is None:
            options["max_topics"] = 1 + options["num_messages"] // 100

        if options["delete"]:
            # Start by clearing all the data in our database
            clear_database()

            # Create our three default realms
            # Could in theory be done via zerver.lib.actions.do_create_realm, but
            # welcome-bot (needed for do_create_realm) hasn't been created yet
            create_internal_realm()
            zulip_realm = Realm.objects.create(
                string_id="zulip",
                name="Zulip Dev",
                emails_restricted_to_domains=False,
                email_address_visibility=Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS,
                description=
                "The Zulip development environment default organization."
                "  It's great for testing!",
                invite_required=False,
                org_type=Realm.CORPORATE)
            RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
            if options["test_suite"]:
                mit_realm = Realm.objects.create(
                    string_id="zephyr",
                    name="MIT",
                    emails_restricted_to_domains=True,
                    invite_required=False,
                    org_type=Realm.CORPORATE)
                RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")

                lear_realm = Realm.objects.create(
                    string_id="lear",
                    name="Lear & Co.",
                    emails_restricted_to_domains=False,
                    invite_required=False,
                    org_type=Realm.CORPORATE)

            # Create test Users (UserProfiles are automatically created,
            # as are subscriptions to the ability to receive personals).
            names = [
                ("Zoe", "*****@*****.**"),
                ("Othello, the Moor of Venice", "*****@*****.**"),
                ("Iago", "*****@*****.**"),
                ("Prospero from The Tempest", "*****@*****.**"),
                ("Cordelia Lear", "*****@*****.**"),
                ("King Hamlet", "*****@*****.**"),
                ("aaron", "*****@*****.**"),
                ("Polonius", "*****@*****.**"),
                ("Desdemona", "*****@*****.**"),
            ]

            # For testing really large batches:
            # Create extra users with semi realistic names to make search
            # functions somewhat realistic.  We'll still create 1000 users
            # like Extra222 User for some predicability.
            num_names = options['extra_users']
            num_boring_names = 300

            for i in range(min(num_names, num_boring_names)):
                full_name = 'Extra%03d User' % (i, )
                names.append((full_name, '*****@*****.**' % (i, )))

            if num_names > num_boring_names:
                fnames = [
                    'Amber', 'Arpita', 'Bob', 'Cindy', 'Daniela', 'Dan',
                    'Dinesh', 'Faye', 'François', 'George', 'Hank', 'Irene',
                    'James', 'Janice', 'Jenny', 'Jill', 'John', 'Kate',
                    'Katelyn', 'Kobe', 'Lexi', 'Manish', 'Mark', 'Matt',
                    'Mayna', 'Michael', 'Pete', 'Peter', 'Phil', 'Phillipa',
                    'Preston', 'Sally', 'Scott', 'Sandra', 'Steve',
                    'Stephanie', 'Vera'
                ]
                mnames = ['de', 'van', 'von', 'Shaw', 'T.']
                lnames = [
                    'Adams', 'Agarwal', 'Beal', 'Benson', 'Bonita', 'Davis',
                    'George', 'Harden', 'James', 'Jones', 'Johnson', 'Jordan',
                    'Lee', 'Leonard', 'Singh', 'Smith', 'Patel', 'Towns',
                    'Wall'
                ]

            for i in range(num_boring_names, num_names):
                fname = random.choice(fnames) + str(i)
                full_name = fname
                if random.random() < 0.7:
                    if random.random() < 0.5:
                        full_name += ' ' + random.choice(mnames)
                    full_name += ' ' + random.choice(lnames)
                email = fname.lower() + '@zulip.com'
                names.append((full_name, email))

            create_users(zulip_realm, names, tos_version=settings.TOS_VERSION)

            iago = get_user_by_delivery_email("*****@*****.**", zulip_realm)
            do_change_user_role(iago, UserProfile.ROLE_REALM_ADMINISTRATOR)
            iago.is_staff = True
            iago.save(update_fields=['is_staff'])

            desdemona = get_user_by_delivery_email("*****@*****.**",
                                                   zulip_realm)
            do_change_user_role(desdemona, UserProfile.ROLE_REALM_OWNER)

            guest_user = get_user_by_delivery_email("*****@*****.**",
                                                    zulip_realm)
            guest_user.role = UserProfile.ROLE_GUEST
            guest_user.save(update_fields=['role'])

            # These bots are directly referenced from code and thus
            # are needed for the test suite.
            zulip_realm_bots = [
                ("Zulip Error Bot", "*****@*****.**"),
                ("Zulip Default Bot", "*****@*****.**"),
            ]
            for i in range(options["extra_bots"]):
                zulip_realm_bots.append(
                    ('Extra Bot %d' % (i, ), '*****@*****.**' % (i, )))

            create_users(zulip_realm,
                         zulip_realm_bots,
                         bot_type=UserProfile.DEFAULT_BOT)

            zoe = get_user_by_delivery_email("*****@*****.**", zulip_realm)
            zulip_webhook_bots = [
                ("Zulip Webhook Bot", "*****@*****.**"),
            ]
            # If a stream is not supplied in the webhook URL, the webhook
            # will (in some cases) send the notification as a PM to the
            # owner of the webhook bot, so bot_owner can't be None
            create_users(zulip_realm,
                         zulip_webhook_bots,
                         bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
                         bot_owner=zoe)
            aaron = get_user_by_delivery_email("*****@*****.**", zulip_realm)

            zulip_outgoing_bots = [
                ("Outgoing Webhook", "*****@*****.**"),
            ]
            create_users(zulip_realm,
                         zulip_outgoing_bots,
                         bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
                         bot_owner=aaron)
            outgoing_webhook = get_user("*****@*****.**",
                                        zulip_realm)
            add_service("outgoing-webhook",
                        user_profile=outgoing_webhook,
                        interface=Service.GENERIC,
                        base_url="http://127.0.0.1:5002",
                        token=generate_api_key())

            # Add the realm internal bots to each realm.
            create_if_missing_realm_internal_bots()

            # Create public streams.
            stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
            stream_dict: Dict[str, Dict[str, Any]] = {
                "Verona": {
                    "description": "A city in Italy"
                },
                "Denmark": {
                    "description": "A Scandinavian country"
                },
                "Scotland": {
                    "description": "Located in the United Kingdom"
                },
                "Venice": {
                    "description": "A northeastern Italian city"
                },
                "Rome": {
                    "description": "Yet another Italian city",
                    "is_web_public": True
                },
            }

            bulk_create_streams(zulip_realm, stream_dict)
            recipient_streams: List[int] = [
                Stream.objects.get(name=name, realm=zulip_realm).id
                for name in stream_list
            ]

            # Create subscriptions to streams.  The following
            # algorithm will give each of the users a different but
            # deterministic subset of the streams (given a fixed list
            # of users). For the test suite, we have a fixed list of
            # subscriptions to make sure test data is consistent
            # across platforms.

            subscriptions_list: List[Tuple[UserProfile, Recipient]] = []
            profiles: Sequence[
                UserProfile] = UserProfile.objects.select_related().filter(
                    is_bot=False).order_by("email")

            if options["test_suite"]:
                subscriptions_map = {
                    '*****@*****.**': ['Verona'],
                    '*****@*****.**': ['Verona'],
                    '*****@*****.**': ['Verona', 'Denmark'],
                    '*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
                    '*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
                    '*****@*****.**':
                    ['Verona', 'Denmark', 'Scotland', 'Venice'],
                    '*****@*****.**':
                    ['Verona', 'Denmark', 'Scotland', 'Venice', 'Rome'],
                    '*****@*****.**': ['Verona'],
                    '*****@*****.**': ['Verona', 'Denmark', 'Venice'],
                }

                for profile in profiles:
                    email = profile.delivery_email
                    if email not in subscriptions_map:
                        raise Exception(
                            f'Subscriptions not listed for user {email}')

                    for stream_name in subscriptions_map[email]:
                        stream = Stream.objects.get(name=stream_name)
                        r = Recipient.objects.get(type=Recipient.STREAM,
                                                  type_id=stream.id)
                        subscriptions_list.append((profile, r))
            else:
                num_streams = len(recipient_streams)
                num_users = len(profiles)
                for i, profile in enumerate(profiles):
                    # Subscribe to some streams.
                    fraction = float(i) / num_users
                    num_recips = int(num_streams * fraction) + 1

                    for type_id in recipient_streams[:num_recips]:
                        r = Recipient.objects.get(type=Recipient.STREAM,
                                                  type_id=type_id)
                        subscriptions_list.append((profile, r))

            subscriptions_to_add: List[Subscription] = []
            event_time = timezone_now()
            all_subscription_logs: (List[RealmAuditLog]) = []

            i = 0
            for profile, recipient in subscriptions_list:
                i += 1
                color = STREAM_ASSIGNMENT_COLORS[i %
                                                 len(STREAM_ASSIGNMENT_COLORS)]
                s = Subscription(recipient=recipient,
                                 user_profile=profile,
                                 color=color)

                subscriptions_to_add.append(s)

                log = RealmAuditLog(
                    realm=profile.realm,
                    modified_user=profile,
                    modified_stream_id=recipient.type_id,
                    event_last_message_id=0,
                    event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
                    event_time=event_time)
                all_subscription_logs.append(log)

            Subscription.objects.bulk_create(subscriptions_to_add)
            RealmAuditLog.objects.bulk_create(all_subscription_logs)

            # Create custom profile field data
            phone_number = try_add_realm_custom_profile_field(
                zulip_realm,
                "Phone number",
                CustomProfileField.SHORT_TEXT,
                hint='')
            biography = try_add_realm_custom_profile_field(
                zulip_realm,
                "Biography",
                CustomProfileField.LONG_TEXT,
                hint='What are you known for?')
            favorite_food = try_add_realm_custom_profile_field(
                zulip_realm,
                "Favorite food",
                CustomProfileField.SHORT_TEXT,
                hint="Or drink, if you'd prefer")
            field_data: ProfileFieldData = {
                'vim': {
                    'text': 'Vim',
                    'order': '1'
                },
                'emacs': {
                    'text': 'Emacs',
                    'order': '2'
                },
            }
            favorite_editor = try_add_realm_custom_profile_field(
                zulip_realm,
                "Favorite editor",
                CustomProfileField.CHOICE,
                field_data=field_data)
            birthday = try_add_realm_custom_profile_field(
                zulip_realm, "Birthday", CustomProfileField.DATE)
            favorite_website = try_add_realm_custom_profile_field(
                zulip_realm,
                "Favorite website",
                CustomProfileField.URL,
                hint="Or your personal blog's URL")
            mentor = try_add_realm_custom_profile_field(
                zulip_realm, "Mentor", CustomProfileField.USER)
            github_profile = try_add_realm_default_custom_profile_field(
                zulip_realm, "github")

            # Fill in values for Iago and Hamlet
            hamlet = get_user_by_delivery_email("*****@*****.**",
                                                zulip_realm)
            do_update_user_custom_profile_data_if_changed(
                iago, [
                    {
                        "id": phone_number.id,
                        "value": "+1-234-567-8901"
                    },
                    {
                        "id": biography.id,
                        "value": "Betrayer of Othello."
                    },
                    {
                        "id": favorite_food.id,
                        "value": "Apples"
                    },
                    {
                        "id": favorite_editor.id,
                        "value": "emacs"
                    },
                    {
                        "id": birthday.id,
                        "value": "2000-1-1"
                    },
                    {
                        "id": favorite_website.id,
                        "value": "https://zulip.readthedocs.io/en/latest/"
                    },
                    {
                        "id": mentor.id,
                        "value": [hamlet.id]
                    },
                    {
                        "id": github_profile.id,
                        "value": 'zulip'
                    },
                ])
            do_update_user_custom_profile_data_if_changed(
                hamlet, [
                    {
                        "id": phone_number.id,
                        "value": "+0-11-23-456-7890"
                    },
                    {
                        "id":
                        biography.id,
                        "value":
                        "I am:\n* The prince of Denmark\n* Nephew to the usurping Claudius",
                    },
                    {
                        "id": favorite_food.id,
                        "value": "Dark chocolate"
                    },
                    {
                        "id": favorite_editor.id,
                        "value": "vim"
                    },
                    {
                        "id": birthday.id,
                        "value": "1900-1-1"
                    },
                    {
                        "id": favorite_website.id,
                        "value": "https://blog.zulig.org"
                    },
                    {
                        "id": mentor.id,
                        "value": [iago.id]
                    },
                    {
                        "id": github_profile.id,
                        "value": 'zulipbot'
                    },
                ])
        else:
            zulip_realm = get_realm("zulip")
            recipient_streams = [
                klass.type_id
                for klass in Recipient.objects.filter(type=Recipient.STREAM)
            ]

        # Extract a list of all users
        user_profiles: List[UserProfile] = list(
            UserProfile.objects.filter(is_bot=False))

        # Create a test realm emoji.
        IMAGE_FILE_PATH = static_path('images/test-images/checkbox.png')
        with open(IMAGE_FILE_PATH, 'rb') as fp:
            check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp)

        if not options["test_suite"]:
            # Populate users with some bar data
            for user in user_profiles:
                status: int = UserPresence.ACTIVE
                date = timezone_now()
                client = get_client("website")
                if user.full_name[0] <= 'H':
                    client = get_client("ZulipAndroid")
                UserPresence.objects.get_or_create(user_profile=user,
                                                   realm_id=user.realm_id,
                                                   client=client,
                                                   timestamp=date,
                                                   status=status)

        user_profiles_ids = [user_profile.id for user_profile in user_profiles]

        # Create several initial huddles
        for i in range(options["num_huddles"]):
            get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))

        # Create several initial pairs for personals
        personals_pairs = [
            random.sample(user_profiles_ids, 2)
            for i in range(options["num_personals"])
        ]

        # Generate a new set of test data.
        create_test_data()

        # prepopulate the URL preview/embed data for the links present
        # in the config.generate_data.json data set.  This makes it
        # possible for populate_db to run happily without Internet
        # access.
        with open("zerver/tests/fixtures/docs_url_preview_data.json") as f:
            urls_with_preview_data = ujson.load(f)
            for url in urls_with_preview_data:
                cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)

        if options["delete"]:
            if options["test_suite"]:
                # Create test users; the MIT ones are needed to test
                # the Zephyr mirroring codepaths.
                testsuite_mit_users = [
                    ("Fred Sipb (MIT)", "*****@*****.**"),
                    ("Athena Consulting Exchange User (MIT)",
                     "*****@*****.**"),
                    ("Esp Classroom (MIT)", "*****@*****.**"),
                ]
                create_users(mit_realm,
                             testsuite_mit_users,
                             tos_version=settings.TOS_VERSION)

                testsuite_lear_users = [
                    ("King Lear", "*****@*****.**"),
                    ("Cordelia Lear", "*****@*****.**"),
                ]
                create_users(lear_realm,
                             testsuite_lear_users,
                             tos_version=settings.TOS_VERSION)

            if not options["test_suite"]:
                # To keep the messages.json fixtures file for the test
                # suite fast, don't add these users and subscriptions
                # when running populate_db for the test suite

                zulip_stream_dict: Dict[str, Dict[str, Any]] = {
                    "devel": {
                        "description": "For developing"
                    },
                    "all": {
                        "description": "For **everything**"
                    },
                    "announce": {
                        "description": "For announcements",
                        'stream_post_policy': Stream.STREAM_POST_POLICY_ADMINS
                    },
                    "design": {
                        "description": "For design"
                    },
                    "support": {
                        "description": "For support"
                    },
                    "social": {
                        "description": "For socializing"
                    },
                    "test": {
                        "description": "For testing `code`"
                    },
                    "errors": {
                        "description": "For errors"
                    },
                    "sales": {
                        "description": "For sales discussion"
                    },
                }

                # Calculate the maximum number of digits in any extra stream's
                # number, since a stream with name "Extra Stream 3" could show
                # up after "Extra Stream 29". (Used later to pad numbers with
                # 0s).
                maximum_digits = len(str(options['extra_streams'] - 1))

                for i in range(options['extra_streams']):
                    # Pad the number with 0s based on `maximum_digits`.
                    number_str = str(i).zfill(maximum_digits)

                    extra_stream_name = 'Extra Stream ' + number_str

                    zulip_stream_dict[extra_stream_name] = {
                        "description": "Auto-generated extra stream.",
                    }

                bulk_create_streams(zulip_realm, zulip_stream_dict)
                # Now that we've created the notifications stream, configure it properly.
                zulip_realm.notifications_stream = get_stream(
                    "announce", zulip_realm)
                zulip_realm.save(update_fields=['notifications_stream'])

                # Add a few default streams
                for default_stream_name in [
                        "design", "devel", "social", "support"
                ]:
                    DefaultStream.objects.create(realm=zulip_realm,
                                                 stream=get_stream(
                                                     default_stream_name,
                                                     zulip_realm))

                # Now subscribe everyone to these streams
                subscribe_users_to_streams(zulip_realm, zulip_stream_dict)

            if not options["test_suite"]:
                # Update pointer of each user to point to the last message in their
                # UserMessage rows with sender_id=user_profile_id.
                users = list(
                    UserMessage.objects.filter(message__sender_id=F(
                        'user_profile_id')).values('user_profile_id').annotate(
                            pointer=Max('message_id')))
                for user in users:
                    UserProfile.objects.filter(
                        id=user['user_profile_id']).update(
                            pointer=user['pointer'])

            create_user_groups()

            if not options["test_suite"]:
                # We populate the analytics database here for
                # development purpose only
                call_command('populate_analytics_db')

        threads = options["threads"]
        jobs: List[Tuple[int, List[List[int]], Dict[str, Any],
                         Callable[[str], int], int]] = []
        for i in range(threads):
            count = options["num_messages"] // threads
            if i < options["num_messages"] % threads:
                count += 1
            jobs.append((count, personals_pairs, options, self.stdout.write,
                         random.randint(0, 10**10)))

        for job in jobs:
            generate_and_send_messages(job)

        if options["delete"]:
            if not options['test_suite']:
                # These bots are not needed by the test suite
                # Also, we don't want interacting with each other
                # in dev setup.
                internal_zulip_users_nosubs = [
                    ("Zulip Commit Bot", "*****@*****.**"),
                    ("Zulip Trac Bot", "*****@*****.**"),
                    ("Zulip Nagios Bot", "*****@*****.**"),
                ]
                create_users(zulip_realm,
                             internal_zulip_users_nosubs,
                             bot_type=UserProfile.DEFAULT_BOT)

            # Mark all messages as read
            UserMessage.objects.all().update(flags=UserMessage.flags.read)
            self.stdout.write("Successfully populated test database.\n")
示例#38
0
    def test_users_to_zerver_userprofile(
            self, mock_get_data_file: mock.Mock) -> None:
        custom_profile_field_user1 = {
            "Xf06054BBB": {
                "value": "random1"
            },
            "Xf023DSCdd": {
                "value": "employee"
            }
        }
        custom_profile_field_user2 = {
            "Xf06054BBB": {
                "value": "random2"
            },
            "Xf023DSCdd": {
                "value": "employer"
            }
        }
        user_data = [{
            "id": "U08RGD1RD",
            "team_id": "T5YFFM2QY",
            "name": "john",
            "deleted": False,
            "is_mirror_dummy": False,
            "real_name": "John Doe",
            "profile": {
                "image_32": "",
                "email": "*****@*****.**",
                "avatar_hash": "hash",
                "phone": "+1-123-456-77-868",
                "fields": custom_profile_field_user1
            }
        }, {
            "id": "U0CBK5KAT",
            "team_id": "T5YFFM2QY",
            "is_admin": True,
            "is_bot": False,
            "is_owner": True,
            "is_primary_owner": True,
            'name': 'Jane',
            "real_name": "Jane Doe",
            "deleted": False,
            "is_mirror_dummy": False,
            "profile": {
                "image_32": "https://secure.gravatar.com/avatar/random.png",
                "fields": custom_profile_field_user2,
                "email": "*****@*****.**",
                "avatar_hash": "hash"
            }
        }, {
            "id": "U09TYF5Sk",
            "team_id": "T5YFFM2QY",
            "name": "Bot",
            "real_name": "Bot",
            "is_bot": True,
            "deleted": False,
            "is_mirror_dummy": False,
            "profile": {
                "image_32": "https://secure.gravatar.com/avatar/random1.png",
                "skype": "test_skype_name",
                "email": "*****@*****.**",
                "avatar_hash": "hash"
            }
        }, {
            "id": "UHSG7OPQN",
            "team_id": "T6LARQE2Z",
            'name': 'matt.perry',
            "color": '9d8eee',
            "is_bot": False,
            "is_app_user": False,
            "is_mirror_dummy": True,
            "team_domain": "foreignteam",
            "profile": {
                "image_32": "https://secure.gravatar.com/avatar/random6.png",
                "avatar_hash": "hash",
                "first_name": "Matt",
                "last_name": "Perry",
                "real_name": "Matt Perry",
                "display_name": "matt.perry",
                "team": "T6LARQE2Z"
            }
        }]

        mock_get_data_file.return_value = user_data
        # As user with slack_id 'U0CBK5KAT' is the primary owner, that user should be imported first
        # and hence has zulip_id = 1
        test_slack_user_id_to_zulip_user_id = {
            'U08RGD1RD': 1,
            'U0CBK5KAT': 0,
            'U09TYF5Sk': 2,
            'UHSG7OPQN': 3
        }
        slack_data_dir = './random_path'
        timestamp = int(timezone_now().timestamp())
        mock_get_data_file.return_value = user_data
        zerver_userprofile, avatar_list, slack_user_id_to_zulip_user_id, customprofilefield, \
            customprofilefield_value = users_to_zerver_userprofile(slack_data_dir, user_data, 1,
                                                                   timestamp, 'test_domain')

        # Test custom profile fields
        self.assertEqual(customprofilefield[0]['field_type'], 1)
        self.assertEqual(customprofilefield[3]['name'], 'skype')
        cpf_name = {cpf['name'] for cpf in customprofilefield}
        self.assertIn('phone', cpf_name)
        self.assertIn('skype', cpf_name)
        cpf_name.remove('phone')
        cpf_name.remove('skype')
        for name in cpf_name:
            self.assertTrue(name.startswith('slack custom field '))

        self.assertEqual(len(customprofilefield_value), 6)
        self.assertEqual(customprofilefield_value[0]['field'], 0)
        self.assertEqual(customprofilefield_value[0]['user_profile'], 1)
        self.assertEqual(customprofilefield_value[3]['user_profile'], 0)
        self.assertEqual(customprofilefield_value[5]['value'],
                         'test_skype_name')

        # test that the primary owner should always be imported first
        self.assertDictEqual(slack_user_id_to_zulip_user_id,
                             test_slack_user_id_to_zulip_user_id)
        self.assertEqual(len(avatar_list), 4)

        self.assertEqual(len(zerver_userprofile), 4)

        self.assertEqual(zerver_userprofile[0]['is_staff'], False)
        self.assertEqual(zerver_userprofile[0]['is_bot'], False)
        self.assertEqual(zerver_userprofile[0]['is_active'], True)
        self.assertEqual(zerver_userprofile[0]['is_mirror_dummy'], False)
        self.assertEqual(zerver_userprofile[0]['role'],
                         UserProfile.ROLE_MEMBER)
        self.assertEqual(zerver_userprofile[0]['enable_desktop_notifications'],
                         True)
        self.assertEqual(zerver_userprofile[0]['email'], '*****@*****.**')
        self.assertEqual(zerver_userprofile[0]['full_name'], 'John Doe')

        self.assertEqual(zerver_userprofile[1]['id'],
                         test_slack_user_id_to_zulip_user_id['U0CBK5KAT'])
        self.assertEqual(zerver_userprofile[1]['role'],
                         UserProfile.ROLE_REALM_ADMINISTRATOR)
        self.assertEqual(zerver_userprofile[1]['is_staff'], False)
        self.assertEqual(zerver_userprofile[1]['is_active'], True)
        self.assertEqual(zerver_userprofile[0]['is_mirror_dummy'], False)

        self.assertEqual(zerver_userprofile[2]['id'],
                         test_slack_user_id_to_zulip_user_id['U09TYF5Sk'])
        self.assertEqual(zerver_userprofile[2]['is_bot'], True)
        self.assertEqual(zerver_userprofile[2]['is_active'], True)
        self.assertEqual(zerver_userprofile[2]['is_mirror_dummy'], False)
        self.assertEqual(zerver_userprofile[2]['email'], '*****@*****.**')
        self.assertEqual(zerver_userprofile[2]['bot_type'], 1)
        self.assertEqual(zerver_userprofile[2]['avatar_source'], 'U')

        self.assertEqual(zerver_userprofile[3]['id'],
                         test_slack_user_id_to_zulip_user_id['UHSG7OPQN'])
        self.assertEqual(zerver_userprofile[3]['role'],
                         UserProfile.ROLE_MEMBER)
        self.assertEqual(zerver_userprofile[3]['is_staff'], False)
        self.assertEqual(zerver_userprofile[3]['is_active'], False)
        self.assertEqual(zerver_userprofile[3]['email'],
                         '*****@*****.**')
        self.assertEqual(zerver_userprofile[3]['realm'], 1)
        self.assertEqual(zerver_userprofile[3]['full_name'], 'Matt Perry')
        self.assertEqual(zerver_userprofile[3]['short_name'], 'matt.perry')
        self.assertEqual(zerver_userprofile[3]['is_mirror_dummy'], True)
        self.assertEqual(zerver_userprofile[3]['is_api_super_user'], False)
def upload_to(instance, filename):
    now = timezone_now()
    base, extension = os.path.splitext(filename)
    extension = extension.lower()
    return f"ideas/{now:%Y/%m}/{instance.pk}{extension}"
示例#40
0
def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
    now = timezone_now()

    query = SQL("""
        SELECT
            realm.string_id,
            realm.date_created,
            realm.plan_type,
            realm.org_type,
            coalesce(wau_table.value, 0) wau_count,
            coalesce(dau_table.value, 0) dau_count,
            coalesce(user_count_table.value, 0) user_profile_count,
            coalesce(bot_count_table.value, 0) bot_count
        FROM
            zerver_realm as realm
            LEFT OUTER JOIN (
                SELECT
                    value _14day_active_humans,
                    realm_id
                from
                    analytics_realmcount
                WHERE
                    property = 'realm_active_humans::day'
                    AND end_time = %(realm_active_humans_end_time)s
            ) as _14day_active_humans_table ON realm.id = _14day_active_humans_table.realm_id
            LEFT OUTER JOIN (
                SELECT
                    value,
                    realm_id
                from
                    analytics_realmcount
                WHERE
                    property = '7day_actives::day'
                    AND end_time = %(seven_day_actives_end_time)s
            ) as wau_table ON realm.id = wau_table.realm_id
            LEFT OUTER JOIN (
                SELECT
                    value,
                    realm_id
                from
                    analytics_realmcount
                WHERE
                    property = '1day_actives::day'
                    AND end_time = %(one_day_actives_end_time)s
            ) as dau_table ON realm.id = dau_table.realm_id
            LEFT OUTER JOIN (
                SELECT
                    value,
                    realm_id
                from
                    analytics_realmcount
                WHERE
                    property = 'active_users_audit:is_bot:day'
                    AND subgroup = 'false'
                    AND end_time = %(active_users_audit_end_time)s
            ) as user_count_table ON realm.id = user_count_table.realm_id
            LEFT OUTER JOIN (
                SELECT
                    value,
                    realm_id
                from
                    analytics_realmcount
                WHERE
                    property = 'active_users_audit:is_bot:day'
                    AND subgroup = 'true'
                    AND end_time = %(active_users_audit_end_time)s
            ) as bot_count_table ON realm.id = bot_count_table.realm_id
        WHERE
            _14day_active_humans IS NOT NULL
            or realm.plan_type = 3
        ORDER BY
            dau_count DESC,
            string_id ASC
    """)

    cursor = connection.cursor()
    cursor.execute(
        query,
        {
            "realm_active_humans_end_time":
            COUNT_STATS["realm_active_humans::day"].last_successful_fill(),
            "seven_day_actives_end_time":
            COUNT_STATS["7day_actives::day"].last_successful_fill(),
            "one_day_actives_end_time":
            COUNT_STATS["1day_actives::day"].last_successful_fill(),
            "active_users_audit_end_time":
            COUNT_STATS["active_users_audit:is_bot:day"].last_successful_fill(
            ),
        },
    )
    rows = dictfetchall(cursor)
    cursor.close()

    # Fetch all the realm administrator users
    realm_owners: Dict[str, List[str]] = defaultdict(list)
    for up in UserProfile.objects.select_related("realm").filter(
            role=UserProfile.ROLE_REALM_OWNER,
            is_active=True,
    ):
        realm_owners[up.realm.string_id].append(up.delivery_email)

    for row in rows:
        row["date_created_day"] = row["date_created"].strftime("%Y-%m-%d")
        row["age_days"] = int(
            (now - row["date_created"]).total_seconds() / 86400)
        row["is_new"] = row["age_days"] < 12 * 7
        row["realm_owner_emails"] = ", ".join(realm_owners[row["string_id"]])

    # get messages sent per day
    counts = get_realm_day_counts()
    for row in rows:
        try:
            row["history"] = counts[row["string_id"]]["cnts"]
        except Exception:
            row["history"] = ""

    # estimate annual subscription revenue
    total_arr = 0
    if settings.BILLING_ENABLED:
        estimated_arrs = estimate_annual_recurring_revenue_by_realm()
        realms_to_default_discount = get_realms_to_default_discount_dict()

        for row in rows:
            row["plan_type_string"] = get_plan_name(row["plan_type"])

            string_id = row["string_id"]

            if string_id in estimated_arrs:
                row["arr"] = estimated_arrs[string_id]

            if row["plan_type"] == Realm.STANDARD:
                row["effective_rate"] = 100 - int(
                    realms_to_default_discount.get(string_id, 0))
            elif row["plan_type"] == Realm.STANDARD_FREE:
                row["effective_rate"] = 0
            elif row[
                    "plan_type"] == Realm.LIMITED and string_id in realms_to_default_discount:
                row["effective_rate"] = 100 - int(
                    realms_to_default_discount[string_id])
            else:
                row["effective_rate"] = ""

        total_arr += sum(estimated_arrs.values())

    for row in rows:
        row["org_type_string"] = get_org_type_display_name(row["org_type"])

    # augment data with realm_minutes
    total_hours = 0.0
    for row in rows:
        string_id = row["string_id"]
        minutes = realm_minutes.get(string_id, 0.0)
        hours = minutes / 60.0
        total_hours += hours
        row["hours"] = str(int(hours))
        try:
            row["hours_per_user"] = "******".format(hours / row["dau_count"])
        except Exception:
            pass

    # formatting
    for row in rows:
        row["stats_link"] = realm_stats_link(row["string_id"])
        row["string_id"] = realm_activity_link(row["string_id"])

    # Count active sites
    def meets_goal(row: Dict[str, int]) -> bool:
        return row["dau_count"] >= 5

    num_active_sites = len(list(filter(meets_goal, rows)))

    # create totals
    total_dau_count = 0
    total_user_profile_count = 0
    total_bot_count = 0
    total_wau_count = 0
    for row in rows:
        total_dau_count += int(row["dau_count"])
        total_user_profile_count += int(row["user_profile_count"])
        total_bot_count += int(row["bot_count"])
        total_wau_count += int(row["wau_count"])

    total_row = dict(
        string_id="Total",
        plan_type_string="",
        org_type_string="",
        effective_rate="",
        arr=total_arr,
        stats_link="",
        date_created_day="",
        realm_owner_emails="",
        dau_count=total_dau_count,
        user_profile_count=total_user_profile_count,
        bot_count=total_bot_count,
        hours=int(total_hours),
        wau_count=total_wau_count,
    )

    rows.insert(0, total_row)

    content = loader.render_to_string(
        "analytics/realm_summary_table.html",
        dict(
            rows=rows,
            num_active_sites=num_active_sites,
            utctime=now.strftime("%Y-%m-%d %H:%MZ"),
            billing_enabled=settings.BILLING_ENABLED,
        ),
    )
    return content
    def test_do_auto_soft_deactivate_users(self) -> None:
        users = [
            self.example_user('iago'),
            self.example_user('cordelia'),
            self.example_user('ZOE'),
            self.example_user('othello'),
            self.example_user('prospero'),
            self.example_user('aaron'),
            self.example_user('polonius'),
        ]
        sender = self.example_user('hamlet')
        realm = get_realm('zulip')
        stream_name = 'announce'
        for user in users + [sender]:
            self.subscribe(user, stream_name)

        client, _ = Client.objects.get_or_create(name='website')
        query = '/json/users/me/pointer'
        last_visit = timezone_now()
        count = 150
        for user_profile in users:
            UserActivity.objects.get_or_create(
                user_profile=user_profile,
                client=client,
                query=query,
                count=count,
                last_visit=last_visit
            )

        with mock.patch('logging.info'):
            users_deactivated = do_auto_soft_deactivate_users(-1, realm)
        self.assert_length(users_deactivated, len(users))
        for user in users:
            self.assertTrue(user in users_deactivated)

        # Verify that deactivated users are caught up automatically

        message_id = self.send_stream_message(sender.email, stream_name)
        received_count = UserMessage.objects.filter(user_profile__in=users,
                                                    message_id=message_id).count()
        self.assertEqual(0, received_count)

        with mock.patch('logging.info'):
            users_deactivated = do_auto_soft_deactivate_users(-1, realm)

        self.assert_length(users_deactivated, 0)   # all users are already deactivated
        received_count = UserMessage.objects.filter(user_profile__in=users,
                                                    message_id=message_id).count()
        self.assertEqual(len(users), received_count)

        # Verify that deactivated users are NOT caught up if
        # AUTO_CATCH_UP_SOFT_DEACTIVATED_USERS is off

        message_id = self.send_stream_message(sender.email, stream_name)
        received_count = UserMessage.objects.filter(user_profile__in=users,
                                                    message_id=message_id).count()
        self.assertEqual(0, received_count)

        with self.settings(AUTO_CATCH_UP_SOFT_DEACTIVATED_USERS=False):
            with mock.patch('logging.info'):
                users_deactivated = do_auto_soft_deactivate_users(-1, realm)

        self.assert_length(users_deactivated, 0)   # all users are already deactivated
        received_count = UserMessage.objects.filter(user_profile__in=users,
                                                    message_id=message_id).count()
        self.assertEqual(0, received_count)
示例#42
0
    def test_edit_message(self) -> None:
        """
        Verify if the time limit imposed on message editing is working correctly.
        """
        iago = self.example_user("iago")
        self.login("iago")

        # Set limit to edit message content.
        MESSAGE_CONTENT_EDIT_LIMIT = 5 * 60  # 5 minutes
        result = self.client_patch(
            "/json/realm",
            {
                "allow_message_editing": "true",
                "message_content_edit_limit_seconds": MESSAGE_CONTENT_EDIT_LIMIT,
            },
        )
        self.assert_json_success(result)

        sent_message_id = self.send_stream_message(
            iago,
            "Scotland",
            topic_name="lunch",
            content="I want pizza!",
        )
        message_sent_time = timezone_now()

        # Verify message sent.
        message = most_recent_message(iago)
        self.assertEqual(message.id, sent_message_id)
        self.assertEqual(message.content, "I want pizza!")

        # Edit message content now. This should work as we're editing
        # it immediately after sending i.e., before the limit exceeds.
        result = self.client_patch(
            f"/json/messages/{sent_message_id}", {"content": "I want burger!"}
        )
        self.assert_json_success(result)
        message = most_recent_message(iago)
        self.assertEqual(message.id, sent_message_id)
        self.assertEqual(message.content, "I want burger!")

        # Now that we tested message editing works within the limit,
        # we want to verify it doesn't work beyond the limit.
        #
        # To do that we'll have to wait for the time limit to pass which is
        # 5 minutes here. Easy, use time.sleep() but mind that it slows down the
        # test to a great extent which isn't good. This is when mocking comes to rescue.
        # We can check what the original code does to determine whether the time limit
        # exceeded and mock that here such that the code runs as if the time limit
        # exceeded without actually waiting for that long!
        #
        # In this case, it is timezone_now, an alias to django.utils.timezone.now,
        # to which the difference with message-sent-time is checked. So, we want
        # that timezone_now() call to return `datetime` object representing time
        # that is beyond the limit.
        #
        # Notice how mock.patch() is used here to do exactly the above mentioned.
        # mock.patch() here makes any calls to `timezone_now` in `zerver.lib.actions`
        # to return the value passed to `return_value` in the its context.
        # You can also use mock.patch() as a decorator depending on the
        # requirements. Read more at the documentation link provided above.

        time_beyond_edit_limit = message_sent_time + datetime.timedelta(
            seconds=MESSAGE_CONTENT_EDIT_LIMIT + 100
        )  # There's a buffer time applied to the limit, hence the extra 100s.

        with mock.patch(
            "zerver.lib.actions.timezone_now",
            return_value=time_beyond_edit_limit,
        ):
            result = self.client_patch(
                f"/json/messages/{sent_message_id}", {"content": "I actually want pizza."}
            )
            self.assert_json_error(result, msg="The time limit for editing this message has passed")
            message = most_recent_message(iago)
            self.assertEqual(message.id, sent_message_id)
            self.assertEqual(message.content, "I want burger!")
示例#43
0
def upload_to(instance, filename):
    now = timezone_now()
    filename_base, filename_ext = os.path.splitext(filename)
    return 'uploads/{}_{}{}'.format(now.strftime("%Y/%m/%d/%Y%m%d%H%M%S"),
                                    create_random_string(),
                                    filename_ext.lower())
    def test_do_auto_soft_deactivate_users(self) -> None:
        users = [
            self.example_user('iago'),
            self.example_user('cordelia'),
            self.example_user('ZOE'),
            self.example_user('othello'),
            self.example_user('prospero'),
            self.example_user('aaron'),
            self.example_user('polonius'),
            self.example_user('desdemona'),
        ]
        sender = self.example_user('hamlet')
        realm = get_realm('zulip')
        stream_name = 'announce'
        for user in [*users, sender]:
            self.subscribe(user, stream_name)

        client, _ = Client.objects.get_or_create(name='website')
        query = '/some/random/endpoint'
        last_visit = timezone_now()
        count = 150
        for user_profile in users:
            UserActivity.objects.get_or_create(
                user_profile=user_profile,
                client=client,
                query=query,
                count=count,
                last_visit=last_visit,
            )

        with self.assertLogs(logger_string, level="INFO") as m:
            users_deactivated = do_auto_soft_deactivate_users(-1, realm)

        log_output = []
        for user in users:
            log_output.append(
                f"INFO:{logger_string}:Soft Deactivated user {user.id}")
        log_output.append(
            f"INFO:{logger_string}:Soft-deactivated batch of {len(users[:100])} users; {len(users[100:])} remain to process"
        )
        log_output.append(
            f"INFO:{logger_string}:Caught up {len(users)} soft-deactivated users"
        )
        self.assertEqual(set(m.output), set(log_output))

        self.assert_length(users_deactivated, len(users))
        for user in users:
            self.assertTrue(user in users_deactivated)

        # Verify that deactivated users are caught up automatically

        message_id = self.send_stream_message(sender, stream_name)
        received_count = UserMessage.objects.filter(
            user_profile__in=users, message_id=message_id).count()
        self.assertEqual(0, received_count)

        with self.assertLogs(logger_string, level="INFO") as m:
            users_deactivated = do_auto_soft_deactivate_users(-1, realm)

        log_output = []
        log_output.append(
            f"INFO:{logger_string}:Caught up {len(users)} soft-deactivated users"
        )
        self.assertEqual(set(m.output), set(log_output))

        self.assert_length(users_deactivated,
                           0)  # all users are already deactivated
        received_count = UserMessage.objects.filter(
            user_profile__in=users, message_id=message_id).count()
        self.assertEqual(len(users), received_count)

        # Verify that deactivated users are NOT caught up if
        # AUTO_CATCH_UP_SOFT_DEACTIVATED_USERS is off

        message_id = self.send_stream_message(sender, stream_name)
        received_count = UserMessage.objects.filter(
            user_profile__in=users, message_id=message_id).count()
        self.assertEqual(0, received_count)

        with self.settings(AUTO_CATCH_UP_SOFT_DEACTIVATED_USERS=False):
            with self.assertLogs(logger_string, level="INFO") as m:
                users_deactivated = do_auto_soft_deactivate_users(-1, realm)
        self.assertEqual(m.output, [
            f"INFO:{logger_string}:Not catching up users since AUTO_CATCH_UP_SOFT_DEACTIVATED_USERS is off"
        ])

        self.assert_length(users_deactivated,
                           0)  # all users are already deactivated
        received_count = UserMessage.objects.filter(
            user_profile__in=users, message_id=message_id).count()
        self.assertEqual(0, received_count)
示例#45
0
    def test_convert_slack_workspace_messages(
            self, mock_get_messages_iterator: mock.Mock,
            mock_message: mock.Mock) -> None:
        output_dir = os.path.join(settings.TEST_WORKER_DIR,
                                  "test-slack-import")
        os.makedirs(output_dir, exist_ok=True)

        added_channels: Dict[str, Tuple[str, int]] = {
            "random": ("c5", 1),
            "general": ("c6", 2)
        }

        time = float(timezone_now().timestamp())
        zerver_message = [{"id": 1, "ts": time}, {"id": 5, "ts": time}]

        def fake_get_messages_iter(
            slack_data_dir: str,
            added_channels: AddedChannelsT,
            added_mpims: AddedMPIMsT,
            dm_members: DMMembersT,
        ) -> Iterator[ZerverFieldsT]:
            import copy

            return iter(copy.deepcopy(zerver_message))

        realm: Dict[str, Any] = {"zerver_subscription": []}
        user_list: List[Dict[str, Any]] = []
        reactions = [{"name": "grinning", "users": ["U061A5N1G"], "count": 1}]
        attachments: List[Dict[str, Any]] = []
        uploads: List[Dict[str, Any]] = []

        zerver_usermessage = [{"id": 3}, {"id": 5}, {"id": 6}, {"id": 9}]

        mock_get_messages_iterator.side_effect = fake_get_messages_iter
        mock_message.side_effect = [
            [
                zerver_message[:1], zerver_usermessage[:2], attachments,
                uploads, reactions[:1]
            ],
            [
                zerver_message[1:2], zerver_usermessage[2:5], attachments,
                uploads, reactions[1:1]
            ],
        ]

        with self.assertLogs(level="INFO"):
            # Hacky: We should include a zerver_userprofile, not the empty []
            test_reactions, uploads, zerver_attachment = convert_slack_workspace_messages(
                "./random_path",
                user_list,
                2,
                {},
                {},
                added_channels,
                {},
                {},
                realm,
                [],
                [],
                "domain",
                output_dir=output_dir,
                chunk_size=1,
            )

        messages_file_1 = os.path.join(output_dir, "messages-000001.json")
        self.assertTrue(os.path.exists(messages_file_1))
        messages_file_2 = os.path.join(output_dir, "messages-000002.json")
        self.assertTrue(os.path.exists(messages_file_2))

        with open(messages_file_1, "rb") as f:
            message_json = orjson.loads(f.read())
        self.assertEqual(message_json["zerver_message"], zerver_message[:1])
        self.assertEqual(message_json["zerver_usermessage"],
                         zerver_usermessage[:2])

        with open(messages_file_2, "rb") as f:
            message_json = orjson.loads(f.read())
        self.assertEqual(message_json["zerver_message"], zerver_message[1:2])
        self.assertEqual(message_json["zerver_usermessage"],
                         zerver_usermessage[2:5])

        self.assertEqual(test_reactions, reactions)
示例#46
0
def do_update_message(
    user_profile: UserProfile,
    target_message: Message,
    new_stream: Optional[Stream],
    topic_name: Optional[str],
    propagate_mode: str,
    send_notification_to_old_thread: bool,
    send_notification_to_new_thread: bool,
    content: Optional[str],
    rendering_result: Optional[MessageRenderingResult],
    prior_mention_user_ids: Set[int],
    mention_data: Optional[MentionData] = None,
) -> int:
    """
    The main function for message editing.  A message edit event can
    modify:
    * the message's content (in which case the caller will have
      set both content and rendered_content),
    * the topic, in which case the caller will have set topic_name
    * or both message's content and the topic
    * or stream and/or topic, in which case the caller will have set
        new_stream and/or topic_name.

    With topic edits, propagate_mode determines whether other message
    also have their topics edited.
    """
    timestamp = timezone_now()
    target_message.last_edit_time = timestamp

    event: Dict[str, Any] = {
        "type": "update_message",
        "user_id": user_profile.id,
        "edit_timestamp": datetime_to_timestamp(timestamp),
        "message_id": target_message.id,
        "rendering_only": False,
    }

    edit_history_event: EditHistoryEvent = {
        "user_id": user_profile.id,
        "timestamp": event["edit_timestamp"],
    }

    changed_messages = [target_message]

    realm = user_profile.realm

    stream_being_edited = None
    if target_message.is_stream_message():
        stream_id = target_message.recipient.type_id
        stream_being_edited = get_stream_by_id_in_realm(stream_id, realm)
        event["stream_name"] = stream_being_edited.name
        event["stream_id"] = stream_being_edited.id

    ums = UserMessage.objects.filter(message=target_message.id)

    if content is not None:
        assert rendering_result is not None

        # mention_data is required if there's a content edit.
        assert mention_data is not None

        # add data from group mentions to mentions_user_ids.
        for group_id in rendering_result.mentions_user_group_ids:
            members = mention_data.get_group_members(group_id)
            rendering_result.mentions_user_ids.update(members)

        update_user_message_flags(rendering_result, ums)

        # One could imagine checking realm.allow_edit_history here and
        # modifying the events based on that setting, but doing so
        # doesn't really make sense.  We need to send the edit event
        # to clients regardless, and a client already had access to
        # the original/pre-edit content of the message anyway.  That
        # setting must be enforced on the client side, and making a
        # change here simply complicates the logic for clients parsing
        # edit history events.
        event["orig_content"] = target_message.content
        event["orig_rendered_content"] = target_message.rendered_content
        edit_history_event["prev_content"] = target_message.content
        edit_history_event[
            "prev_rendered_content"] = target_message.rendered_content
        edit_history_event[
            "prev_rendered_content_version"] = target_message.rendered_content_version
        target_message.content = content
        target_message.rendered_content = rendering_result.rendered_content
        target_message.rendered_content_version = markdown_version
        event["content"] = content
        event["rendered_content"] = rendering_result.rendered_content
        event[
            "prev_rendered_content_version"] = target_message.rendered_content_version
        event["is_me_message"] = Message.is_status_message(
            content, rendering_result.rendered_content)

        # target_message.has_image and target_message.has_link will have been
        # already updated by Markdown rendering in the caller.
        target_message.has_attachment = check_attachment_reference_change(
            target_message, rendering_result)

        if target_message.is_stream_message():
            if topic_name is not None:
                new_topic_name = topic_name
            else:
                new_topic_name = target_message.topic_name()

            stream_topic: Optional[StreamTopicTarget] = StreamTopicTarget(
                stream_id=stream_id,
                topic_name=new_topic_name,
            )
        else:
            stream_topic = None

        info = get_recipient_info(
            realm_id=realm.id,
            recipient=target_message.recipient,
            sender_id=target_message.sender_id,
            stream_topic=stream_topic,
            possible_wildcard_mention=mention_data.message_has_wildcards(),
        )

        event["online_push_user_ids"] = list(info["online_push_user_ids"])
        event["pm_mention_push_disabled_user_ids"] = list(
            info["pm_mention_push_disabled_user_ids"])
        event["pm_mention_email_disabled_user_ids"] = list(
            info["pm_mention_email_disabled_user_ids"])
        event["stream_push_user_ids"] = list(info["stream_push_user_ids"])
        event["stream_email_user_ids"] = list(info["stream_email_user_ids"])
        event["muted_sender_user_ids"] = list(info["muted_sender_user_ids"])
        event["prior_mention_user_ids"] = list(prior_mention_user_ids)
        event["presence_idle_user_ids"] = filter_presence_idle_user_ids(
            info["active_user_ids"])
        event["all_bot_user_ids"] = list(info["all_bot_user_ids"])
        if rendering_result.mentions_wildcard:
            event["wildcard_mention_user_ids"] = list(
                info["wildcard_mention_user_ids"])
        else:
            event["wildcard_mention_user_ids"] = []

        do_update_mobile_push_notification(
            target_message,
            prior_mention_user_ids,
            rendering_result.mentions_user_ids,
            info["stream_push_user_ids"],
        )

    if topic_name is not None or new_stream is not None:
        orig_topic_name = target_message.topic_name()
        event["propagate_mode"] = propagate_mode

    if new_stream is not None:
        assert content is None
        assert target_message.is_stream_message()
        assert stream_being_edited is not None

        edit_history_event["prev_stream"] = stream_being_edited.id
        edit_history_event["stream"] = new_stream.id
        event[ORIG_TOPIC] = orig_topic_name
        target_message.recipient_id = new_stream.recipient_id

        event["new_stream_id"] = new_stream.id
        event["propagate_mode"] = propagate_mode

        # When messages are moved from one stream to another, some
        # users may lose access to those messages, including guest
        # users and users not subscribed to the new stream (if it is a
        # private stream).  For those users, their experience is as
        # though the messages were deleted, and we should send a
        # delete_message event to them instead.

        subs_to_old_stream = get_active_subscriptions_for_stream_id(
            stream_id,
            include_deactivated_users=True).select_related("user_profile")
        subs_to_new_stream = list(
            get_active_subscriptions_for_stream_id(
                new_stream.id,
                include_deactivated_users=True).select_related("user_profile"))

        old_stream_sub_ids = [
            user.user_profile_id for user in subs_to_old_stream
        ]
        new_stream_sub_ids = [
            user.user_profile_id for user in subs_to_new_stream
        ]

        # Get users who aren't subscribed to the new_stream.
        subs_losing_usermessages = [
            sub for sub in subs_to_old_stream
            if sub.user_profile_id not in new_stream_sub_ids
        ]
        # Users who can longer access the message without some action
        # from administrators.
        subs_losing_access = [
            sub for sub in subs_losing_usermessages
            if sub.user_profile.is_guest or not new_stream.is_public()
        ]
        ums = ums.exclude(user_profile_id__in=[
            sub.user_profile_id for sub in subs_losing_usermessages
        ])

        subs_gaining_usermessages = []
        if not new_stream.is_history_public_to_subscribers():
            # For private streams, with history not public to subscribers,
            # We find out users who are not present in the msgs' old stream
            # and create new UserMessage for these users so that they can
            # access this message.
            subs_gaining_usermessages += [
                user_id for user_id in new_stream_sub_ids
                if user_id not in old_stream_sub_ids
            ]

    if topic_name is not None:
        topic_name = truncate_topic(topic_name)
        target_message.set_topic_name(topic_name)

        # These fields have legacy field names.
        event[ORIG_TOPIC] = orig_topic_name
        event[TOPIC_NAME] = topic_name
        event[TOPIC_LINKS] = topic_links(target_message.sender.realm_id,
                                         topic_name)
        edit_history_event["prev_topic"] = orig_topic_name
        edit_history_event["topic"] = topic_name

    update_edit_history(target_message, timestamp, edit_history_event)

    delete_event_notify_user_ids: List[int] = []
    if propagate_mode in ["change_later", "change_all"]:
        assert topic_name is not None or new_stream is not None
        assert stream_being_edited is not None

        # Other messages should only get topic/stream fields in their edit history.
        topic_only_edit_history_event: EditHistoryEvent = {
            "user_id": edit_history_event["user_id"],
            "timestamp": edit_history_event["timestamp"],
        }
        if topic_name is not None:
            topic_only_edit_history_event["prev_topic"] = edit_history_event[
                "prev_topic"]
            topic_only_edit_history_event["topic"] = edit_history_event[
                "topic"]
        if new_stream is not None:
            topic_only_edit_history_event["prev_stream"] = edit_history_event[
                "prev_stream"]
            topic_only_edit_history_event["stream"] = edit_history_event[
                "stream"]

        messages_list = update_messages_for_topic_edit(
            acting_user=user_profile,
            edited_message=target_message,
            propagate_mode=propagate_mode,
            orig_topic_name=orig_topic_name,
            topic_name=topic_name,
            new_stream=new_stream,
            old_stream=stream_being_edited,
            edit_history_event=topic_only_edit_history_event,
            last_edit_time=timestamp,
        )
        changed_messages += messages_list

        if new_stream is not None:
            assert stream_being_edited is not None
            changed_message_ids = [msg.id for msg in changed_messages]

            if subs_gaining_usermessages:
                ums_to_create = []
                for message_id in changed_message_ids:
                    for user_profile_id in subs_gaining_usermessages:
                        # The fact that the user didn't have a UserMessage originally means we can infer that the user
                        # was not mentioned in the original message (even if mention syntax was present, it would not
                        # take effect for a user who was not subscribed). If we were editing the message's content, we
                        # would rerender the message and then use the new stream's data to determine whether this is
                        # a mention of a subscriber; but as we are not doing so, we choose to preserve the "was this
                        # mention syntax an actual mention" decision made during the original rendering for implementation
                        # simplicity. As a result, the only flag to consider applying here is read.
                        um = UserMessageLite(
                            user_profile_id=user_profile_id,
                            message_id=message_id,
                            flags=UserMessage.flags.read,
                        )
                        ums_to_create.append(um)
                bulk_insert_ums(ums_to_create)

            # Delete UserMessage objects for users who will no
            # longer have access to these messages.  Note: This could be
            # very expensive, since it's N guest users x M messages.
            UserMessage.objects.filter(
                user_profile_id__in=[
                    sub.user_profile_id for sub in subs_losing_usermessages
                ],
                message_id__in=changed_message_ids,
            ).delete()

            delete_event: DeleteMessagesEvent = {
                "type": "delete_message",
                "message_ids": changed_message_ids,
                "message_type": "stream",
                "stream_id": stream_being_edited.id,
                "topic": orig_topic_name,
            }
            delete_event_notify_user_ids = [
                sub.user_profile_id for sub in subs_losing_access
            ]
            send_event(user_profile.realm, delete_event,
                       delete_event_notify_user_ids)

            # Reset the Attachment.is_*_public caches for all messages
            # moved to another stream with different access permissions.
            if new_stream.invite_only != stream_being_edited.invite_only:
                Attachment.objects.filter(
                    messages__in=changed_message_ids).update(
                        is_realm_public=None, )
                ArchivedAttachment.objects.filter(
                    messages__in=changed_message_ids).update(
                        is_realm_public=None, )

            if new_stream.is_web_public != stream_being_edited.is_web_public:
                Attachment.objects.filter(
                    messages__in=changed_message_ids).update(
                        is_web_public=None, )
                ArchivedAttachment.objects.filter(
                    messages__in=changed_message_ids).update(
                        is_web_public=None, )

    # This does message.save(update_fields=[...])
    save_message_for_edit_use_case(message=target_message)

    realm_id: Optional[int] = None
    if stream_being_edited is not None:
        realm_id = stream_being_edited.realm_id

    event["message_ids"] = update_to_dict_cache(changed_messages, realm_id)

    def user_info(um: UserMessage) -> Dict[str, Any]:
        return {
            "id": um.user_profile_id,
            "flags": um.flags_list(),
        }

    # The following blocks arranges that users who are subscribed to a
    # stream and can see history from before they subscribed get
    # live-update when old messages are edited (e.g. if the user does
    # a topic edit themself).
    #
    # We still don't send an update event to users who are not
    # subscribed to this stream and don't have a UserMessage row. This
    # means if a non-subscriber is viewing the narrow, they won't get
    # a real-time updates. This is a balance between sending
    # message-edit notifications for every public stream to every user
    # in the organization (too expansive, and also not what we do for
    # newly sent messages anyway) and having magical live-updates
    # where possible.
    users_to_be_notified = list(map(user_info, ums))
    if stream_being_edited is not None:
        if stream_being_edited.is_history_public_to_subscribers:
            subscriptions = get_active_subscriptions_for_stream_id(
                stream_id, include_deactivated_users=False)
            # We exclude long-term idle users, since they by
            # definition have no active clients.
            subscriptions = subscriptions.exclude(
                user_profile__long_term_idle=True)
            # Remove duplicates by excluding the id of users already
            # in users_to_be_notified list.  This is the case where a
            # user both has a UserMessage row and is a current
            # Subscriber
            subscriptions = subscriptions.exclude(
                user_profile_id__in=[um.user_profile_id for um in ums])

            if new_stream is not None:
                assert delete_event_notify_user_ids is not None
                subscriptions = subscriptions.exclude(
                    user_profile_id__in=delete_event_notify_user_ids)

            # All users that are subscribed to the stream must be
            # notified when a message is edited
            subscriber_ids = set(
                subscriptions.values_list("user_profile_id", flat=True))

            if new_stream is not None:
                # TODO: Guest users don't see the new moved topic
                # unless breadcrumb message for new stream is
                # enabled. Excluding these users from receiving this
                # event helps us avoid a error traceback for our
                # clients. We should figure out a way to inform the
                # guest users of this new topic if sending a 'message'
                # event for these messages is not an option.
                #
                # Don't send this event to guest subs who are not
                # subscribers of the old stream but are subscribed to
                # the new stream; clients will be confused.
                old_stream_unsubbed_guests = [
                    sub for sub in subs_to_new_stream
                    if sub.user_profile.is_guest
                    and sub.user_profile_id not in subscriber_ids
                ]
                subscriptions = subscriptions.exclude(user_profile_id__in=[
                    sub.user_profile_id for sub in old_stream_unsubbed_guests
                ])
                subscriber_ids = set(
                    subscriptions.values_list("user_profile_id", flat=True))

            users_to_be_notified += list(
                map(subscriber_info, sorted(list(subscriber_ids))))

    # UserTopic updates and the content of notifications depend on
    # whether we've moved the entire topic, or just part of it. We
    # make that determination here.
    moved_all_visible_messages = False
    if topic_name is not None or new_stream is not None:
        assert stream_being_edited is not None

        if propagate_mode == "change_all":
            moved_all_visible_messages = True
        else:
            # With other propagate modes, if the user in fact moved
            # all messages in the stream, we want to explain it was a
            # full-topic move.
            #
            # For security model reasons, we don't want to allow a
            # user to take any action that would leak information
            # about older messages they cannot access (E.g. the only
            # remaining messages are in a stream without shared
            # history). The bulk_access_messages call below addresses
            # that concern.
            #
            # bulk_access_messages is inefficient for this task, since
            # we just want to do the exists() version of this
            # query. But it's nice to reuse code, and this bulk
            # operation is likely cheaper than a `GET /messages`
            # unless the topic has thousands of messages of history.
            unmoved_messages = messages_for_topic(
                stream_being_edited.recipient_id,
                orig_topic_name,
            )
            visible_unmoved_messages = bulk_access_messages(
                user_profile, unmoved_messages, stream=stream_being_edited)
            moved_all_visible_messages = len(visible_unmoved_messages) == 0

    # Migrate muted topic configuration in the following circumstances:
    #
    # * If propagate_mode is change_all, do so unconditionally.
    #
    # * If propagate_mode is change_later or change_one, do so when
    #   the acting user has moved the entire topic (as visible to them).
    #
    # This rule corresponds to checking moved_all_visible_messages.
    #
    # We may want more complex behavior in cases where one appears to
    # be merging topics (E.g. there are existing messages in the
    # target topic).
    if moved_all_visible_messages:
        assert stream_being_edited is not None
        assert topic_name is not None or new_stream is not None

        for muting_user in get_users_muting_topic(stream_being_edited.id,
                                                  orig_topic_name):
            # TODO: Ideally, this would be a bulk update operation,
            # because we are doing database operations in a loop here.
            #
            # This loop is only acceptable in production because it is
            # rare for more than a few users to have muted an
            # individual topic that is being moved; as of this
            # writing, no individual topic in Zulip Cloud had been
            # muted by more than 100 users.

            if new_stream is not None and muting_user.id in delete_event_notify_user_ids:
                # If the messages are being moved to a stream the user
                # cannot access, then we treat this as the
                # messages/topic being deleted for this user. This is
                # important for security reasons; we don't want to
                # give users a UserTopic row in a stream they cannot
                # access.  Unmute the topic for such users.
                do_unmute_topic(muting_user, stream_being_edited,
                                orig_topic_name)
            else:
                # Otherwise, we move the muted topic record for the user.
                # We call remove_topic_mute rather than do_unmute_topic to
                # avoid sending two events with new muted topics in
                # immediate succession; this is correct only because
                # muted_topics events always send the full set of topics.
                remove_topic_mute(muting_user, stream_being_edited.id,
                                  orig_topic_name)
                do_mute_topic(
                    muting_user,
                    new_stream
                    if new_stream is not None else stream_being_edited,
                    topic_name if topic_name is not None else orig_topic_name,
                    ignore_duplicate=True,
                )

    send_event(user_profile.realm, event, users_to_be_notified)

    if len(
            changed_messages
    ) > 0 and new_stream is not None and stream_being_edited is not None:
        # Notify users that the topic was moved.
        changed_messages_count = len(changed_messages)

        old_thread_notification_string = None
        if send_notification_to_old_thread:
            if moved_all_visible_messages:
                old_thread_notification_string = gettext_lazy(
                    "This topic was moved to {new_location} by {user}.")
            elif changed_messages_count == 1:
                old_thread_notification_string = gettext_lazy(
                    "A message was moved from this topic to {new_location} by {user}."
                )
            else:
                old_thread_notification_string = gettext_lazy(
                    "{changed_messages_count} messages were moved from this topic to {new_location} by {user}."
                )

        new_thread_notification_string = None
        if send_notification_to_new_thread:
            if moved_all_visible_messages:
                new_thread_notification_string = gettext_lazy(
                    "This topic was moved here from {old_location} by {user}.")
            elif changed_messages_count == 1:
                new_thread_notification_string = gettext_lazy(
                    "A message was moved here from {old_location} by {user}.")
            else:
                new_thread_notification_string = gettext_lazy(
                    "{changed_messages_count} messages were moved here from {old_location} by {user}."
                )

        send_message_moved_breadcrumbs(
            user_profile,
            stream_being_edited,
            orig_topic_name,
            old_thread_notification_string,
            new_stream,
            topic_name,
            new_thread_notification_string,
            changed_messages_count,
        )

    if (topic_name is not None and new_stream is None and content is None
            and len(changed_messages) > 0):
        assert stream_being_edited is not None
        maybe_send_resolve_topic_notifications(
            user_profile=user_profile,
            stream=stream_being_edited,
            old_topic=orig_topic_name,
            new_topic=topic_name,
            changed_messages=changed_messages,
        )

    return len(changed_messages)
示例#47
0
    def handle(self, **options: Any) -> None:
        if options["percent_huddles"] + options["percent_personals"] > 100:
            self.stderr.write(
                "Error!  More than 100% of messages allocated.\n")
            return

        if options["delete"]:
            # Start by clearing all the data in our database
            clear_database()

            # Create our two default realms
            # Could in theory be done via zerver.lib.actions.do_create_realm, but
            # welcome-bot (needed for do_create_realm) hasn't been created yet
            zulip_realm = Realm.objects.create(
                string_id="zulip",
                name="Zulip Dev",
                restricted_to_domain=True,
                description=
                "The Zulip development environment default organization."
                "  It's great for testing!",
                invite_required=False,
                org_type=Realm.CORPORATE)
            RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
            if options["test_suite"]:
                mit_realm = Realm.objects.create(string_id="zephyr",
                                                 name="MIT",
                                                 restricted_to_domain=True,
                                                 invite_required=False,
                                                 org_type=Realm.CORPORATE)
                RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")

                lear_realm = Realm.objects.create(string_id="lear",
                                                  name="Lear & Co.",
                                                  restricted_to_domain=False,
                                                  invite_required=False,
                                                  org_type=Realm.CORPORATE)

            # Create test Users (UserProfiles are automatically created,
            # as are subscriptions to the ability to receive personals).
            names = [
                ("Zoe", "*****@*****.**"),
                ("Othello, the Moor of Venice", "*****@*****.**"),
                ("Iago", "*****@*****.**"),
                ("Prospero from The Tempest", "*****@*****.**"),
                ("Cordelia Lear", "*****@*****.**"),
                ("King Hamlet", "*****@*****.**"),
                ("aaron", "*****@*****.**"),
                ("Polonius", "*****@*****.**"),
            ]
            for i in range(options["extra_users"]):
                names.append(
                    ('Extra User %d' % (i, ), '*****@*****.**' % (i, )))
            create_users(zulip_realm, names)

            iago = get_user("*****@*****.**", zulip_realm)
            do_change_is_admin(iago, True)
            iago.is_staff = True
            iago.save(update_fields=['is_staff'])

            guest_user = get_user("*****@*****.**", zulip_realm)
            guest_user.is_guest = True
            guest_user.save(update_fields=['is_guest'])

            # These bots are directly referenced from code and thus
            # are needed for the test suite.
            all_realm_bots = [
                (bot['name'],
                 bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN, ))
                for bot in settings.INTERNAL_BOTS
            ]
            zulip_realm_bots = [
                ("Zulip New User Bot", "*****@*****.**"),
                ("Zulip Error Bot", "*****@*****.**"),
                ("Zulip Default Bot", "*****@*****.**"),
                ("Welcome Bot", "*****@*****.**"),
            ]

            for i in range(options["extra_bots"]):
                zulip_realm_bots.append(
                    ('Extra Bot %d' % (i, ), '*****@*****.**' % (i, )))
            zulip_realm_bots.extend(all_realm_bots)
            create_users(zulip_realm,
                         zulip_realm_bots,
                         bot_type=UserProfile.DEFAULT_BOT)

            zoe = get_user("*****@*****.**", zulip_realm)
            zulip_webhook_bots = [
                ("Zulip Webhook Bot", "*****@*****.**"),
            ]
            # If a stream is not supplied in the webhook URL, the webhook
            # will (in some cases) send the notification as a PM to the
            # owner of the webhook bot, so bot_owner can't be None
            create_users(zulip_realm,
                         zulip_webhook_bots,
                         bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
                         bot_owner=zoe)
            aaron = get_user("*****@*****.**", zulip_realm)
            zulip_outgoing_bots = [("Outgoing Webhook",
                                    "*****@*****.**")]
            create_users(zulip_realm,
                         zulip_outgoing_bots,
                         bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
                         bot_owner=aaron)
            # TODO: Clean up this initial bot creation code
            Service.objects.create(
                name="test",
                user_profile=get_user("*****@*****.**",
                                      zulip_realm),
                base_url="http://127.0.0.1:5002/bots/followup",
                token="abcd1234",
                interface=1)

            # Create public streams.
            stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
            stream_dict = {
                "Verona": {
                    "description": "A city in Italy",
                    "invite_only": False
                },
                "Denmark": {
                    "description": "A Scandinavian country",
                    "invite_only": False
                },
                "Scotland": {
                    "description": "Located in the United Kingdom",
                    "invite_only": False
                },
                "Venice": {
                    "description": "A northeastern Italian city",
                    "invite_only": False
                },
                "Rome": {
                    "description": "Yet another Italian city",
                    "invite_only": False
                }
            }  # type: Dict[Text, Dict[Text, Any]]

            bulk_create_streams(zulip_realm, stream_dict)
            recipient_streams = [
                Stream.objects.get(name=name, realm=zulip_realm).id
                for name in stream_list
            ]  # type: List[int]
            # Create subscriptions to streams.  The following
            # algorithm will give each of the users a different but
            # deterministic subset of the streams (given a fixed list
            # of users).
            subscriptions_to_add = []  # type: List[Subscription]
            event_time = timezone_now()
            all_subscription_logs = []  # type: (List[RealmAuditLog])
            profiles = UserProfile.objects.select_related().filter(
                is_bot=False, is_guest=False).order_by(
                    "email")  # type: Sequence[UserProfile]
            for i, profile in enumerate(profiles):
                # Subscribe to some streams.
                for type_id in recipient_streams[:int(
                        len(recipient_streams) * float(i) / len(profiles)) +
                                                 1]:
                    r = Recipient.objects.get(type=Recipient.STREAM,
                                              type_id=type_id)
                    s = Subscription(recipient=r,
                                     user_profile=profile,
                                     color=STREAM_ASSIGNMENT_COLORS[
                                         i % len(STREAM_ASSIGNMENT_COLORS)])

                    subscriptions_to_add.append(s)

                    log = RealmAuditLog(realm=profile.realm,
                                        modified_user=profile,
                                        modified_stream_id=type_id,
                                        event_last_message_id=0,
                                        event_type='subscription_created',
                                        event_time=event_time)
                    all_subscription_logs.append(log)

            Subscription.objects.bulk_create(subscriptions_to_add)
            RealmAuditLog.objects.bulk_create(all_subscription_logs)

            # Create custom profile field data
            phone_number = try_add_realm_custom_profile_field(
                zulip_realm,
                "Phone number",
                CustomProfileField.SHORT_TEXT,
                hint='')
            biography = try_add_realm_custom_profile_field(
                zulip_realm,
                "Biography",
                CustomProfileField.LONG_TEXT,
                hint='What are you known for?')
            favorite_food = try_add_realm_custom_profile_field(
                zulip_realm,
                "Favorite food",
                CustomProfileField.SHORT_TEXT,
                hint="Or drink, if you'd prefer")

            # Fill in values for Iago and Hamlet
            hamlet = get_user("*****@*****.**", zulip_realm)
            do_update_user_custom_profile_data(iago, [
                {
                    "id": phone_number.id,
                    "value": "+1-234-567-8901"
                },
                {
                    "id": biography.id,
                    "value": "Betrayer of Othello."
                },
                {
                    "id": favorite_food.id,
                    "value": "Apples"
                },
            ])
            do_update_user_custom_profile_data(hamlet, [
                {
                    "id": phone_number.id,
                    "value": "+0-11-23-456-7890"
                },
                {
                    "id": biography.id,
                    "value": "Prince of Denmark, and other things!"
                },
                {
                    "id": favorite_food.id,
                    "value": "Dark chocolate"
                },
            ])
        else:
            zulip_realm = get_realm("zulip")
            recipient_streams = [
                klass.type_id
                for klass in Recipient.objects.filter(type=Recipient.STREAM)
            ]

        # Extract a list of all users
        user_profiles = list(UserProfile.objects.filter(
            is_bot=False))  # type: List[UserProfile]

        # Create a test realm emoji.
        IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images',
                                       'test-images', 'checkbox.png')
        with open(IMAGE_FILE_PATH, 'rb') as fp:
            check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp)

        if not options["test_suite"]:
            # Populate users with some bar data
            for user in user_profiles:
                status = UserPresence.ACTIVE  # type: int
                date = timezone_now()
                client = get_client("website")
                if user.full_name[0] <= 'H':
                    client = get_client("ZulipAndroid")
                UserPresence.objects.get_or_create(user_profile=user,
                                                   client=client,
                                                   timestamp=date,
                                                   status=status)

        user_profiles_ids = [user_profile.id for user_profile in user_profiles]

        # Create several initial huddles
        for i in range(options["num_huddles"]):
            get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))

        # Create several initial pairs for personals
        personals_pairs = [
            random.sample(user_profiles_ids, 2)
            for i in range(options["num_personals"])
        ]

        # Generate a new set of test data.
        create_test_data()

        # prepopulate the URL preview/embed data for the links present
        # in the config.generate_data.json data set.  This makes it
        # possible for populate_db to run happily without Internet
        # access.
        with open("zerver/tests/fixtures/docs_url_preview_data.json",
                  "r") as f:
            urls_with_preview_data = ujson.load(f)
            for url in urls_with_preview_data:
                cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)

        threads = options["threads"]
        jobs = [
        ]  # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
        for i in range(threads):
            count = options["num_messages"] // threads
            if i < options["num_messages"] % threads:
                count += 1
            jobs.append((count, personals_pairs, options, self.stdout.write,
                         random.randint(0, 10**10)))

        for job in jobs:
            send_messages(job)

        if options["delete"]:
            # Create the "website" and "API" clients; if we don't, the
            # default values in zerver/decorators.py will not work
            # with the Django test suite.
            get_client("website")
            get_client("API")

            if options["test_suite"]:
                # Create test users; the MIT ones are needed to test
                # the Zephyr mirroring codepaths.
                testsuite_mit_users = [
                    ("Fred Sipb (MIT)", "*****@*****.**"),
                    ("Athena Consulting Exchange User (MIT)",
                     "*****@*****.**"),
                    ("Esp Classroom (MIT)", "*****@*****.**"),
                ]
                create_users(mit_realm, testsuite_mit_users)

                testsuite_lear_users = [
                    ("King Lear", "*****@*****.**"),
                    ("Cordelia Lear", "*****@*****.**"),
                ]
                create_users(lear_realm, testsuite_lear_users)

            if not options["test_suite"]:
                # Initialize the email gateway bot as an API Super User
                email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
                email_gateway_bot.is_api_super_user = True
                email_gateway_bot.save()

                # To keep the messages.json fixtures file for the test
                # suite fast, don't add these users and subscriptions
                # when running populate_db for the test suite

                zulip_stream_dict = {
                    "devel": {
                        "description": "For developing",
                        "invite_only": False
                    },
                    "all": {
                        "description": "For everything",
                        "invite_only": False
                    },
                    "announce": {
                        "description": "For announcements",
                        "invite_only": False
                    },
                    "design": {
                        "description": "For design",
                        "invite_only": False
                    },
                    "support": {
                        "description": "For support",
                        "invite_only": False
                    },
                    "social": {
                        "description": "For socializing",
                        "invite_only": False
                    },
                    "test": {
                        "description": "For testing",
                        "invite_only": False
                    },
                    "errors": {
                        "description": "For errors",
                        "invite_only": False
                    },
                    "sales": {
                        "description": "For sales discussion",
                        "invite_only": False
                    }
                }  # type: Dict[Text, Dict[Text, Any]]

                # Calculate the maximum number of digits in any extra stream's
                # number, since a stream with name "Extra Stream 3" could show
                # up after "Extra Stream 29". (Used later to pad numbers with
                # 0s).
                maximum_digits = len(str(options['extra_streams'] - 1))

                for i in range(options['extra_streams']):
                    # Pad the number with 0s based on `maximum_digits`.
                    number_str = str(i).zfill(maximum_digits)

                    extra_stream_name = 'Extra Stream ' + number_str

                    zulip_stream_dict[extra_stream_name] = {
                        "description": "Auto-generated extra stream.",
                        "invite_only": False,
                    }

                bulk_create_streams(zulip_realm, zulip_stream_dict)
                # Now that we've created the notifications stream, configure it properly.
                zulip_realm.notifications_stream = get_stream(
                    "announce", zulip_realm)
                zulip_realm.save(update_fields=['notifications_stream'])

                # Add a few default streams
                for default_stream_name in [
                        "design", "devel", "social", "support"
                ]:
                    DefaultStream.objects.create(realm=zulip_realm,
                                                 stream=get_stream(
                                                     default_stream_name,
                                                     zulip_realm))

                # Now subscribe everyone to these streams
                subscriptions_to_add = []
                event_time = timezone_now()
                all_subscription_logs = []
                profiles = UserProfile.objects.select_related().filter(
                    realm=zulip_realm)
                for i, stream_name in enumerate(zulip_stream_dict):
                    stream = Stream.objects.get(name=stream_name,
                                                realm=zulip_realm)
                    recipient = Recipient.objects.get(type=Recipient.STREAM,
                                                      type_id=stream.id)
                    for profile in profiles:
                        # Subscribe to some streams.
                        s = Subscription(
                            recipient=recipient,
                            user_profile=profile,
                            color=STREAM_ASSIGNMENT_COLORS[
                                i % len(STREAM_ASSIGNMENT_COLORS)])
                        subscriptions_to_add.append(s)

                        log = RealmAuditLog(realm=profile.realm,
                                            modified_user=profile,
                                            modified_stream=stream,
                                            event_last_message_id=0,
                                            event_type='subscription_created',
                                            event_time=event_time)
                        all_subscription_logs.append(log)
                Subscription.objects.bulk_create(subscriptions_to_add)
                RealmAuditLog.objects.bulk_create(all_subscription_logs)

                # These bots are not needed by the test suite
                internal_zulip_users_nosubs = [
                    ("Zulip Commit Bot", "*****@*****.**"),
                    ("Zulip Trac Bot", "*****@*****.**"),
                    ("Zulip Nagios Bot", "*****@*****.**"),
                ]
                create_users(zulip_realm,
                             internal_zulip_users_nosubs,
                             bot_type=UserProfile.DEFAULT_BOT)

            zulip_cross_realm_bots = [
                ("Zulip Feedback Bot", "*****@*****.**"),
            ]
            create_users(zulip_realm,
                         zulip_cross_realm_bots,
                         bot_type=UserProfile.DEFAULT_BOT)

            # Mark all messages as read
            UserMessage.objects.all().update(flags=UserMessage.flags.read)

            if not options["test_suite"]:
                # Update pointer of each user to point to the last message in their
                # UserMessage rows with sender_id=user_profile_id.
                users = list(
                    UserMessage.objects.filter(message__sender_id=F(
                        'user_profile_id')).values('user_profile_id').annotate(
                            pointer=Max('message_id')))
                for user in users:
                    UserProfile.objects.filter(
                        id=user['user_profile_id']).update(
                            pointer=user['pointer'])

            create_user_groups()
            self.stdout.write("Successfully populated test database.\n")
示例#48
0
    def test_users_to_zerver_userprofile(
            self, mock_get_data_file: mock.Mock) -> None:
        custom_profile_field_user1 = {
            "Xf06054BBB": {
                "value": "random1"
            },
            "Xf023DSCdd": {
                "value": "employee"
            },
        }
        custom_profile_field_user2 = {
            "Xf06054BBB": {
                "value": "random2"
            },
            "Xf023DSCdd": {
                "value": "employer"
            },
        }
        user_data = [
            {
                "id": "U08RGD1RD",
                "team_id": "T5YFFM2QY",
                "name": "john",
                "deleted": False,
                "is_mirror_dummy": False,
                "real_name": "John Doe",
                "profile": {
                    "image_32": "",
                    "email": "*****@*****.**",
                    "avatar_hash": "hash",
                    "phone": "+1-123-456-77-868",
                    "fields": custom_profile_field_user1,
                },
            },
            {
                "id": "U0CBK5KAT",
                "team_id": "T5YFFM2QY",
                "is_admin": True,
                "is_bot": False,
                "is_owner": True,
                "is_primary_owner": True,
                "name": "Jane",
                "real_name": "Jane Doe",
                "deleted": False,
                "is_mirror_dummy": False,
                "profile": {
                    "image_32":
                    "https://secure.gravatar.com/avatar/random.png",
                    "fields": custom_profile_field_user2,
                    "email": "*****@*****.**",
                    "avatar_hash": "hash",
                },
            },
            {
                "id": "U09TYF5Sk",
                "team_id": "T5YFFM2QY",
                "name": "Bot",
                "real_name": "Bot",
                "is_bot": True,
                "deleted": False,
                "is_mirror_dummy": False,
                "profile": {
                    "image_32":
                    "https://secure.gravatar.com/avatar/random1.png",
                    "skype": "test_skype_name",
                    "email": "*****@*****.**",
                    "avatar_hash": "hash",
                },
            },
            {
                "id": "UHSG7OPQN",
                "team_id": "T6LARQE2Z",
                "name": "matt.perry",
                "color": "9d8eee",
                "is_bot": False,
                "is_app_user": False,
                "is_mirror_dummy": True,
                "team_domain": "foreignteam",
                "profile": {
                    "image_32":
                    "https://secure.gravatar.com/avatar/random6.png",
                    "avatar_hash": "hash",
                    "first_name": "Matt",
                    "last_name": "Perry",
                    "real_name": "Matt Perry",
                    "display_name": "matt.perry",
                    "team": "T6LARQE2Z",
                },
            },
            {
                "id": "U8VAHEVUY",
                "team_id": "T5YFFM2QY",
                "name": "steviejacob34",
                "real_name": "Steve Jacob",
                "is_admin": False,
                "is_owner": False,
                "is_primary_owner": False,
                "is_restricted": True,
                "is_ultra_restricted": False,
                "is_bot": False,
                "is_mirror_dummy": False,
                "profile": {
                    "email": "*****@*****.**",
                    "avatar_hash": "hash",
                    "image_32":
                    "https://secure.gravatar.com/avatar/random6.png",
                },
            },
            {
                "id": "U8X25EBAB",
                "team_id": "T5YFFM2QY",
                "name": "pratikweb_0",
                "real_name": "Pratik",
                "is_admin": False,
                "is_owner": False,
                "is_primary_owner": False,
                "is_restricted": True,
                "is_ultra_restricted": True,
                "is_bot": False,
                "is_mirror_dummy": False,
                "profile": {
                    "email": "*****@*****.**",
                    "avatar_hash": "hash",
                    "image_32":
                    "https://secure.gravatar.com/avatar/random.png",
                },
            },
            {
                "id": "U015J7JSE",
                "team_id": "T5YFFM2QY",
                "name": "georgesm27",
                "real_name": "George",
                "is_admin": True,
                "is_owner": True,
                "is_primary_owner": False,
                "is_restricted": False,
                "is_ultra_restricted": False,
                "is_bot": False,
                "is_mirror_dummy": False,
                "profile": {
                    "email": "*****@*****.**",
                    "avatar_hash": "hash",
                    "image_32":
                    "https://secure.gravatar.com/avatar/random5.png",
                },
            },
            {
                "id": "U1RDFEC80",
                "team_id": "T5YFFM2QY",
                "name": "daniel.smith",
                "real_name": "Daniel Smith",
                "is_admin": True,
                "is_owner": False,
                "is_primary_owner": False,
                "is_restricted": False,
                "is_ultra_restricted": False,
                "is_bot": False,
                "is_mirror_dummy": False,
                "profile": {
                    "email": "*****@*****.**",
                    "avatar_hash": "hash",
                    "image_32":
                    "https://secure.gravatar.com/avatar/random7.png",
                },
            },
        ]

        mock_get_data_file.return_value = user_data
        # As user with slack_id 'U0CBK5KAT' is the primary owner, that user should be imported first
        # and hence has zulip_id = 1
        test_slack_user_id_to_zulip_user_id = {
            "U08RGD1RD": 1,
            "U0CBK5KAT": 0,
            "U09TYF5Sk": 2,
            "UHSG7OPQN": 3,
            "U8VAHEVUY": 4,
            "U8X25EBAB": 5,
            "U015J7JSE": 6,
            "U1RDFEC80": 7,
        }
        slack_data_dir = "./random_path"
        timestamp = int(timezone_now().timestamp())
        mock_get_data_file.return_value = user_data

        with self.assertLogs(level="INFO"):
            (
                zerver_userprofile,
                avatar_list,
                slack_user_id_to_zulip_user_id,
                customprofilefield,
                customprofilefield_value,
            ) = users_to_zerver_userprofile(slack_data_dir, user_data, 1,
                                            timestamp, "test_domain")

        # Test custom profile fields
        self.assertEqual(customprofilefield[0]["field_type"], 1)
        self.assertEqual(customprofilefield[3]["name"], "skype")
        cpf_name = {cpf["name"] for cpf in customprofilefield}
        self.assertIn("phone", cpf_name)
        self.assertIn("skype", cpf_name)
        cpf_name.remove("phone")
        cpf_name.remove("skype")
        for name in cpf_name:
            self.assertTrue(name.startswith("Slack custom field "))

        self.assertEqual(len(customprofilefield_value), 6)
        self.assertEqual(customprofilefield_value[0]["field"], 0)
        self.assertEqual(customprofilefield_value[0]["user_profile"], 1)
        self.assertEqual(customprofilefield_value[3]["user_profile"], 0)
        self.assertEqual(customprofilefield_value[5]["value"],
                         "test_skype_name")

        # test that the primary owner should always be imported first
        self.assertDictEqual(slack_user_id_to_zulip_user_id,
                             test_slack_user_id_to_zulip_user_id)
        self.assertEqual(len(avatar_list), 8)

        self.assertEqual(len(zerver_userprofile), 8)

        self.assertEqual(zerver_userprofile[0]["is_staff"], False)
        self.assertEqual(zerver_userprofile[0]["is_bot"], False)
        self.assertEqual(zerver_userprofile[0]["is_active"], True)
        self.assertEqual(zerver_userprofile[0]["is_mirror_dummy"], False)
        self.assertEqual(zerver_userprofile[0]["role"],
                         UserProfile.ROLE_MEMBER)
        self.assertEqual(zerver_userprofile[0]["enable_desktop_notifications"],
                         True)
        self.assertEqual(zerver_userprofile[0]["email"], "*****@*****.**")
        self.assertEqual(zerver_userprofile[0]["full_name"], "John Doe")

        self.assertEqual(zerver_userprofile[1]["id"],
                         test_slack_user_id_to_zulip_user_id["U0CBK5KAT"])
        self.assertEqual(zerver_userprofile[1]["role"],
                         UserProfile.ROLE_REALM_OWNER)
        self.assertEqual(zerver_userprofile[1]["is_staff"], False)
        self.assertEqual(zerver_userprofile[1]["is_active"], True)
        self.assertEqual(zerver_userprofile[0]["is_mirror_dummy"], False)

        self.assertEqual(zerver_userprofile[2]["id"],
                         test_slack_user_id_to_zulip_user_id["U09TYF5Sk"])
        self.assertEqual(zerver_userprofile[2]["is_bot"], True)
        self.assertEqual(zerver_userprofile[2]["is_active"], True)
        self.assertEqual(zerver_userprofile[2]["is_mirror_dummy"], False)
        self.assertEqual(zerver_userprofile[2]["email"], "*****@*****.**")
        self.assertEqual(zerver_userprofile[2]["bot_type"], 1)
        self.assertEqual(zerver_userprofile[2]["avatar_source"], "U")

        self.assertEqual(zerver_userprofile[3]["id"],
                         test_slack_user_id_to_zulip_user_id["UHSG7OPQN"])
        self.assertEqual(zerver_userprofile[3]["role"],
                         UserProfile.ROLE_MEMBER)
        self.assertEqual(zerver_userprofile[3]["is_staff"], False)
        self.assertEqual(zerver_userprofile[3]["is_active"], False)
        self.assertEqual(zerver_userprofile[3]["email"],
                         "*****@*****.**")
        self.assertEqual(zerver_userprofile[3]["realm"], 1)
        self.assertEqual(zerver_userprofile[3]["full_name"], "Matt Perry")
        self.assertEqual(zerver_userprofile[3]["is_mirror_dummy"], True)
        self.assertEqual(zerver_userprofile[3]["can_forge_sender"], False)

        self.assertEqual(zerver_userprofile[4]["id"],
                         test_slack_user_id_to_zulip_user_id["U8VAHEVUY"])
        self.assertEqual(zerver_userprofile[4]["role"], UserProfile.ROLE_GUEST)
        self.assertEqual(zerver_userprofile[4]["is_staff"], False)
        self.assertEqual(zerver_userprofile[4]["is_active"], True)
        self.assertEqual(zerver_userprofile[4]["is_mirror_dummy"], False)

        self.assertEqual(zerver_userprofile[5]["id"],
                         test_slack_user_id_to_zulip_user_id["U8X25EBAB"])
        self.assertEqual(zerver_userprofile[5]["role"], UserProfile.ROLE_GUEST)
        self.assertEqual(zerver_userprofile[5]["is_staff"], False)
        self.assertEqual(zerver_userprofile[5]["is_active"], True)
        self.assertEqual(zerver_userprofile[5]["is_mirror_dummy"], False)

        self.assertEqual(zerver_userprofile[6]["id"],
                         test_slack_user_id_to_zulip_user_id["U015J7JSE"])
        self.assertEqual(zerver_userprofile[6]["role"],
                         UserProfile.ROLE_REALM_OWNER)
        self.assertEqual(zerver_userprofile[6]["is_staff"], False)
        self.assertEqual(zerver_userprofile[6]["is_active"], True)
        self.assertEqual(zerver_userprofile[6]["is_mirror_dummy"], False)

        self.assertEqual(zerver_userprofile[7]["id"],
                         test_slack_user_id_to_zulip_user_id["U1RDFEC80"])
        self.assertEqual(zerver_userprofile[7]["role"],
                         UserProfile.ROLE_REALM_ADMINISTRATOR)
        self.assertEqual(zerver_userprofile[7]["is_staff"], False)
        self.assertEqual(zerver_userprofile[7]["is_active"], True)
        self.assertEqual(zerver_userprofile[7]["is_mirror_dummy"], False)
示例#49
0
    def test_search(self) -> None:
        reset_emails_in_zulip_realm()

        def check_hamlet_user_query_result(result: HttpResponse) -> None:
            self.assert_in_success_response(['<span class="label">user</span>\n', '<h3>King Hamlet</h3>',
                                             '<b>Email</b>: [email protected]', '<b>Is active</b>: True<br>',
                                             '<b>Admins</b>: [email protected], [email protected]\n',
                                             'class="copy-button" data-copytext="[email protected], [email protected]"',
                                             ], result)

        def check_zulip_realm_query_result(result: HttpResponse) -> None:
            zulip_realm = get_realm("zulip")
            self.assert_in_success_response([f'<input type="hidden" name="realm_id" value="{zulip_realm.id}"',
                                             'Zulip Dev</h3>',
                                             '<option value="1" selected>Self Hosted</option>',
                                             '<option value="2" >Limited</option>',
                                             'input type="number" name="discount" value="None"',
                                             '<option value="active" selected>Active</option>',
                                             '<option value="deactivated" >Deactivated</option>',
                                             'scrub-realm-button">',
                                             'data-string-id="zulip"'], result)

        def check_lear_realm_query_result(result: HttpResponse) -> None:
            lear_realm = get_realm("lear")
            self.assert_in_success_response([f'<input type="hidden" name="realm_id" value="{lear_realm.id}"',
                                             'Lear &amp; Co.</h3>',
                                             '<option value="1" selected>Self Hosted</option>',
                                             '<option value="2" >Limited</option>',
                                             'input type="number" name="discount" value="None"',
                                             '<option value="active" selected>Active</option>',
                                             '<option value="deactivated" >Deactivated</option>',
                                             'scrub-realm-button">',
                                             'data-string-id="lear"',
                                             '<b>Name</b>: Zulip Standard',
                                             '<b>Status</b>: Active',
                                             '<b>Billing schedule</b>: Annual',
                                             '<b>Licenses</b>: 2/10 (Manual)',
                                             '<b>Price per license</b>: $80.0',
                                             '<b>Payment method</b>: Send invoice',
                                             '<b>Next invoice date</b>: 02 January 2017',
                                             ], result)

        def check_preregistration_user_query_result(result: HttpResponse, email: str, invite: bool=False) -> None:
            self.assert_in_success_response(['<span class="label">preregistration user</span>\n',
                                             f'<b>Email</b>: {email}',
                                             ], result)
            if invite:
                self.assert_in_success_response(['<span class="label">invite</span>'], result)
                self.assert_in_success_response(['<b>Expires in</b>: 1\xa0week, 3',
                                                 '<b>Status</b>: Link has never been clicked'], result)
                self.assert_in_success_response([], result)
            else:
                self.assert_not_in_success_response(['<span class="label">invite</span>'], result)
                self.assert_in_success_response(['<b>Expires in</b>: 1\xa0day',
                                                 '<b>Status</b>: Link has never been clicked'], result)

        def check_realm_creation_query_result(result: HttpResponse, email: str) -> None:
            self.assert_in_success_response(['<span class="label">preregistration user</span>\n',
                                             '<span class="label">realm creation</span>\n',
                                             '<b>Link</b>: http://testserver/accounts/do_confirm/',
                                             '<b>Expires in</b>: 1\xa0day<br>\n',
                                             ], result)

        def check_multiuse_invite_link_query_result(result: HttpResponse) -> None:
            self.assert_in_success_response(['<span class="label">multiuse invite</span>\n',
                                             '<b>Link</b>: http://zulip.testserver/join/',
                                             '<b>Expires in</b>: 1\xa0week, 3',
                                             ], result)

        def check_realm_reactivation_link_query_result(result: HttpResponse) -> None:
            self.assert_in_success_response(['<span class="label">realm reactivation</span>\n',
                                             '<b>Link</b>: http://zulip.testserver/reactivate/',
                                             '<b>Expires in</b>: 1\xa0day',
                                             ], result)

        self.login('cordelia')

        result = self.client_get("/activity/support")
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")

        self.login('iago')

        customer = Customer.objects.create(realm=get_realm("lear"), stripe_customer_id='cus_123')
        now = datetime(2016, 1, 2, tzinfo=timezone.utc)
        plan = CustomerPlan.objects.create(customer=customer, billing_cycle_anchor=now,
                                           billing_schedule=CustomerPlan.ANNUAL, tier=CustomerPlan.STANDARD,
                                           price_per_license=8000, next_invoice_date=add_months(now, 12))
        LicenseLedger.objects.create(licenses=10, licenses_at_next_renewal=10, event_time=timezone_now(),
                                     is_renewal=True, plan=plan)

        result = self.client_get("/activity/support")
        self.assert_in_success_response(['<input type="text" name="q" class="input-xxlarge search-query"'], result)

        result = self.client_get("/activity/support", {"q": "*****@*****.**"})
        check_hamlet_user_query_result(result)
        check_zulip_realm_query_result(result)

        result = self.client_get("/activity/support", {"q": "lear"})
        check_lear_realm_query_result(result)

        result = self.client_get("/activity/support", {"q": "http://lear.testserver"})
        check_lear_realm_query_result(result)

        with self.settings(REALM_HOSTS={'zulip': 'localhost'}):
            result = self.client_get("/activity/support", {"q": "http://localhost"})
            check_zulip_realm_query_result(result)

        result = self.client_get("/activity/support", {"q": "[email protected], lear"})
        check_hamlet_user_query_result(result)
        check_zulip_realm_query_result(result)
        check_lear_realm_query_result(result)

        result = self.client_get("/activity/support", {"q": "lear, Hamlet <*****@*****.**>"})
        check_hamlet_user_query_result(result)
        check_zulip_realm_query_result(result)
        check_lear_realm_query_result(result)

        self.client_post('/accounts/home/', {'email': self.nonreg_email("test")})
        self.login('iago')
        result = self.client_get("/activity/support", {"q": self.nonreg_email("test")})
        check_preregistration_user_query_result(result, self.nonreg_email("test"))
        check_zulip_realm_query_result(result)

        stream_ids = [self.get_stream_id("Denmark")]
        invitee_emails = [self.nonreg_email("test1")]
        self.client_post("/json/invites", {"invitee_emails": invitee_emails,
                                           "stream_ids": orjson.dumps(stream_ids).decode(),
                                           "invite_as": PreregistrationUser.INVITE_AS['MEMBER']})
        result = self.client_get("/activity/support", {"q": self.nonreg_email("test1")})
        check_preregistration_user_query_result(result, self.nonreg_email("test1"), invite=True)
        check_zulip_realm_query_result(result)

        email = self.nonreg_email('alice')
        self.client_post('/new/', {'email': email})
        result = self.client_get("/activity/support", {"q": email})
        check_realm_creation_query_result(result, email)

        do_create_multiuse_invite_link(self.example_user("hamlet"), invited_as=1)
        result = self.client_get("/activity/support", {"q": "zulip"})
        check_multiuse_invite_link_query_result(result)
        check_zulip_realm_query_result(result)
        MultiuseInvite.objects.all().delete()

        do_send_realm_reactivation_email(get_realm("zulip"))
        result = self.client_get("/activity/support", {"q": "zulip"})
        check_realm_reactivation_link_query_result(result)
        check_zulip_realm_query_result(result)
示例#50
0
    def handle(self, *args: Any, **options: Any) -> None:
        # TODO: This should arguably only delete the objects
        # associated with the "analytics" realm.
        do_drop_all_analytics_tables()

        # This also deletes any objects with this realm as a foreign key
        Realm.objects.filter(string_id="analytics").delete()

        # Because we just deleted a bunch of objects in the database
        # directly (rather than deleting individual objects in Django,
        # in which case our post_save hooks would have flushed the
        # individual objects from memcached for us), we need to flush
        # memcached in order to ensure deleted objects aren't still
        # present in the memcached cache.
        from zerver.apps import flush_cache

        flush_cache(None)

        installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
        last_end_time = floor_to_day(timezone_now())
        realm = do_create_realm(string_id="analytics",
                                name="Analytics",
                                date_created=installation_time)

        with mock.patch("zerver.lib.create_user.timezone_now",
                        return_value=installation_time):
            shylock = create_user(
                "*****@*****.**",
                "Shylock",
                realm,
                full_name="Shylock",
                role=UserProfile.ROLE_REALM_OWNER,
            )
        do_change_user_role(shylock,
                            UserProfile.ROLE_REALM_OWNER,
                            acting_user=None)
        stream = Stream.objects.create(name="all",
                                       realm=realm,
                                       date_created=installation_time)
        recipient = Recipient.objects.create(type_id=stream.id,
                                             type=Recipient.STREAM)
        stream.recipient = recipient
        stream.save(update_fields=["recipient"])

        # Subscribe shylock to the stream to avoid invariant failures.
        # TODO: This should use subscribe_users_to_streams from populate_db.
        subs = [
            Subscription(
                recipient=recipient,
                user_profile=shylock,
                is_user_active=shylock.is_active,
                color=STREAM_ASSIGNMENT_COLORS[0],
            ),
        ]
        Subscription.objects.bulk_create(subs)

        FixtureData = Mapping[Union[str, int, None], List[int]]

        def insert_fixture_data(
            stat: CountStat,
            fixture_data: FixtureData,
            table: Type[BaseCount],
        ) -> None:
            end_times = time_range(last_end_time, last_end_time,
                                   stat.frequency,
                                   len(list(fixture_data.values())[0]))
            if table == InstallationCount:
                id_args: Dict[str, Any] = {}
            if table == RealmCount:
                id_args = {"realm": realm}
            if table == UserCount:
                id_args = {"realm": realm, "user": shylock}
            if table == StreamCount:
                id_args = {"stream": stream, "realm": realm}

            for subgroup, values in fixture_data.items():
                table.objects.bulk_create(
                    table(
                        property=stat.property,
                        subgroup=subgroup,
                        end_time=end_time,
                        value=value,
                        **id_args,
                    ) for end_time, value in zip(end_times, values)
                    if value != 0)

        stat = COUNT_STATS["1day_actives::day"]
        realm_data: FixtureData = {
            None:
            self.generate_fixture_data(stat,
                                       0.08,
                                       0.02,
                                       3,
                                       0.3,
                                       6,
                                       partial_sum=True),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data: FixtureData = {
            None:
            self.generate_fixture_data(stat,
                                       0.8,
                                       0.2,
                                       4,
                                       0.3,
                                       6,
                                       partial_sum=True),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property,
                                 end_time=last_end_time,
                                 state=FillState.DONE)

        stat = COUNT_STATS["7day_actives::day"]
        realm_data = {
            None:
            self.generate_fixture_data(stat,
                                       0.2,
                                       0.07,
                                       3,
                                       0.3,
                                       6,
                                       partial_sum=True),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            None:
            self.generate_fixture_data(stat,
                                       2,
                                       0.7,
                                       4,
                                       0.3,
                                       6,
                                       partial_sum=True),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property,
                                 end_time=last_end_time,
                                 state=FillState.DONE)

        stat = COUNT_STATS["realm_active_humans::day"]
        realm_data = {
            None:
            self.generate_fixture_data(stat,
                                       0.8,
                                       0.08,
                                       3,
                                       0.5,
                                       3,
                                       partial_sum=True),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            None:
            self.generate_fixture_data(stat,
                                       1,
                                       0.3,
                                       4,
                                       0.5,
                                       3,
                                       partial_sum=True),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property,
                                 end_time=last_end_time,
                                 state=FillState.DONE)

        stat = COUNT_STATS["active_users_audit:is_bot:day"]
        realm_data = {
            "false":
            self.generate_fixture_data(stat,
                                       1,
                                       0.2,
                                       3.5,
                                       0.8,
                                       2,
                                       partial_sum=True),
            "true":
            self.generate_fixture_data(stat,
                                       0.3,
                                       0.05,
                                       3,
                                       0.3,
                                       2,
                                       partial_sum=True),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            "false":
            self.generate_fixture_data(stat, 3, 1, 4, 0.8, 2,
                                       partial_sum=True),
            "true":
            self.generate_fixture_data(stat,
                                       1,
                                       0.4,
                                       4,
                                       0.8,
                                       2,
                                       partial_sum=True),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property,
                                 end_time=last_end_time,
                                 state=FillState.DONE)

        stat = COUNT_STATS["messages_sent:is_bot:hour"]
        user_data: FixtureData = {
            "false":
            self.generate_fixture_data(stat,
                                       2,
                                       1,
                                       1.5,
                                       0.6,
                                       8,
                                       holiday_rate=0.1),
        }
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {
            "false": self.generate_fixture_data(stat, 35, 15, 6, 0.6, 4),
            "true": self.generate_fixture_data(stat, 15, 15, 3, 0.4, 2),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            "false": self.generate_fixture_data(stat, 350, 150, 6, 0.6, 4),
            "true": self.generate_fixture_data(stat, 150, 150, 3, 0.4, 2),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property,
                                 end_time=last_end_time,
                                 state=FillState.DONE)

        stat = COUNT_STATS["messages_sent:message_type:day"]
        user_data = {
            "public_stream":
            self.generate_fixture_data(stat, 1.5, 1, 3, 0.6, 8),
            "private_message":
            self.generate_fixture_data(stat, 0.5, 0.3, 1, 0.6, 8),
            "huddle_message":
            self.generate_fixture_data(stat, 0.2, 0.2, 2, 0.6, 8),
        }
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {
            "public_stream":
            self.generate_fixture_data(stat, 30, 8, 5, 0.6, 4),
            "private_stream":
            self.generate_fixture_data(stat, 7, 7, 5, 0.6, 4),
            "private_message":
            self.generate_fixture_data(stat, 13, 5, 5, 0.6, 4),
            "huddle_message":
            self.generate_fixture_data(stat, 6, 3, 3, 0.6, 4),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            "public_stream":
            self.generate_fixture_data(stat, 300, 80, 5, 0.6, 4),
            "private_stream":
            self.generate_fixture_data(stat, 70, 70, 5, 0.6, 4),
            "private_message":
            self.generate_fixture_data(stat, 130, 50, 5, 0.6, 4),
            "huddle_message":
            self.generate_fixture_data(stat, 60, 30, 3, 0.6, 4),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property,
                                 end_time=last_end_time,
                                 state=FillState.DONE)

        website, created = Client.objects.get_or_create(name="website")
        old_desktop, created = Client.objects.get_or_create(
            name="desktop app Linux 0.3.7")
        android, created = Client.objects.get_or_create(name="ZulipAndroid")
        iOS, created = Client.objects.get_or_create(name="ZulipiOS")
        react_native, created = Client.objects.get_or_create(
            name="ZulipMobile")
        API, created = Client.objects.get_or_create(name="API: Python")
        zephyr_mirror, created = Client.objects.get_or_create(
            name="zephyr_mirror")
        unused, created = Client.objects.get_or_create(name="unused")
        long_webhook, created = Client.objects.get_or_create(
            name="ZulipLooooooooooongNameWebhook")

        stat = COUNT_STATS["messages_sent:client:day"]
        user_data = {
            website.id:
            self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8),
            zephyr_mirror.id:
            self.generate_fixture_data(stat, 0, 0.3, 1.5, 0.6, 8),
        }
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {
            website.id: self.generate_fixture_data(stat, 30, 20, 5, 0.6, 3),
            old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, 0.6, 3),
            android.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
            iOS.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
            react_native.id:
            self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
            API.id: self.generate_fixture_data(stat, 5, 5, 5, 0.6, 3),
            zephyr_mirror.id:
            self.generate_fixture_data(stat, 1, 1, 3, 0.6, 3),
            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
            long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            website.id: self.generate_fixture_data(stat, 300, 200, 5, 0.6, 3),
            old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, 0.6,
                                                       3),
            android.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
            iOS.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
            react_native.id:
            self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
            API.id: self.generate_fixture_data(stat, 50, 50, 5, 0.6, 3),
            zephyr_mirror.id:
            self.generate_fixture_data(stat, 10, 10, 3, 0.6, 3),
            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
            long_webhook.id:
            self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property,
                                 end_time=last_end_time,
                                 state=FillState.DONE)

        stat = COUNT_STATS["messages_in_stream:is_bot:day"]
        realm_data = {
            "false": self.generate_fixture_data(stat, 30, 5, 6, 0.6, 4),
            "true": self.generate_fixture_data(stat, 20, 2, 3, 0.2, 3),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        stream_data: Mapping[Union[int, str, None], List[int]] = {
            "false": self.generate_fixture_data(stat, 10, 7, 5, 0.6, 4),
            "true": self.generate_fixture_data(stat, 5, 3, 2, 0.4, 2),
        }
        insert_fixture_data(stat, stream_data, StreamCount)
        FillState.objects.create(property=stat.property,
                                 end_time=last_end_time,
                                 state=FillState.DONE)

        stat = COUNT_STATS["messages_read::hour"]
        user_data = {
            None:
            self.generate_fixture_data(stat, 7, 3, 2, 0.6, 8,
                                       holiday_rate=0.1),
        }
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {
            None: self.generate_fixture_data(stat, 50, 35, 6, 0.6, 4)
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        FillState.objects.create(property=stat.property,
                                 end_time=last_end_time,
                                 state=FillState.DONE)
示例#51
0
def is_analytics_ready(realm: Realm) -> bool:
    return (timezone_now() -
            realm.date_created) > MAX_TIME_FOR_FULL_ANALYTICS_GENERATION
示例#52
0
    def consume(self, event: Dict[str, Any]) -> None:
        start = time.time()
        if event["type"] == "mark_stream_messages_as_read":
            user_profile = get_user_profile_by_id(event["user_profile_id"])

            for recipient_id in event["stream_recipient_ids"]:
                do_mark_stream_messages_as_read(user_profile, recipient_id)
        elif event["type"] == "mark_stream_messages_as_read_for_everyone":
            # This event is generated by the stream deactivation code path.
            batch_size = 100
            offset = 0
            while True:
                messages = Message.objects.filter(
                    recipient_id=event["stream_recipient_id"]
                ).order_by("id")[offset : offset + batch_size]
                UserMessage.objects.filter(message__in=messages).extra(
                    where=[UserMessage.where_unread()]
                ).update(flags=F("flags").bitor(UserMessage.flags.read))
                offset += len(messages)
                if len(messages) < batch_size:
                    break
        elif event["type"] == "clear_push_device_tokens":
            try:
                clear_push_device_tokens(event["user_profile_id"])
            except PushNotificationBouncerRetryLaterError:

                def failure_processor(event: Dict[str, Any]) -> None:
                    logger.warning(
                        "Maximum retries exceeded for trigger:%s event:clear_push_device_tokens",
                        event["user_profile_id"],
                    )

                retry_event(self.queue_name, event, failure_processor)
        elif event["type"] == "realm_export":
            realm = Realm.objects.get(id=event["realm_id"])
            output_dir = tempfile.mkdtemp(prefix="zulip-export-")
            export_event = RealmAuditLog.objects.get(id=event["id"])
            user_profile = get_user_profile_by_id(event["user_profile_id"])

            try:
                public_url = export_realm_wrapper(
                    realm=realm,
                    output_dir=output_dir,
                    threads=6,
                    upload=True,
                    public_only=True,
                    delete_after_upload=True,
                )
            except Exception:
                export_event.extra_data = orjson.dumps(
                    dict(
                        failed_timestamp=timezone_now().timestamp(),
                    )
                ).decode()
                export_event.save(update_fields=["extra_data"])
                logging.error(
                    "Data export for %s failed after %s",
                    user_profile.realm.string_id,
                    time.time() - start,
                )
                notify_realm_export(user_profile)
                return

            assert public_url is not None

            # Update the extra_data field now that the export is complete.
            export_event.extra_data = orjson.dumps(
                dict(
                    export_path=urllib.parse.urlparse(public_url).path,
                )
            ).decode()
            export_event.save(update_fields=["extra_data"])

            # Send a private message notification letting the user who
            # triggered the export know the export finished.
            with override_language(user_profile.default_language):
                content = _(
                    "Your data export is complete and has been uploaded here:\n\n{public_url}"
                ).format(public_url=public_url)
            internal_send_private_message(
                sender=get_system_bot(settings.NOTIFICATION_BOT),
                recipient_user=user_profile,
                content=content,
            )

            # For future frontend use, also notify administrator
            # clients that the export happened.
            notify_realm_export(user_profile)
            logging.info(
                "Completed data export for %s in %s",
                user_profile.realm.string_id,
                time.time() - start,
            )

        end = time.time()
        logger.info("deferred_work processed %s event (%dms)", event["type"], (end - start) * 1000)
示例#53
0
    def handle(self, *args: Any, **options: Any) -> None:
        # Suppress spammy output from the push notifications logger
        push_notifications_logger.disabled = True

        if options["percent_huddles"] + options["percent_personals"] > 100:
            self.stderr.write("Error!  More than 100% of messages allocated.\n")
            return

        # Get consistent data for backend tests.
        if options["test_suite"]:
            random.seed(0)

            with connection.cursor() as cursor:
                # Sometimes bugs relating to confusing recipient.id for recipient.type_id
                # or <object>.id for <object>.recipient_id remain undiscovered by the test suite
                # due to these numbers happening to coincide in such a way that it makes tests
                # accidentally pass. By bumping the Recipient.id sequence by a large enough number,
                # we can have those ids in a completely different range of values than object ids,
                # eliminating the possibility of such coincidences.
                cursor.execute("SELECT setval('zerver_recipient_id_seq', 100)")

        if options["max_topics"] is None:
            # If max_topics is not set, we use a default that's big
            # enough "more topics" should appear, and scales slowly
            # with the number of messages.
            options["max_topics"] = 8 + options["num_messages"] // 1000

        if options["delete"]:
            # Start by clearing all the data in our database
            clear_database()

            # Create our three default realms
            # Could in theory be done via zerver.actions.create_realm.do_create_realm, but
            # welcome-bot (needed for do_create_realm) hasn't been created yet
            create_internal_realm()
            zulip_realm = do_create_realm(
                string_id="zulip",
                name="Zulip Dev",
                emails_restricted_to_domains=False,
                email_address_visibility=Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS,
                description="The Zulip development environment default organization."
                "  It's great for testing!",
                invite_required=False,
                plan_type=Realm.PLAN_TYPE_SELF_HOSTED,
                org_type=Realm.ORG_TYPES["business"]["id"],
                enable_spectator_access=True,
            )
            RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
            assert zulip_realm.notifications_stream is not None
            zulip_realm.notifications_stream.name = "Verona"
            zulip_realm.notifications_stream.description = "A city in Italy"
            zulip_realm.notifications_stream.save(update_fields=["name", "description"])

            realm_user_default = RealmUserDefault.objects.get(realm=zulip_realm)
            realm_user_default.enter_sends = True
            realm_user_default.save()

            if options["test_suite"]:
                mit_realm = do_create_realm(
                    string_id="zephyr",
                    name="MIT",
                    emails_restricted_to_domains=True,
                    invite_required=False,
                    plan_type=Realm.PLAN_TYPE_SELF_HOSTED,
                    org_type=Realm.ORG_TYPES["business"]["id"],
                )
                RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")

                lear_realm = do_create_realm(
                    string_id="lear",
                    name="Lear & Co.",
                    emails_restricted_to_domains=False,
                    invite_required=False,
                    plan_type=Realm.PLAN_TYPE_SELF_HOSTED,
                    org_type=Realm.ORG_TYPES["business"]["id"],
                )

                # Default to allowing all members to send mentions in
                # large streams for the test suite to keep
                # mention-related tests simple.
                zulip_realm.wildcard_mention_policy = Realm.WILDCARD_MENTION_POLICY_MEMBERS
                zulip_realm.save(update_fields=["wildcard_mention_policy"])

            # Create test Users (UserProfiles are automatically created,
            # as are subscriptions to the ability to receive personals).
            names = [
                ("Zoe", "*****@*****.**"),
                ("Othello, the Moor of Venice", "*****@*****.**"),
                ("Iago", "*****@*****.**"),
                ("Prospero from The Tempest", "*****@*****.**"),
                ("Cordelia, Lear's daughter", "*****@*****.**"),
                ("King Hamlet", "*****@*****.**"),
                ("aaron", "*****@*****.**"),
                ("Polonius", "*****@*****.**"),
                ("Desdemona", "*****@*****.**"),
                ("शिव", "*****@*****.**"),
            ]

            # For testing really large batches:
            # Create extra users with semi realistic names to make search
            # functions somewhat realistic.  We'll still create 1000 users
            # like Extra222 User for some predictability.
            num_names = options["extra_users"]
            num_boring_names = 300

            for i in range(min(num_names, num_boring_names)):
                full_name = f"Extra{i:03} User"
                names.append((full_name, f"extrauser{i}@zulip.com"))

            if num_names > num_boring_names:
                fnames = [
                    "Amber",
                    "Arpita",
                    "Bob",
                    "Cindy",
                    "Daniela",
                    "Dan",
                    "Dinesh",
                    "Faye",
                    "François",
                    "George",
                    "Hank",
                    "Irene",
                    "James",
                    "Janice",
                    "Jenny",
                    "Jill",
                    "John",
                    "Kate",
                    "Katelyn",
                    "Kobe",
                    "Lexi",
                    "Manish",
                    "Mark",
                    "Matt",
                    "Mayna",
                    "Michael",
                    "Pete",
                    "Peter",
                    "Phil",
                    "Phillipa",
                    "Preston",
                    "Sally",
                    "Scott",
                    "Sandra",
                    "Steve",
                    "Stephanie",
                    "Vera",
                ]
                mnames = ["de", "van", "von", "Shaw", "T."]
                lnames = [
                    "Adams",
                    "Agarwal",
                    "Beal",
                    "Benson",
                    "Bonita",
                    "Davis",
                    "George",
                    "Harden",
                    "James",
                    "Jones",
                    "Johnson",
                    "Jordan",
                    "Lee",
                    "Leonard",
                    "Singh",
                    "Smith",
                    "Patel",
                    "Towns",
                    "Wall",
                ]
                non_ascii_names = [
                    "Günter",
                    "أحمد",
                    "Magnús",
                    "आशी",
                    "イツキ",
                    "语嫣",
                    "அருண்",
                    "Александр",
                    "José",
                ]
                # to imitate emoji insertions in usernames
                raw_emojis = ["😎", "😂", "🐱‍👤"]

            for i in range(num_boring_names, num_names):
                fname = random.choice(fnames) + str(i)
                full_name = fname
                if random.random() < 0.7:
                    if random.random() < 0.3:
                        full_name += " " + random.choice(non_ascii_names)
                    else:
                        full_name += " " + random.choice(mnames)
                    if random.random() < 0.1:
                        full_name += f" {random.choice(raw_emojis)} "
                    else:
                        full_name += " " + random.choice(lnames)
                email = fname.lower() + "@zulip.com"
                names.append((full_name, email))

            create_users(zulip_realm, names, tos_version=settings.TERMS_OF_SERVICE_VERSION)

            # Add time zones to some users. Ideally, this would be
            # done in the initial create_users calls, but the
            # tuple-based interface for that function doesn't support
            # doing so.
            def assign_time_zone_by_delivery_email(delivery_email: str, new_time_zone: str) -> None:
                u = get_user_by_delivery_email(delivery_email, zulip_realm)
                u.timezone = new_time_zone
                u.save(update_fields=["timezone"])

            # Note: Hamlet keeps default time zone of "".
            assign_time_zone_by_delivery_email("*****@*****.**", "US/Pacific")
            assign_time_zone_by_delivery_email("*****@*****.**", "US/Pacific")
            assign_time_zone_by_delivery_email("*****@*****.**", "US/Eastern")
            assign_time_zone_by_delivery_email("*****@*****.**", "US/Eastern")
            assign_time_zone_by_delivery_email("*****@*****.**", "Canada/Newfoundland")
            assign_time_zone_by_delivery_email("*****@*****.**", "Asia/Shanghai")  # China
            assign_time_zone_by_delivery_email("*****@*****.**", "Asia/Kolkata")  # India
            assign_time_zone_by_delivery_email("*****@*****.**", "UTC")

            users = UserProfile.objects.filter(realm=zulip_realm)
            # All users in development environment are full members initially because
            # waiting period threshold is 0. Groups of Iago, Dedemona, Shiva and
            # Polonius will be updated according to their role in do_change_user_role.
            full_members_user_group = UserGroup.objects.get(
                realm=zulip_realm, name="@role:fullmembers", is_system_group=True
            )
            members_user_group = UserGroup.objects.get(
                realm=zulip_realm, name="@role:members", is_system_group=True
            )
            user_group_memberships = []
            for user_profile in list(users):
                for group in [full_members_user_group, members_user_group]:
                    user_group_membership = UserGroupMembership(
                        user_group=group, user_profile=user_profile
                    )
                    user_group_memberships.append(user_group_membership)
            UserGroupMembership.objects.bulk_create(user_group_memberships)

            iago = get_user_by_delivery_email("*****@*****.**", zulip_realm)
            do_change_user_role(iago, UserProfile.ROLE_REALM_ADMINISTRATOR, acting_user=None)
            iago.is_staff = True
            iago.save(update_fields=["is_staff"])

            # We need to create at least two test draft for Iago for the sake
            # of the cURL tests. Two since one will be deleted.
            Draft.objects.create(
                user_profile=iago,
                recipient=None,
                topic="Release Notes",
                content="Release 4.0 will contain ...",
                last_edit_time=timezone_now(),
            )
            Draft.objects.create(
                user_profile=iago,
                recipient=None,
                topic="Release Notes",
                content="Release 4.0 will contain many new features such as ... ",
                last_edit_time=timezone_now(),
            )

            desdemona = get_user_by_delivery_email("*****@*****.**", zulip_realm)
            do_change_user_role(desdemona, UserProfile.ROLE_REALM_OWNER, acting_user=None)

            shiva = get_user_by_delivery_email("*****@*****.**", zulip_realm)
            do_change_user_role(shiva, UserProfile.ROLE_MODERATOR, acting_user=None)

            polonius = get_user_by_delivery_email("*****@*****.**", zulip_realm)
            do_change_user_role(polonius, UserProfile.ROLE_GUEST, acting_user=None)

            # These bots are directly referenced from code and thus
            # are needed for the test suite.
            zulip_realm_bots = [
                ("Zulip Error Bot", "*****@*****.**"),
                ("Zulip Default Bot", "*****@*****.**"),
            ]
            for i in range(options["extra_bots"]):
                zulip_realm_bots.append((f"Extra Bot {i}", f"extrabot{i}@zulip.com"))

            create_users(
                zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT, bot_owner=desdemona
            )

            zoe = get_user_by_delivery_email("*****@*****.**", zulip_realm)
            zulip_webhook_bots = [
                ("Zulip Webhook Bot", "*****@*****.**"),
            ]
            # If a stream is not supplied in the webhook URL, the webhook
            # will (in some cases) send the notification as a PM to the
            # owner of the webhook bot, so bot_owner can't be None
            create_users(
                zulip_realm,
                zulip_webhook_bots,
                bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
                bot_owner=zoe,
            )
            aaron = get_user_by_delivery_email("*****@*****.**", zulip_realm)

            zulip_outgoing_bots = [
                ("Outgoing Webhook", "*****@*****.**"),
            ]
            create_users(
                zulip_realm,
                zulip_outgoing_bots,
                bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
                bot_owner=aaron,
            )
            outgoing_webhook = get_user("*****@*****.**", zulip_realm)
            add_service(
                "outgoing-webhook",
                user_profile=outgoing_webhook,
                interface=Service.GENERIC,
                base_url="http://127.0.0.1:5002",
                token=generate_api_key(),
            )

            # Add the realm internal bots to each realm.
            create_if_missing_realm_internal_bots()

            # Create public streams.
            signups_stream = Realm.INITIAL_PRIVATE_STREAM_NAME

            stream_list = [
                "Verona",
                "Denmark",
                "Scotland",
                "Venice",
                "Rome",
                signups_stream,
            ]
            stream_dict: Dict[str, Dict[str, Any]] = {
                "Denmark": {"description": "A Scandinavian country"},
                "Scotland": {"description": "Located in the United Kingdom"},
                "Venice": {"description": "A northeastern Italian city"},
                "Rome": {"description": "Yet another Italian city", "is_web_public": True},
            }

            bulk_create_streams(zulip_realm, stream_dict)
            recipient_streams: List[int] = [
                Stream.objects.get(name=name, realm=zulip_realm).id for name in stream_list
            ]

            # Create subscriptions to streams.  The following
            # algorithm will give each of the users a different but
            # deterministic subset of the streams (given a fixed list
            # of users). For the test suite, we have a fixed list of
            # subscriptions to make sure test data is consistent
            # across platforms.

            subscriptions_list: List[Tuple[UserProfile, Recipient]] = []
            profiles: Sequence[UserProfile] = list(
                UserProfile.objects.select_related().filter(is_bot=False).order_by("email")
            )

            if options["test_suite"]:
                subscriptions_map = {
                    "*****@*****.**": ["Verona"],
                    "*****@*****.**": ["Verona"],
                    "*****@*****.**": ["Verona", "Denmark", signups_stream],
                    "*****@*****.**": [
                        "Verona",
                        "Denmark",
                        "Scotland",
                        signups_stream,
                    ],
                    "*****@*****.**": ["Verona", "Denmark", "Scotland"],
                    "*****@*****.**": ["Verona", "Denmark", "Scotland", "Venice"],
                    "*****@*****.**": ["Verona", "Denmark", "Scotland", "Venice", "Rome"],
                    "*****@*****.**": ["Verona"],
                    "*****@*****.**": [
                        "Verona",
                        "Denmark",
                        "Venice",
                        signups_stream,
                    ],
                    "*****@*****.**": ["Verona", "Denmark", "Scotland"],
                }

                for profile in profiles:
                    email = profile.delivery_email
                    if email not in subscriptions_map:
                        raise Exception(f"Subscriptions not listed for user {email}")

                    for stream_name in subscriptions_map[email]:
                        stream = Stream.objects.get(name=stream_name, realm=zulip_realm)
                        r = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id)
                        subscriptions_list.append((profile, r))
            else:
                num_streams = len(recipient_streams)
                num_users = len(profiles)
                for i, profile in enumerate(profiles):
                    # Subscribe to some streams.
                    fraction = float(i) / num_users
                    num_recips = int(num_streams * fraction) + 1

                    for type_id in recipient_streams[:num_recips]:
                        r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id)
                        subscriptions_list.append((profile, r))

            subscriptions_to_add: List[Subscription] = []
            event_time = timezone_now()
            all_subscription_logs: (List[RealmAuditLog]) = []

            i = 0
            for profile, recipient in subscriptions_list:
                i += 1
                color = STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)]
                s = Subscription(
                    recipient=recipient,
                    user_profile=profile,
                    is_user_active=profile.is_active,
                    color=color,
                )

                subscriptions_to_add.append(s)

                log = RealmAuditLog(
                    realm=profile.realm,
                    modified_user=profile,
                    modified_stream_id=recipient.type_id,
                    event_last_message_id=0,
                    event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
                    event_time=event_time,
                )
                all_subscription_logs.append(log)

            Subscription.objects.bulk_create(subscriptions_to_add)
            RealmAuditLog.objects.bulk_create(all_subscription_logs)

            # Create custom profile field data
            phone_number = try_add_realm_custom_profile_field(
                zulip_realm, "Phone number", CustomProfileField.SHORT_TEXT, hint=""
            )
            biography = try_add_realm_custom_profile_field(
                zulip_realm,
                "Biography",
                CustomProfileField.LONG_TEXT,
                hint="What are you known for?",
            )
            favorite_food = try_add_realm_custom_profile_field(
                zulip_realm,
                "Favorite food",
                CustomProfileField.SHORT_TEXT,
                hint="Or drink, if you'd prefer",
            )
            field_data: ProfileFieldData = {
                "vim": {"text": "Vim", "order": "1"},
                "emacs": {"text": "Emacs", "order": "2"},
            }
            favorite_editor = try_add_realm_custom_profile_field(
                zulip_realm, "Favorite editor", CustomProfileField.SELECT, field_data=field_data
            )
            birthday = try_add_realm_custom_profile_field(
                zulip_realm, "Birthday", CustomProfileField.DATE
            )
            favorite_website = try_add_realm_custom_profile_field(
                zulip_realm,
                "Favorite website",
                CustomProfileField.URL,
                hint="Or your personal blog's URL",
            )
            mentor = try_add_realm_custom_profile_field(
                zulip_realm, "Mentor", CustomProfileField.USER
            )
            github_profile = try_add_realm_default_custom_profile_field(zulip_realm, "github")

            # Fill in values for Iago and Hamlet
            hamlet = get_user_by_delivery_email("*****@*****.**", zulip_realm)
            do_update_user_custom_profile_data_if_changed(
                iago,
                [
                    {"id": phone_number.id, "value": "+1-234-567-8901"},
                    {"id": biography.id, "value": "Betrayer of Othello."},
                    {"id": favorite_food.id, "value": "Apples"},
                    {"id": favorite_editor.id, "value": "emacs"},
                    {"id": birthday.id, "value": "2000-01-01"},
                    {"id": favorite_website.id, "value": "https://zulip.readthedocs.io/en/latest/"},
                    {"id": mentor.id, "value": [hamlet.id]},
                    {"id": github_profile.id, "value": "zulip"},
                ],
            )
            do_update_user_custom_profile_data_if_changed(
                hamlet,
                [
                    {"id": phone_number.id, "value": "+0-11-23-456-7890"},
                    {
                        "id": biography.id,
                        "value": "I am:\n* The prince of Denmark\n* Nephew to the usurping Claudius",
                    },
                    {"id": favorite_food.id, "value": "Dark chocolate"},
                    {"id": favorite_editor.id, "value": "vim"},
                    {"id": birthday.id, "value": "1900-01-01"},
                    {"id": favorite_website.id, "value": "https://blog.zulig.org"},
                    {"id": mentor.id, "value": [iago.id]},
                    {"id": github_profile.id, "value": "zulipbot"},
                ],
            )
        else:
            zulip_realm = get_realm("zulip")
            recipient_streams = [
                klass.type_id for klass in Recipient.objects.filter(type=Recipient.STREAM)
            ]

        # Extract a list of all users
        user_profiles: List[UserProfile] = list(UserProfile.objects.filter(is_bot=False))

        # Create a test realm emoji.
        IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
        with open(IMAGE_FILE_PATH, "rb") as fp:
            check_add_realm_emoji(zulip_realm, "green_tick", iago, File(fp))

        if not options["test_suite"]:
            # Populate users with some bar data
            for user in user_profiles:
                status: int = UserPresence.ACTIVE
                date = timezone_now()
                client = get_client("website")
                if user.full_name[0] <= "H":
                    client = get_client("ZulipAndroid")
                UserPresence.objects.get_or_create(
                    user_profile=user,
                    realm_id=user.realm_id,
                    client=client,
                    timestamp=date,
                    status=status,
                )

        user_profiles_ids = [user_profile.id for user_profile in user_profiles]

        # Create several initial huddles
        for i in range(options["num_huddles"]):
            get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))

        # Create several initial pairs for personals
        personals_pairs = [
            random.sample(user_profiles_ids, 2) for i in range(options["num_personals"])
        ]

        create_alert_words(zulip_realm.id)

        # Generate a new set of test data.
        create_test_data()

        if options["delete"]:
            if options["test_suite"]:
                # Create test users; the MIT ones are needed to test
                # the Zephyr mirroring codepaths.
                event_time = timezone_now()
                testsuite_mit_users = [
                    ("Fred Sipb (MIT)", "*****@*****.**"),
                    ("Athena Consulting Exchange User (MIT)", "*****@*****.**"),
                    ("Esp Classroom (MIT)", "*****@*****.**"),
                ]
                create_users(
                    mit_realm, testsuite_mit_users, tos_version=settings.TERMS_OF_SERVICE_VERSION
                )

                mit_user = get_user_by_delivery_email("*****@*****.**", mit_realm)
                mit_signup_stream = Stream.objects.get(
                    name=Realm.INITIAL_PRIVATE_STREAM_NAME, realm=mit_realm
                )
                bulk_add_subscriptions(mit_realm, [mit_signup_stream], [mit_user], acting_user=None)

                testsuite_lear_users = [
                    ("King Lear", "*****@*****.**"),
                    ("Cordelia, Lear's daughter", "*****@*****.**"),
                ]
                create_users(
                    lear_realm, testsuite_lear_users, tos_version=settings.TERMS_OF_SERVICE_VERSION
                )

                lear_user = get_user_by_delivery_email("*****@*****.**", lear_realm)
                lear_signup_stream = Stream.objects.get(
                    name=Realm.INITIAL_PRIVATE_STREAM_NAME, realm=lear_realm
                )
                bulk_add_subscriptions(
                    lear_realm, [lear_signup_stream], [lear_user], acting_user=None
                )

            if not options["test_suite"]:
                # To keep the messages.json fixtures file for the test
                # suite fast, don't add these users and subscriptions
                # when running populate_db for the test suite

                # to imitate emoji insertions in stream names
                raw_emojis = ["😎", "😂", "🐱‍👤"]

                zulip_stream_dict: Dict[str, Dict[str, Any]] = {
                    "devel": {"description": "For developing"},
                    # ビデオゲーム - VideoGames (japanese)
                    "ビデオゲーム": {"description": f"Share your favorite video games!  {raw_emojis[2]}"},
                    "announce": {
                        "description": "For announcements",
                        "stream_post_policy": Stream.STREAM_POST_POLICY_ADMINS,
                    },
                    "design": {"description": "For design"},
                    "support": {"description": "For support"},
                    "social": {"description": "For socializing"},
                    "test": {"description": "For testing `code`"},
                    "errors": {"description": "For errors"},
                    # 조리법 - Recipes (Korean) , Пельмени - Dumplings (Russian)
                    "조리법 "
                    + raw_emojis[0]: {"description": "Everything cooking, from pasta to Пельмени"},
                }

                extra_stream_names = [
                    "802.11a",
                    "Ad Hoc Network",
                    "Augmented Reality",
                    "Cycling",
                    "DPI",
                    "FAQ",
                    "FiFo",
                    "commits",
                    "Control panel",
                    "desktop",
                    "компьютеры",
                    "Data security",
                    "desktop",
                    "काम",
                    "discussions",
                    "Cloud storage",
                    "GCI",
                    "Vaporware",
                    "Recent Trends",
                    "issues",
                    "live",
                    "Health",
                    "mobile",
                    "空間",
                    "provision",
                    "hidrógeno",
                    "HR",
                    "アニメ",
                ]

                # Add stream names and stream descriptions
                for i in range(options["extra_streams"]):
                    extra_stream_name = random.choice(extra_stream_names) + " " + str(i)

                    # to imitate emoji insertions in stream names
                    if random.random() <= 0.15:
                        extra_stream_name += random.choice(raw_emojis)

                    zulip_stream_dict[extra_stream_name] = {
                        "description": "Auto-generated extra stream.",
                    }

                bulk_create_streams(zulip_realm, zulip_stream_dict)
                # Now that we've created the notifications stream, configure it properly.
                zulip_realm.notifications_stream = get_stream("announce", zulip_realm)
                zulip_realm.save(update_fields=["notifications_stream"])

                # Add a few default streams
                for default_stream_name in ["design", "devel", "social", "support"]:
                    DefaultStream.objects.create(
                        realm=zulip_realm, stream=get_stream(default_stream_name, zulip_realm)
                    )

                # Now subscribe everyone to these streams
                subscribe_users_to_streams(zulip_realm, zulip_stream_dict)

            create_user_groups()

            if not options["test_suite"]:
                # We populate the analytics database here for
                # development purpose only
                call_command("populate_analytics_db")

        threads = options["threads"]
        jobs: List[Tuple[int, List[List[int]], Dict[str, Any], int]] = []
        for i in range(threads):
            count = options["num_messages"] // threads
            if i < options["num_messages"] % threads:
                count += 1
            jobs.append((count, personals_pairs, options, random.randint(0, 10**10)))

        for job in jobs:
            generate_and_send_messages(job)

        if options["delete"]:
            if not options["test_suite"]:
                # These bots are not needed by the test suite
                # Also, we don't want interacting with each other
                # in dev setup.
                internal_zulip_users_nosubs = [
                    ("Zulip Commit Bot", "*****@*****.**"),
                    ("Zulip Trac Bot", "*****@*****.**"),
                    ("Zulip Nagios Bot", "*****@*****.**"),
                ]
                create_users(
                    zulip_realm,
                    internal_zulip_users_nosubs,
                    bot_type=UserProfile.DEFAULT_BOT,
                    bot_owner=desdemona,
                )

            mark_all_messages_as_read()
            self.stdout.write("Successfully populated test database.\n")

        push_notifications_logger.disabled = False
示例#54
0
 def back_date(num_weeks: int) -> None:
     user_presence = UserPresence.objects.filter(
         user_profile=user_profile)[0]
     user_presence.timestamp = timezone_now() - datetime.timedelta(
         weeks=num_weeks)
     user_presence.save()
示例#55
0
def convert_channel_data(channel_data: List[ZerverFieldsT],
                         user_data_map: Dict[str, Dict[str, Any]],
                         subscriber_handler: SubscriberHandler,
                         stream_id_mapper: IdMapper,
                         user_id_mapper: IdMapper,
                         realm_id: int,
                         team_name: str) -> List[ZerverFieldsT]:
    channel_data_list = [
        d
        for d in channel_data
        if d['team'] == team_name
    ]

    channel_members_map = {}  # type: Dict[str, List[str]]
    channel_admins_map = {}  # type: Dict[str, List[str]]

    def initialize_stream_membership_dicts() -> None:
        for channel in channel_data:
            channel_name = channel["name"]
            channel_members_map[channel_name] = []
            channel_admins_map[channel_name] = []

        for username in user_data_map:
            user_dict = user_data_map[username]
            teams = user_dict["teams"]
            for team in teams:
                if team["name"] != team_name:
                    continue
                for channel in team["channels"]:
                    channel_roles = channel["roles"]
                    channel_name = channel["name"]
                    if "channel_admin" in channel_roles:
                        channel_admins_map[channel_name].append(username)
                    elif "channel_user" in channel_roles:
                        channel_members_map[channel_name].append(username)

    def get_invite_only_value_from_channel_type(channel_type: str) -> bool:
        # Channel can have two types in Mattermost
        # "O" for a public channel.
        # "P" for a private channel.
        if channel_type == 'O':
            return False
        elif channel_type == 'P':
            return True
        else:  # nocoverage
            raise Exception('unexpected value')

    streams = []
    initialize_stream_membership_dicts()

    for channel_dict in channel_data_list:
        now = int(timezone_now().timestamp())
        stream_id = stream_id_mapper.get(channel_dict['name'])
        stream_name = channel_dict["name"]
        invite_only = get_invite_only_value_from_channel_type(channel_dict['type'])

        stream = build_stream(
            date_created=now,
            realm_id=realm_id,
            name=channel_dict['display_name'],
            # Purpose describes how the channel should be used. It is similar to
            # stream description and is shown in channel list to help others decide
            # whether to join.
            # Header text always appears right next to channel name in channel header.
            # Can be used for advertising the purpose of stream, making announcements as
            # well as including frequently used links. So probably not a bad idea to use
            # this as description if the channel purpose is empty.
            description=channel_dict["purpose"] or channel_dict['header'],
            stream_id=stream_id,
            # Mattermost export don't include data of archived(~ deactivated) channels.
            deactivated=False,
            invite_only=invite_only,
        )

        channel_users = set()
        for username in channel_admins_map[stream_name]:
            channel_users.add(user_id_mapper.get(username))

        for username in channel_members_map[stream_name]:
            channel_users.add(user_id_mapper.get(username))

        if channel_users:
            subscriber_handler.set_info(
                stream_id=stream_id,
                users=channel_users,
            )
        streams.append(stream)

    return streams
示例#56
0
    def test_analytics_not_running(self) -> None:
        realm = get_realm("zulip")

        self.assertEqual(FillState.objects.count(), 0)

        realm.date_created = timezone_now() - timedelta(days=3)
        realm.save(update_fields=["date_created"])
        with mock.patch('logging.warning'):
            result = self.client_get('/json/analytics/chart_data',
                                     {'chart_name': 'messages_sent_over_time'})
        self.assert_json_error_contains(result, 'No analytics data available')

        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
        realm.save(update_fields=["date_created"])
        with mock.patch('logging.warning'):
            result = self.client_get('/json/analytics/chart_data',
                                     {'chart_name': 'messages_sent_over_time'})
        self.assert_json_error_contains(result, 'No analytics data available')

        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
        realm.save(update_fields=["date_created"])
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'messages_sent_over_time'})
        self.assert_json_success(result)

        realm.date_created = timezone_now() - timedelta(hours=10)
        realm.save(update_fields=["date_created"])
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'messages_sent_over_time'})
        self.assert_json_success(result)

        end_time = timezone_now() - timedelta(days=5)
        fill_state = FillState.objects.create(
            property='messages_sent:is_bot:hour',
            end_time=end_time,
            state=FillState.DONE)

        realm.date_created = timezone_now() - timedelta(days=3)
        realm.save(update_fields=["date_created"])
        with mock.patch('logging.warning'):
            result = self.client_get('/json/analytics/chart_data',
                                     {'chart_name': 'messages_sent_over_time'})
        self.assert_json_error_contains(result, 'No analytics data available')

        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
        realm.save(update_fields=["date_created"])
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'messages_sent_over_time'})
        self.assert_json_success(result)

        end_time = timezone_now() - timedelta(days=2)
        fill_state.end_time = end_time
        fill_state.save(update_fields=["end_time"])

        realm.date_created = timezone_now() - timedelta(days=3)
        realm.save(update_fields=["date_created"])
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'messages_sent_over_time'})
        self.assert_json_success(result)

        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
        realm.save(update_fields=["date_created"])
        with mock.patch('logging.warning'):
            result = self.client_get('/json/analytics/chart_data',
                                     {'chart_name': 'messages_sent_over_time'})
        self.assert_json_error_contains(result, 'No analytics data available')

        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
        realm.save(update_fields=["date_created"])
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'messages_sent_over_time'})
        self.assert_json_success(result)
示例#57
0
    def maybe_send_batched_emails(self) -> None:
        with self.lock:
            # self.timer_event just triggered execution of this
            # function in a thread, so now that we hold the lock, we
            # clear the timer_event attribute to record that no Timer
            # is active.
            self.timer_event = None

            current_time = timezone_now()

            with transaction.atomic():
                events_to_process = ScheduledMessageNotificationEmail.objects.filter(
                    scheduled_timestamp__lte=current_time).select_related()

                # Batch the entries by user
                events_by_recipient: Dict[int, List[Dict[str, Any]]] = {}
                for event in events_to_process:
                    entry = dict(
                        user_profile_id=event.user_profile_id,
                        message_id=event.message_id,
                        trigger=event.trigger,
                        mentioned_user_group_id=event.mentioned_user_group_id,
                    )
                    if event.user_profile_id in events_by_recipient:
                        events_by_recipient[event.user_profile_id].append(
                            entry)
                    else:
                        events_by_recipient[event.user_profile_id] = [entry]

                for user_profile_id in events_by_recipient.keys():
                    events: List[Dict[
                        str, Any]] = events_by_recipient[user_profile_id]

                    logging.info(
                        "Batch-processing %s missedmessage_emails events for user %s",
                        len(events),
                        user_profile_id,
                    )
                    try:
                        # Because we process events in batches, an
                        # escaped exception here would lead to
                        # duplicate messages being sent for other
                        # users in the same events_to_process batch,
                        # and no guarantee of forward progress.
                        handle_missedmessage_emails(user_profile_id, events)
                    except Exception:
                        logging.exception(
                            "Failed to process %d missedmessage_emails for user %s",
                            len(events),
                            user_profile_id,
                            stack_info=True,
                        )

                events_to_process.delete()

            # By only restarting the timer if there are actually events in
            # the queue, we ensure this queue processor is idle when there
            # are no missed-message emails to process.  This avoids
            # constant CPU usage when there is no work to do.
            if ScheduledMessageNotificationEmail.objects.exists():
                self.ensure_timer()
示例#58
0
def estimate_recent_messages(realm: Realm, hours: int) -> int:
    stat = COUNT_STATS['messages_sent:is_bot:hour']
    d = timezone_now() - datetime.timedelta(hours=hours)
    return RealmCount.objects.filter(
        property=stat.property, end_time__gt=d, realm=realm).aggregate(
            Sum('value'))['value__sum'] or 0
示例#59
0
    def test_convert_slack_workspace_messages(
            self, mock_get_messages_iterator: mock.Mock,
            mock_message: mock.Mock) -> None:
        output_dir = os.path.join(settings.TEST_WORKER_DIR,
                                  'test-slack-import')
        os.makedirs(output_dir, exist_ok=True)

        added_channels = {
            'random': ('c5', 1),
            'general': ('c6', 2)
        }  # type: Dict[str, Tuple[str, int]]

        time = float(timezone_now().timestamp())
        zerver_message = [{'id': 1, 'ts': time}, {'id': 5, 'ts': time}]

        def fake_get_messages_iter(
                slack_data_dir: str, added_channels: AddedChannelsT,
                added_mpims: AddedMPIMsT,
                dm_members: DMMembersT) -> Iterator[ZerverFieldsT]:
            import copy
            return iter(copy.deepcopy(zerver_message))

        realm = {'zerver_subscription': []}  # type: Dict[str, Any]
        user_list = []  # type: List[Dict[str, Any]]
        reactions = [{"name": "grinning", "users": ["U061A5N1G"], "count": 1}]
        attachments = uploads = []  # type: List[Dict[str, Any]]

        zerver_usermessage = [{'id': 3}, {'id': 5}, {'id': 6}, {'id': 9}]

        mock_get_messages_iterator.side_effect = fake_get_messages_iter
        mock_message.side_effect = [[
            zerver_message[:1], zerver_usermessage[:2], attachments, uploads,
            reactions[:1]
        ],
                                    [
                                        zerver_message[1:2],
                                        zerver_usermessage[2:5], attachments,
                                        uploads, reactions[1:1]
                                    ]]
        # Hacky: We should include a zerver_userprofile, not the empty []
        test_reactions, uploads, zerver_attachment = convert_slack_workspace_messages(
            './random_path',
            user_list,
            2, {}, {},
            added_channels, {}, {},
            realm, [], [],
            'domain',
            output_dir=output_dir,
            chunk_size=1)
        messages_file_1 = os.path.join(output_dir, 'messages-000001.json')
        self.assertTrue(os.path.exists(messages_file_1))
        messages_file_2 = os.path.join(output_dir, 'messages-000002.json')
        self.assertTrue(os.path.exists(messages_file_2))

        with open(messages_file_1) as f:
            message_json = ujson.load(f)
        self.assertEqual(message_json['zerver_message'], zerver_message[:1])
        self.assertEqual(message_json['zerver_usermessage'],
                         zerver_usermessage[:2])

        with open(messages_file_2) as f:
            message_json = ujson.load(f)
        self.assertEqual(message_json['zerver_message'], zerver_message[1:2])
        self.assertEqual(message_json['zerver_usermessage'],
                         zerver_usermessage[2:5])

        self.assertEqual(test_reactions, reactions)
示例#60
0
def slack_workspace_to_realm(domain_name: str, realm_id: int, user_list: List[ZerverFieldsT],
                             realm_subdomain: str, slack_data_dir: str,
                             custom_emoji_list: ZerverFieldsT)-> Tuple[ZerverFieldsT, AddedUsersT,
                                                                       AddedRecipientsT,
                                                                       AddedChannelsT,
                                                                       List[ZerverFieldsT],
                                                                       ZerverFieldsT]:
    """
    Returns:
    1. realm, Converted Realm data
    2. added_users, which is a dictionary to map from slack user id to zulip user id
    3. added_recipient, which is a dictionary to map from channel name to zulip recipient_id
    4. added_channels, which is a dictionary to map from channel name to channel id, zulip stream_id
    5. avatars, which is list to map avatars to zulip avatar records.json
    6. emoji_url_map, which is maps emoji name to its slack url
    """
    NOW = float(timezone_now().timestamp())

    zerver_realm = build_zerver_realm(realm_id, realm_subdomain, NOW)

    realm = dict(zerver_client=[{"name": "populate_db", "id": 1},
                                {"name": "website", "id": 2},
                                {"name": "API", "id": 3}],
                 zerver_userpresence=[],  # shows last logged in data, which is not available in slack
                 zerver_userprofile_mirrordummy=[],
                 zerver_realmdomain=[{"realm": realm_id,
                                      "allow_subdomains": False,
                                      "domain": domain_name,
                                      "id": realm_id}],
                 zerver_useractivity=[],
                 zerver_realm=zerver_realm,
                 zerver_huddle=[],
                 zerver_userprofile_crossrealm=[],
                 zerver_useractivityinterval=[],
                 zerver_realmfilter=[])

    zerver_userprofile, avatars, added_users, zerver_customprofilefield, \
        zerver_customprofilefield_value = users_to_zerver_userprofile(slack_data_dir, user_list,
                                                                      realm_id, int(NOW), domain_name)
    channels_to_zerver_stream_fields = channels_to_zerver_stream(slack_data_dir,
                                                                 realm_id,
                                                                 added_users,
                                                                 zerver_userprofile)
    zerver_realmemoji, emoji_url_map = build_realmemoji(custom_emoji_list, realm_id)
    realm['zerver_realmemoji'] = zerver_realmemoji

    # See https://zulipchat.com/help/set-default-streams-for-new-users
    # for documentation on zerver_defaultstream
    realm['zerver_userprofile'] = zerver_userprofile

    # Custom profile fields
    realm['zerver_customprofilefield'] = zerver_customprofilefield
    realm['zerver_customprofilefieldvalue'] = zerver_customprofilefield_value

    realm['zerver_defaultstream'] = channels_to_zerver_stream_fields[0]
    realm['zerver_stream'] = channels_to_zerver_stream_fields[1]
    realm['zerver_subscription'] = channels_to_zerver_stream_fields[3]
    realm['zerver_recipient'] = channels_to_zerver_stream_fields[4]
    added_channels = channels_to_zerver_stream_fields[2]
    added_recipient = channels_to_zerver_stream_fields[5]

    return realm, added_users, added_recipient, added_channels, avatars, emoji_url_map