示例#1
0
    def message_header(user_profile: UserProfile, message: Message) -> Dict[str, Any]:
        if message.recipient.type == Recipient.PERSONAL:
            header = "You and %s" % (message.sender.full_name,)
            html_link = personal_narrow_url(
                realm=user_profile.realm,
                sender=message.sender,
            )
            header_html = "<a style='color: #ffffff;' href='%s'>%s</a>" % (html_link, header)
        elif message.recipient.type == Recipient.HUDDLE:
            disp_recipient = get_display_recipient(message.recipient)
            assert not isinstance(disp_recipient, str)
            other_recipients = [r['full_name'] for r in disp_recipient
                                if r['id'] != user_profile.id]
            header = "You and %s" % (", ".join(other_recipients),)
            other_user_ids = [r['id'] for r in disp_recipient
                              if r['id'] != user_profile.id]
            html_link = huddle_narrow_url(
                realm=user_profile.realm,
                other_user_ids=other_user_ids,
            )

            header_html = "<a style='color: #ffffff;' href='%s'>%s</a>" % (html_link, header)
        else:
            stream = Stream.objects.only('id', 'name').get(id=message.recipient.type_id)
            header = "%s > %s" % (stream.name, message.topic_name())
            stream_link = stream_narrow_url(user_profile.realm, stream)
            topic_link = topic_narrow_url(user_profile.realm, stream, message.topic_name())
            header_html = "<a href='%s'>%s</a> > <a href='%s'>%s</a>" % (
                stream_link, stream.name, topic_link, message.topic_name())
        return {"plain": header,
                "html": header_html,
                "stream_message": message.recipient.type_name() == "stream"}
示例#2
0
    def test_mention_invalid(self):
        sender_user_profile = get_user_profile_by_email("*****@*****.**")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))

        content = "Hey @**Nonexistent User**"
        self.assertEqual(msg.render_markdown(content),
                         '<p>Hey @<strong>Nonexistent User</strong></p>')
        self.assertEqual(msg.mentions_user_ids, set())
示例#3
0
    def test_mention_everyone(self):
        user_profile = get_user_profile_by_email("*****@*****.**")
        msg = Message(sender=user_profile, sending_client=get_client("test"))

        content = "@everyone test"
        self.assertEqual(msg.render_markdown(content),
                         '<p><span class="user-mention" data-user-email="*">@everyone</span> test</p>')
        self.assertTrue(msg.mentions_wildcard)
示例#4
0
    def test_mention_shortname(self):
        sender_user_profile = get_user_profile_by_email("*****@*****.**")
        user_profile = get_user_profile_by_email("*****@*****.**")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))

        content = "@**hamlet**"
        self.assertEqual(msg.render_markdown(content),
                         '<p><span class="user-mention" data-user-email="*****@*****.**">@King Hamlet</span></p>')
        self.assertEqual(msg.mentions_user_ids, set([user_profile.id]))
示例#5
0
def render_message_backend(request, user_profile, content=REQ()):
    # type: (HttpRequest, UserProfile, Text) -> HttpResponse
    message = Message()
    message.sender = user_profile
    message.content = content
    message.sending_client = request.client

    rendered_content = render_markdown(message, content, realm=user_profile.realm)
    return json_success({"rendered": rendered_content})
示例#6
0
def get_apns_alert_title(message: Message) -> str:
    """
    On an iOS notification, this is the first bolded line.
    """
    if message.recipient.type == Recipient.HUDDLE:
        recipients = cast(List[Dict[str, Any]], get_display_recipient(message.recipient))
        return ', '.join(sorted(r['full_name'] for r in recipients))
    elif message.is_stream_message():
        return "#%s > %s" % (get_display_recipient(message.recipient), message.topic_name(),)
    # For personal PMs, we just show the sender name.
    return message.sender.full_name
示例#7
0
    def test_alert_words(self):
        user_profile = get_user_profile_by_email("*****@*****.**")
        do_set_alert_words(user_profile, ["ALERTWORD", "scaryword"])
        msg = Message(sender=user_profile, sending_client=get_client("test"))

        content = "We have an ALERTWORD day today!"
        self.assertEqual(msg.render_markdown(content), "<p>We have an ALERTWORD day today!</p>")
        self.assertEqual(msg.user_ids_with_alert_words, set([user_profile.id]))

        msg = Message(sender=user_profile, sending_client=get_client("test"))
        content = "We have a NOTHINGWORD day today!"
        self.assertEqual(msg.render_markdown(content), "<p>We have a NOTHINGWORD day today!</p>")
        self.assertEqual(msg.user_ids_with_alert_words, set())
示例#8
0
    def test_mention_multiple(self):
        sender_user_profile = get_user_profile_by_email("*****@*****.**")
        hamlet = get_user_profile_by_email("*****@*****.**")
        cordelia = get_user_profile_by_email("*****@*****.**")
        msg = Message(sender=sender_user_profile, sending_client=get_client("test"))

        content = "@**King Hamlet** and @**cordelia**, check this out"
        self.assertEqual(msg.render_markdown(content),
                         '<p>'
                         '<span class="user-mention" '
                         'data-user-email="*****@*****.**">@King Hamlet</span> and '
                         '<span class="user-mention" '
                         'data-user-email="*****@*****.**">@Cordelia Lear</span>, '
                         'check this out</p>')
        self.assertEqual(msg.mentions_user_ids, set([hamlet.id, cordelia.id]))
示例#9
0
def get_alert_from_message(message: Message) -> str:
    """
    Determine what alert string to display based on the missed messages.
    """
    sender_str = message.sender.full_name
    if message.recipient.type == Recipient.HUDDLE and message.trigger == 'private_message':
        return "New private group message from %s" % (sender_str,)
    elif message.recipient.type == Recipient.PERSONAL and message.trigger == 'private_message':
        return "New private message from %s" % (sender_str,)
    elif message.is_stream_message() and message.trigger == 'mentioned':
        return "New mention from %s" % (sender_str,)
    elif (message.is_stream_message() and
            (message.trigger == 'stream_push_notify' and message.stream_name)):
        return "New stream message from %s in %s" % (sender_str, message.stream_name,)
    else:
        return "New Zulip mentions and private messages from %s" % (sender_str,)
示例#10
0
def archive(request: HttpRequest,
            stream_id: int,
            topic_name: str) -> HttpResponse:

    def get_response(rendered_message_list: List[str],
                     is_web_public: bool,
                     stream_name: str) -> HttpResponse:
        return render(
            request,
            'zerver/archive/index.html',
            context={
                'is_web_public': is_web_public,
                'message_list': rendered_message_list,
                'stream': stream_name,
                'topic': topic_name,
            }
        )

    try:
        stream = get_stream_by_id(stream_id)
    except JsonableError:
        return get_response([], False, '')

    if not stream.is_web_public:
        return get_response([], False, '')

    all_messages = list(Message.objects.select_related(
        'sender').filter(recipient__type_id=stream_id, subject=topic_name).order_by('pub_date'))
    if not all_messages:
        return get_response([], True, stream.name)

    rendered_message_list = []
    prev_sender = None
    for msg in all_messages:
        include_sender = False
        status_message = Message.is_status_message(msg.content, msg.rendered_content)
        if not prev_sender or prev_sender != msg.sender or status_message:
            if status_message:
                prev_sender = None
            else:
                prev_sender = msg.sender
            include_sender = True
        if status_message:
            status_message = msg.rendered_content[4+3: -4]
        context = {
            'sender_full_name': msg.sender.full_name,
            'timestampstr': datetime_to_timestamp(msg.last_edit_time
                                                  if msg.last_edit_time
                                                  else msg.pub_date),
            'message_content': msg.rendered_content,
            'avatar_url': get_gravatar_url(msg.sender.email, 1),
            'include_sender': include_sender,
            'status_message': status_message,
        }
        rendered_msg = loader.render_to_string('zerver/archive/single_message.html', context)
        rendered_message_list.append(rendered_msg)
    return get_response(rendered_message_list, True, stream.name)
示例#11
0
def render_markdown(message, content, realm=None, realm_alert_words=None, message_users=None):
    # type: (Message, Text, Optional[Realm], Optional[RealmAlertWords], Set[UserProfile]) -> Text
    """Return HTML for given markdown. Bugdown may add properties to the
    message object such as `mentions_user_ids` and `mentions_wildcard`.
    These are only on this Django object and are not saved in the
    database.
    """

    if message_users is None:
        message_user_ids = set()  # type: Set[int]
    else:
        message_user_ids = {u.id for u in message_users}

    if message is not None:
        message.mentions_wildcard = False
        message.is_me_message = False
        message.mentions_user_ids = set()
        message.alert_words = set()
        message.links_for_preview = set()
        message.outgoing_webhook_bot_triggers = []

        if realm is None:
            realm = message.get_realm()

    possible_words = set()  # type: Set[Text]
    if realm_alert_words is not None:
        for user_id, words in realm_alert_words.items():
            if user_id in message_user_ids:
                possible_words.update(set(words))

    if message is None:
        # If we don't have a message, then we are in the compose preview
        # codepath, so we know we are dealing with a human.
        sent_by_bot = False
    else:
        sent_by_bot = get_user_profile_by_id(message.sender_id).is_bot

    # DO MAIN WORK HERE -- call bugdown to convert
    rendered_content = bugdown.convert(content, message=message, message_realm=realm,
                                       possible_words=possible_words,
                                       sent_by_bot=sent_by_bot)

    if message is not None:
        message.user_ids_with_alert_words = set()

        if realm_alert_words is not None:
            for user_id, words in realm_alert_words.items():
                if user_id in message_user_ids:
                    if set(words).intersection(message.alert_words):
                        message.user_ids_with_alert_words.add(user_id)

        message.is_me_message = Message.is_status_message(content, rendered_content)

    return rendered_content
示例#12
0
def render_markdown(message: Message,
                    content: str,
                    realm: Optional[Realm]=None,
                    realm_alert_words: Optional[RealmAlertWords]=None,
                    user_ids: Optional[Set[int]]=None,
                    mention_data: Optional[bugdown.MentionData]=None,
                    email_gateway: Optional[bool]=False) -> str:
    """Return HTML for given markdown. Bugdown may add properties to the
    message object such as `mentions_user_ids`, `mentions_user_group_ids`, and
    `mentions_wildcard`.  These are only on this Django object and are not
    saved in the database.
    """

    if user_ids is None:
        message_user_ids = set()  # type: Set[int]
    else:
        message_user_ids = user_ids

    message.mentions_wildcard = False
    message.mentions_user_ids = set()
    message.mentions_user_group_ids = set()
    message.alert_words = set()
    message.links_for_preview = set()

    if realm is None:
        realm = message.get_realm()

    possible_words = set()  # type: Set[str]
    if realm_alert_words is not None:
        for user_id, words in realm_alert_words.items():
            if user_id in message_user_ids:
                possible_words.update(set(words))

    sent_by_bot = get_user_profile_by_id(message.sender_id).is_bot

    # DO MAIN WORK HERE -- call bugdown to convert
    rendered_content = bugdown.convert(
        content,
        message=message,
        message_realm=realm,
        possible_words=possible_words,
        sent_by_bot=sent_by_bot,
        mention_data=mention_data,
        email_gateway=email_gateway
    )

    if message is not None:
        message.user_ids_with_alert_words = set()

        if realm_alert_words is not None:
            for user_id, words in realm_alert_words.items():
                if user_id in message_user_ids:
                    if set(words).intersection(message.alert_words):
                        message.user_ids_with_alert_words.add(user_id)

    return rendered_content
示例#13
0
def build_message(topic_name: str, pub_date: float, message_id: int, content: str,
                  rendered_content: Optional[str], user_id: int, recipient_id: int,
                  has_image: bool=False, has_link: bool=False,
                  has_attachment: bool=True) -> ZerverFieldsT:
    zulip_message = Message(
        rendered_content_version=1,  # this is Zulip specific
        pub_date=pub_date,
        id=message_id,
        content=content,
        rendered_content=rendered_content,
        has_image=has_image,
        has_attachment=has_attachment,
        has_link=has_link)
    zulip_message.set_topic_name(topic_name)
    zulip_message_dict = model_to_dict(zulip_message,
                                       exclude=['recipient', 'sender', 'sending_client'])
    zulip_message_dict['sender'] = user_id
    zulip_message_dict['sending_client'] = 1
    zulip_message_dict['recipient'] = recipient_id

    return zulip_message_dict
示例#14
0
def do_render_markdown(message: Message,
                       content: str,
                       realm: Realm,
                       message_user_ids: Set[int],
                       sent_by_bot: bool,
                       translate_emoticons: bool,
                       realm_alert_words_automaton: Optional[ahocorasick.Automaton]=None,
                       mention_data: Optional[bugdown.MentionData]=None,
                       email_gateway: Optional[bool]=False) -> str:
    """Return HTML for given markdown. Bugdown may add properties to the
    message object such as `mentions_user_ids`, `mentions_user_group_ids`, and
    `mentions_wildcard`.  These are only on this Django object and are not
    saved in the database.
    """

    message.mentions_wildcard = False
    message.mentions_user_ids = set()
    message.mentions_user_group_ids = set()
    message.alert_words = set()
    message.links_for_preview = set()
    message.user_ids_with_alert_words = set()

    # DO MAIN WORK HERE -- call bugdown to convert
    rendered_content = bugdown.convert(
        content,
        realm_alert_words_automaton=realm_alert_words_automaton,
        message=message,
        message_realm=realm,
        sent_by_bot=sent_by_bot,
        translate_emoticons=translate_emoticons,
        mention_data=mention_data,
        email_gateway=email_gateway
    )
    return rendered_content
示例#15
0
def re_render_content_for_management_command(message):
    # type: (Message) -> None
    '''
    Please avoid using this function, as its only used in a management command that
    is somewhat deprecated.
    '''
    assert Message.need_to_render_content(message.rendered_content,
                                          message.rendered_content_version,
                                          bugdown.version)

    rendered_content = render_markdown(message, message.content)
    message.rendered_content = rendered_content
    message.rendered_content_version = bugdown.version
    message.save_rendered_content()
示例#16
0
def render_markdown(message, content, realm_id=None, realm_alert_words=None, message_users=None):
    # type: (Message, Text, Optional[int], Optional[RealmAlertWords], Set[UserProfile]) -> Text
    """Return HTML for given markdown. Bugdown may add properties to the
    message object such as `mentions_user_ids` and `mentions_wildcard`.
    These are only on this Django object and are not saved in the
    database.
    """

    if message_users is None:
        message_user_ids = set() # type: Set[int]
    else:
        message_user_ids = {u.id for u in message_users}

    if message is not None:
        message.mentions_wildcard = False
        message.is_me_message = False
        message.mentions_user_ids = set()
        message.alert_words = set()
        message.links_for_preview = set()

        if realm_id is None:
            realm_id = message.sender.realm_id
        if message.sending_client.name == "zephyr_mirror" and message.sender.realm.is_zephyr_mirror_realm:
            # Use slightly customized Markdown processor for content
            # delivered via zephyr_mirror
            realm_id = bugdown.ZEPHYR_MIRROR_BUGDOWN_KEY

    possible_words = set() # type: Set[Text]
    if realm_alert_words is not None:
        for user_id, words in realm_alert_words.items():
            if user_id in message_user_ids:
                possible_words.update(set(words))

    # DO MAIN WORK HERE -- call bugdown to convert
    rendered_content = bugdown.convert(content, realm_filters_key=realm_id, message=message,
                                       possible_words=possible_words)

    if message is not None:
        message.user_ids_with_alert_words = set()

        if realm_alert_words is not None:
            for user_id, words in realm_alert_words.items():
                if user_id in message_user_ids:
                    if set(words).intersection(message.alert_words):
                        message.user_ids_with_alert_words.add(user_id)

        message.is_me_message = Message.is_status_message(content, rendered_content)

    return rendered_content
    def construct_message(self, scheduled_message: ScheduledMessage) -> Dict[str, Any]:
        message = Message()
        original_sender = scheduled_message.sender
        message.content = scheduled_message.content
        message.recipient = scheduled_message.recipient
        message.subject = scheduled_message.subject
        message.pub_date = timezone_now()
        message.sending_client = scheduled_message.sending_client

        delivery_type = scheduled_message.delivery_type
        if delivery_type == ScheduledMessage.SEND_LATER:
            message.sender = original_sender
        elif delivery_type == ScheduledMessage.REMIND:
            message.sender = get_user(settings.REMINDER_BOT, original_sender.realm)

        return {'message': message, 'stream': scheduled_message.stream,
                'realm': scheduled_message.realm}
示例#18
0
def get_common_payload(message: Message) -> Dict[str, Any]:
    data = {}  # type: Dict[str, Any]

    # These will let the app support logging into multiple realms and servers.
    data['server'] = settings.EXTERNAL_HOST
    data['realm_id'] = message.sender.realm.id

    # `sender_id` is preferred, but some existing versions use `sender_email`.
    data['sender_id'] = message.sender.id
    data['sender_email'] = message.sender.email

    if message.is_stream_message():
        data['recipient_type'] = "stream"
        data['stream'] = get_display_recipient(message.recipient)
        data['topic'] = message.subject
    else:
        data['recipient_type'] = "private"

    return data
示例#19
0
def get_message_payload(message: Message) -> Dict[str, Any]:
    '''Common fields for `message` payloads, for all platforms.'''
    data = get_base_payload(message.sender.realm)

    # `sender_id` is preferred, but some existing versions use `sender_email`.
    data['sender_id'] = message.sender.id
    data['sender_email'] = message.sender.email

    if message.recipient.type == Recipient.STREAM:
        data['recipient_type'] = "stream"
        data['stream'] = get_display_recipient(message.recipient)
        data['topic'] = message.topic_name()
    elif message.recipient.type == Recipient.HUDDLE:
        data['recipient_type'] = "private"
        data['pm_users'] = huddle_users(message.recipient.id)
    else:  # Recipient.PERSONAL
        data['recipient_type'] = "private"

    return data
示例#20
0
 def to_dict_uncached_helper(message: Message) -> Dict[str, Any]:
     return MessageDict.build_message_dict(
         message = message,
         message_id = message.id,
         last_edit_time = message.last_edit_time,
         edit_history = message.edit_history,
         content = message.content,
         topic_name = message.topic_name(),
         pub_date = message.pub_date,
         rendered_content = message.rendered_content,
         rendered_content_version = message.rendered_content_version,
         sender_id = message.sender.id,
         sender_realm_id = message.sender.realm_id,
         sending_client_name = message.sending_client.name,
         recipient_id = message.recipient.id,
         recipient_type = message.recipient.type,
         recipient_type_id = message.recipient.type_id,
         reactions = Reaction.get_raw_db_rows([message.id]),
         submessages = SubMessage.get_raw_db_rows([message.id]),
     )
示例#21
0
def render_markdown(message: Message,
                    content: str,
                    realm: Optional[Realm]=None,
                    realm_alert_words: Optional[RealmAlertWords]=None,
                    user_ids: Optional[Set[int]]=None,
                    mention_data: Optional[bugdown.MentionData]=None,
                    email_gateway: Optional[bool]=False) -> str:
    '''
    This is basically just a wrapper for do_render_markdown.
    '''

    if user_ids is None:
        message_user_ids = set()  # type: Set[int]
    else:
        message_user_ids = user_ids

    if realm is None:
        realm = message.get_realm()

    if realm_alert_words is None:
        realm_alert_words = dict()

    sender = get_user_profile_by_id(message.sender_id)
    sent_by_bot = sender.is_bot
    translate_emoticons = sender.translate_emoticons

    rendered_content = do_render_markdown(
        message=message,
        content=content,
        realm=realm,
        realm_alert_words=realm_alert_words,
        message_user_ids=message_user_ids,
        sent_by_bot=sent_by_bot,
        translate_emoticons=translate_emoticons,
        mention_data=mention_data,
        email_gateway=email_gateway,
    )

    return rendered_content
示例#22
0
def do_render_markdown(message: Message,
                       content: str,
                       realm: Realm,
                       realm_alert_words: RealmAlertWords,
                       message_user_ids: Set[int],
                       sent_by_bot: bool,
                       translate_emoticons: bool,
                       mention_data: Optional[bugdown.MentionData]=None,
                       email_gateway: Optional[bool]=False) -> str:
    """Return HTML for given markdown. Bugdown may add properties to the
    message object such as `mentions_user_ids`, `mentions_user_group_ids`, and
    `mentions_wildcard`.  These are only on this Django object and are not
    saved in the database.
    """

    message.mentions_wildcard = False
    message.mentions_user_ids = set()
    message.mentions_user_group_ids = set()
    message.alert_words = set()
    message.links_for_preview = set()

    possible_words = set()  # type: Set[str]
    for user_id, words in realm_alert_words.items():
        if user_id in message_user_ids:
            possible_words.update(set(words))

    # DO MAIN WORK HERE -- call bugdown to convert
    rendered_content = bugdown.convert(
        content,
        message=message,
        message_realm=realm,
        possible_words=possible_words,
        sent_by_bot=sent_by_bot,
        translate_emoticons=translate_emoticons,
        mention_data=mention_data,
        email_gateway=email_gateway
    )

    message.user_ids_with_alert_words = set()

    for user_id, words in realm_alert_words.items():
        if user_id in message_user_ids:
            if set(words).intersection(message.alert_words):
                message.user_ids_with_alert_words.add(user_id)

    return rendered_content
示例#23
0
def get_common_payload(message: Message) -> Dict[str, Any]:
    data = {}  # type: Dict[str, Any]

    # These will let the app support logging into multiple realms and servers.
    data['server'] = settings.EXTERNAL_HOST
    data['realm_id'] = message.sender.realm.id
    data['realm_uri'] = message.sender.realm.uri

    # `sender_id` is preferred, but some existing versions use `sender_email`.
    data['sender_id'] = message.sender.id
    data['sender_email'] = message.sender.email

    if message.recipient.type == Recipient.STREAM:
        data['recipient_type'] = "stream"
        data['stream'] = get_display_recipient(message.recipient)
        data['topic'] = message.topic_name()
    elif message.recipient.type == Recipient.HUDDLE:
        data['recipient_type'] = "private"
        data['pm_users'] = huddle_users(message.recipient.id)
    else:  # Recipient.PERSONAL
        data['recipient_type'] = "private"

    return data
示例#24
0
def do_render_markdown(message: Message,
                       content: str,
                       realm: Realm,
                       realm_alert_words: RealmAlertWords,
                       message_user_ids: Set[int],
                       sent_by_bot: bool,
                       translate_emoticons: bool,
                       mention_data: Optional[bugdown.MentionData] = None,
                       email_gateway: Optional[bool] = False) -> str:
    """Return HTML for given markdown. Bugdown may add properties to the
    message object such as `mentions_user_ids`, `mentions_user_group_ids`, and
    `mentions_wildcard`.  These are only on this Django object and are not
    saved in the database.
    """

    message.mentions_wildcard = False
    message.mentions_user_ids = set()
    message.mentions_user_group_ids = set()
    message.alert_words = set()
    message.links_for_preview = set()

    possible_words = set()  # type: Set[str]
    for user_id, words in realm_alert_words.items():
        if user_id in message_user_ids:
            possible_words.update(set(words))

    # DO MAIN WORK HERE -- call bugdown to convert
    rendered_content = bugdown.convert(content,
                                       message=message,
                                       message_realm=realm,
                                       possible_words=possible_words,
                                       sent_by_bot=sent_by_bot,
                                       translate_emoticons=translate_emoticons,
                                       mention_data=mention_data,
                                       email_gateway=email_gateway)

    message.user_ids_with_alert_words = set()

    for user_id, words in realm_alert_words.items():
        if user_id in message_user_ids:
            if set(words).intersection(message.alert_words):
                message.user_ids_with_alert_words.add(user_id)

    return rendered_content
示例#25
0
 def message_header(user_profile: UserProfile, message: Message) -> Dict[str, Any]:
     disp_recipient = get_display_recipient(message.recipient)
     if message.recipient.type == Recipient.PERSONAL:
         header = u"You and %s" % (message.sender.full_name,)
         html_link = pm_narrow_url(user_profile.realm, [message.sender.email])
         header_html = u"<a style='color: #ffffff;' href='%s'>%s</a>" % (html_link, header)
     elif message.recipient.type == Recipient.HUDDLE:
         assert not isinstance(disp_recipient, Text)
         other_recipients = [r['full_name'] for r in disp_recipient
                             if r['email'] != user_profile.email]
         header = u"You and %s" % (", ".join(other_recipients),)
         html_link = pm_narrow_url(user_profile.realm, [r["email"] for r in disp_recipient
                                   if r["email"] != user_profile.email])
         header_html = u"<a style='color: #ffffff;' href='%s'>%s</a>" % (html_link, header)
     else:
         assert isinstance(disp_recipient, Text)
         header = u"%s > %s" % (disp_recipient, message.topic_name())
         stream_link = stream_narrow_url(user_profile.realm, disp_recipient)
         topic_link = topic_narrow_url(user_profile.realm, disp_recipient, message.subject)
         header_html = u"<a href='%s'>%s</a> > <a href='%s'>%s</a>" % (
             stream_link, disp_recipient, topic_link, message.subject)
     return {"plain": header,
             "html": header_html,
             "stream_message": message.recipient.type_name() == "stream"}
示例#26
0
def get_narrow_url(user_profile: UserProfile,
                   message: Message,
                   display_recipient: Optional[DisplayRecipientT] = None,
                   stream: Optional[Stream] = None) -> str:
    """The display_recipient and stream arguments are optional.  If not
    provided, we'll compute them from the message; they exist as a
    performance optimization for cases where the caller needs those
    data too.
    """
    if message.recipient.type == Recipient.PERSONAL:
        assert stream is None
        assert display_recipient is None
        return personal_narrow_url(
            realm=user_profile.realm,
            sender=message.sender,
        )
    elif message.recipient.type == Recipient.HUDDLE:
        assert stream is None
        if display_recipient is None:
            display_recipient = get_display_recipient(message.recipient)
        assert display_recipient is not None
        assert not isinstance(display_recipient, str)
        other_user_ids = [
            r['id'] for r in display_recipient if r['id'] != user_profile.id
        ]
        return huddle_narrow_url(
            realm=user_profile.realm,
            other_user_ids=other_user_ids,
        )
    else:
        assert display_recipient is None
        if stream is None:
            stream = Stream.objects.only(
                'id', 'name').get(id=message.recipient.type_id)
        return topic_narrow_url(user_profile.realm, stream,
                                message.topic_name())
示例#27
0
def do_claim_attachments(message: Message,
                         potential_path_ids: List[str]) -> bool:
    claimed = False
    for path_id in potential_path_ids:
        user_profile = message.sender
        is_message_realm_public = False
        is_message_web_public = False
        if message.is_stream_message():
            stream = Stream.objects.get(id=message.recipient.type_id)
            is_message_realm_public = stream.is_public()
            is_message_web_public = stream.is_web_public

        if not validate_attachment_request(user_profile, path_id):
            # Technically, there are 2 cases here:
            # * The user put something in their message that has the form
            # of an upload, but doesn't correspond to a file that doesn't
            # exist.  validate_attachment_request will return None.
            # * The user is trying to send a link to a file they don't have permission to
            # access themselves.  validate_attachment_request will return False.
            #
            # Either case is unusual and suggests a UI bug that got
            # the user in this situation, so we log in these cases.
            logging.warning(
                "User %s tried to share upload %s in message %s, but lacks permission",
                user_profile.id,
                path_id,
                message.id,
            )
            continue

        claimed = True
        attachment = claim_attachment(user_profile, path_id, message,
                                      is_message_realm_public,
                                      is_message_web_public)
        notify_attachment_update(user_profile, "update", attachment.to_dict())
    return claimed
示例#28
0
文件: message.py 项目: zqyoung1/zulip
def render_markdown(message: Message,
                    content: str,
                    realm: Optional[Realm]=None,
                    realm_alert_words_automaton: Optional[ahocorasick.Automaton]=None,
                    user_ids: Optional[Set[int]]=None,
                    mention_data: Optional[bugdown.MentionData]=None,
                    email_gateway: Optional[bool]=False) -> str:
    '''
    This is basically just a wrapper for do_render_markdown.
    '''

    if user_ids is None:
        message_user_ids = set()  # type: Set[int]
    else:
        message_user_ids = user_ids

    if realm is None:
        realm = message.get_realm()

    sender = get_user_profile_by_id(message.sender_id)
    sent_by_bot = sender.is_bot
    translate_emoticons = sender.translate_emoticons

    rendered_content = do_render_markdown(
        message=message,
        content=content,
        realm=realm,
        realm_alert_words_automaton=realm_alert_words_automaton,
        message_user_ids=message_user_ids,
        sent_by_bot=sent_by_bot,
        translate_emoticons=translate_emoticons,
        mention_data=mention_data,
        email_gateway=email_gateway,
    )

    return rendered_content
示例#29
0
 def message_header(user_profile: UserProfile,
                    message: Message) -> Dict[str, Any]:
     if message.recipient.type == Recipient.PERSONAL:
         header = "You and %s" % (message.sender.full_name, )
         html_link = pm_narrow_url(user_profile.realm,
                                   [message.sender.email])
         header_html = "<a style='color: #ffffff;' href='%s'>%s</a>" % (
             html_link, header)
     elif message.recipient.type == Recipient.HUDDLE:
         disp_recipient = get_display_recipient(message.recipient)
         assert not isinstance(disp_recipient, str)
         other_recipients = [
             r['full_name'] for r in disp_recipient
             if r['email'] != user_profile.email
         ]
         header = "You and %s" % (", ".join(other_recipients), )
         html_link = pm_narrow_url(user_profile.realm, [
             r["email"]
             for r in disp_recipient if r["email"] != user_profile.email
         ])
         header_html = "<a style='color: #ffffff;' href='%s'>%s</a>" % (
             html_link, header)
     else:
         stream = Stream.objects.only(
             'id', 'name').get(id=message.recipient.type_id)
         header = "%s > %s" % (stream.name, message.topic_name())
         stream_link = stream_narrow_url(user_profile.realm, stream)
         topic_link = topic_narrow_url(user_profile.realm, stream,
                                       message.subject)
         header_html = "<a href='%s'>%s</a> > <a href='%s'>%s</a>" % (
             stream_link, stream.name, topic_link, message.subject)
     return {
         "plain": header,
         "html": header_html,
         "stream_message": message.recipient.type_name() == "stream"
     }
示例#30
0
    def test_bulk_message_fetching(self) -> None:
        sender = self.example_user("othello")
        receiver = self.example_user("hamlet")
        pm_recipient = Recipient.objects.get(type_id=receiver.id, type=Recipient.PERSONAL)
        stream_name = "Çiğdem"
        stream = self.make_stream(stream_name)
        stream_recipient = Recipient.objects.get(type_id=stream.id, type=Recipient.STREAM)
        sending_client = make_client(name="test suite")

        ids = []
        for i in range(300):
            for recipient in [pm_recipient, stream_recipient]:
                message = Message(
                    sender=sender,
                    recipient=recipient,
                    content=f"whatever {i}",
                    rendered_content="DOES NOT MATTER",
                    rendered_content_version=markdown_version,
                    date_sent=timezone_now(),
                    sending_client=sending_client,
                    last_edit_time=timezone_now(),
                    edit_history="[]",
                )
                message.set_topic_name("whatever")
                message.save()
                ids.append(message.id)

                Reaction.objects.create(
                    user_profile=sender, message=message, emoji_name="simple_smile"
                )

        num_ids = len(ids)
        self.assertTrue(num_ids >= 600)

        flush_per_request_caches()
        with queries_captured() as queries:
            rows = list(MessageDict.get_raw_db_rows(ids))

            objs = [MessageDict.build_dict_from_raw_db_row(row) for row in rows]
            MessageDict.post_process_dicts(objs, apply_markdown=False, client_gravatar=False)

        self.assert_length(queries, 7)
        self.assertEqual(len(rows), num_ids)
示例#31
0
    def test_sew_messages_and_reaction(self) -> None:
        sender = self.example_user("othello")
        receiver = self.example_user("hamlet")
        pm_recipient = Recipient.objects.get(type_id=receiver.id,
                                             type=Recipient.PERSONAL)
        stream_name = "Çiğdem"
        stream = self.make_stream(stream_name)
        stream_recipient = Recipient.objects.get(type_id=stream.id,
                                                 type=Recipient.STREAM)
        sending_client = make_client(name="test suite")

        needed_ids = []
        for i in range(5):
            for recipient in [pm_recipient, stream_recipient]:
                message = Message(
                    sender=sender,
                    recipient=recipient,
                    content=f"whatever {i}",
                    date_sent=timezone_now(),
                    sending_client=sending_client,
                    last_edit_time=timezone_now(),
                    edit_history="[]",
                )
                message.set_topic_name("whatever")
                message.save()
                needed_ids.append(message.id)
                reaction = Reaction(user_profile=sender,
                                    message=message,
                                    emoji_name="simple_smile")
                reaction.save()

        messages = Message.objects.filter(id__in=needed_ids).values(
            *["id", "content"])
        reactions = Reaction.get_raw_db_rows(needed_ids)
        tied_data = sew_messages_and_reactions(messages, reactions)
        for data in tied_data:
            self.assert_length(data["reactions"], 1)
            self.assertEqual(data["reactions"][0]["emoji_name"],
                             "simple_smile")
            self.assertTrue(data["id"])
            self.assertTrue(data["content"])
示例#32
0
    def test_sew_messages_and_reaction(self) -> None:
        sender = self.example_user('othello')
        receiver = self.example_user('hamlet')
        pm_recipient = Recipient.objects.get(type_id=receiver.id,
                                             type=Recipient.PERSONAL)
        stream_name = 'Çiğdem'
        stream = self.make_stream(stream_name)
        stream_recipient = Recipient.objects.get(type_id=stream.id,
                                                 type=Recipient.STREAM)
        sending_client = make_client(name="test suite")

        needed_ids = []
        for i in range(5):
            for recipient in [pm_recipient, stream_recipient]:
                message = Message(
                    sender=sender,
                    recipient=recipient,
                    content=f'whatever {i}',
                    date_sent=timezone_now(),
                    sending_client=sending_client,
                    last_edit_time=timezone_now(),
                    edit_history='[]',
                )
                message.set_topic_name('whatever')
                message.save()
                needed_ids.append(message.id)
                reaction = Reaction(user_profile=sender,
                                    message=message,
                                    emoji_name='simple_smile')
                reaction.save()

        messages = Message.objects.filter(id__in=needed_ids).values(
            *['id', 'content'])
        reactions = Reaction.get_raw_db_rows(needed_ids)
        tied_data = sew_messages_and_reactions(messages, reactions)
        for data in tied_data:
            self.assertEqual(len(data['reactions']), 1)
            self.assertEqual(data['reactions'][0]['emoji_name'],
                             'simple_smile')
            self.assertTrue(data['id'])
            self.assertTrue(data['content'])
示例#33
0
        def create_test_message(topic: str) -> int:
            # TODO: Clean this up to send messages the normal way.

            hamlet = self.example_user("hamlet")
            message = Message(
                sender=hamlet,
                recipient=recipient,
                content="whatever",
                date_sent=timezone_now(),
                sending_client=get_client("whatever"),
            )
            message.set_topic_name(topic)
            message.save()

            UserMessage.objects.create(
                user_profile=user_profile,
                message=message,
                flags=0,
            )

            return message.id
示例#34
0
    def build_message_dict(apply_markdown, message, message_id, last_edit_time,
                           edit_history, content, subject, pub_date,
                           rendered_content, rendered_content_version,
                           sender_id, sender_email, sender_realm_id,
                           sender_realm_str, sender_full_name,
                           sender_short_name, sender_avatar_source,
                           sender_avatar_version, sender_is_mirror_dummy,
                           sending_client_name, recipient_id, recipient_type,
                           recipient_type_id, reactions):
        # type: (bool, Optional[Message], int, Optional[datetime.datetime], Optional[Text], Text, Text, datetime.datetime, Optional[Text], Optional[int], int, Text, int, Text, Text, Text, Text, int, bool, Text, int, int, int, List[Dict[str, Any]]) -> Dict[str, Any]

        avatar_url = get_avatar_url(sender_avatar_source, sender_email,
                                    sender_avatar_version)

        display_recipient = get_display_recipient_by_id(
            recipient_id, recipient_type, recipient_type_id)

        if recipient_type == Recipient.STREAM:
            display_type = "stream"
        elif recipient_type in (Recipient.HUDDLE, Recipient.PERSONAL):
            assert not isinstance(display_recipient, Text)
            display_type = "private"
            if len(display_recipient) == 1:
                # add the sender in if this isn't a message between
                # someone and his self, preserving ordering
                recip = {
                    'email': sender_email,
                    'full_name': sender_full_name,
                    'short_name': sender_short_name,
                    'id': sender_id,
                    'is_mirror_dummy': sender_is_mirror_dummy
                }
                if recip['email'] < display_recipient[0]['email']:
                    display_recipient = [recip, display_recipient[0]]
                elif recip['email'] > display_recipient[0]['email']:
                    display_recipient = [display_recipient[0], recip]

        obj = dict(
            id=message_id,
            sender_email=sender_email,
            sender_full_name=sender_full_name,
            sender_short_name=sender_short_name,
            sender_realm_str=sender_realm_str,
            sender_id=sender_id,
            type=display_type,
            display_recipient=display_recipient,
            recipient_id=recipient_id,
            subject=subject,
            timestamp=datetime_to_timestamp(pub_date),
            gravatar_hash=gravatar_hash(sender_email),  # Deprecated June 2013
            avatar_url=avatar_url,
            client=sending_client_name)

        if obj['type'] == 'stream':
            obj['stream_id'] = recipient_type_id

        obj['subject_links'] = bugdown.subject_links(sender_realm_id, subject)

        if last_edit_time is not None:
            obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
            obj['edit_history'] = ujson.loads(edit_history)

        if apply_markdown:
            if Message.need_to_render_content(rendered_content,
                                              rendered_content_version,
                                              bugdown.version):
                if message is None:
                    # We really shouldn't be rendering objects in this method, but there is
                    # a scenario where we upgrade the version of bugdown and fail to run
                    # management commands to re-render historical messages, and then we
                    # need to have side effects.  This method is optimized to not need full
                    # blown ORM objects, but the bugdown renderer is unfortunately highly
                    # coupled to Message, and we also need to persist the new rendered content.
                    # If we don't have a message object passed in, we get one here.  The cost
                    # of going to the DB here should be overshadowed by the cost of rendering
                    # and updating the row.
                    # TODO: see #1379 to eliminate bugdown dependencies
                    message = Message.objects.select_related().get(
                        id=message_id)

                # It's unfortunate that we need to have side effects on the message
                # in some cases.
                rendered_content = render_markdown(message,
                                                   content,
                                                   realm=message.get_realm())
                message.rendered_content = rendered_content
                message.rendered_content_version = bugdown.version
                message.save_rendered_content()

            if rendered_content is not None:
                obj['content'] = rendered_content
            else:
                obj['content'] = u'<p>[Zulip note: Sorry, we could not understand the formatting of your message]</p>'

            obj['content_type'] = 'text/html'
        else:
            obj['content'] = content
            obj['content_type'] = 'text/x-markdown'

        obj['reactions'] = [
            ReactionDict.build_dict_from_raw_db_row(reaction)
            for reaction in reactions
        ]
        return obj
示例#35
0
def archive(request: HttpRequest, stream_id: int,
            topic_name: str) -> HttpResponse:
    def get_response(rendered_message_list: List[str], is_web_public: bool,
                     stream_name: str) -> HttpResponse:
        return render(request,
                      'zerver/archive/index.html',
                      context={
                          'is_web_public': is_web_public,
                          'message_list': rendered_message_list,
                          'stream': stream_name,
                          'topic': topic_name,
                      })

    try:
        stream = get_stream_by_id(stream_id)
    except JsonableError:
        return get_response([], False, '')

    if not stream.is_web_public:
        return get_response([], False, '')

    all_messages = list(
        Message.objects.select_related('sender').filter(
            recipient__type_id=stream_id,
            subject=topic_name).order_by('pub_date'))
    if not all_messages:
        return get_response([], True, stream.name)

    rendered_message_list = []
    prev_sender = None
    for msg in all_messages:
        include_sender = False
        status_message = Message.is_status_message(msg.content,
                                                   msg.rendered_content)
        if not prev_sender or prev_sender != msg.sender or status_message:
            if status_message:
                prev_sender = None
            else:
                prev_sender = msg.sender
            include_sender = True
        if status_message:
            status_message = msg.rendered_content[4 + 3:-4]
        context = {
            'sender_full_name':
            msg.sender.full_name,
            'timestampstr':
            datetime_to_timestamp(
                msg.last_edit_time if msg.last_edit_time else msg.pub_date),
            'message_content':
            msg.rendered_content,
            'avatar_url':
            get_gravatar_url(msg.sender.email, 1),
            'include_sender':
            include_sender,
            'status_message':
            status_message,
        }
        rendered_msg = loader.render_to_string(
            'zerver/archive/single_message.html', context)
        rendered_message_list.append(rendered_msg)
    return get_response(rendered_message_list, True, stream.name)
示例#36
0
文件: message.py 项目: emillind/zulip
def render_markdown(message: Message,
                    content: Text,
                    realm: Optional[Realm]=None,
                    realm_alert_words: Optional[RealmAlertWords]=None,
                    user_ids: Optional[Set[int]]=None,
                    mention_data: Optional[bugdown.MentionData]=None,
                    email_gateway: Optional[bool]=False) -> Text:
    """Return HTML for given markdown. Bugdown may add properties to the
    message object such as `mentions_user_ids`, `mentions_user_group_ids`, and
    `mentions_wildcard`.  These are only on this Django object and are not
    saved in the database.
    """

    if user_ids is None:
        message_user_ids = set()  # type: Set[int]
    else:
        message_user_ids = user_ids

    if message is not None:
        message.mentions_wildcard = False
        message.mentions_user_ids = set()
        message.mentions_user_group_ids = set()
        message.alert_words = set()
        message.links_for_preview = set()

        if realm is None:
            realm = message.get_realm()

    possible_words = set()  # type: Set[Text]
    if realm_alert_words is not None:
        for user_id, words in realm_alert_words.items():
            if user_id in message_user_ids:
                possible_words.update(set(words))

    if message is None:
        # If we don't have a message, then we are in the compose preview
        # codepath, so we know we are dealing with a human.
        sent_by_bot = False
    else:
        sent_by_bot = get_user_profile_by_id(message.sender_id).is_bot

    # DO MAIN WORK HERE -- call bugdown to convert
    rendered_content = bugdown.convert(
        content,
        message=message,
        message_realm=realm,
        possible_words=possible_words,
        sent_by_bot=sent_by_bot,
        mention_data=mention_data,
        email_gateway=email_gateway
    )

    if message is not None:
        message.user_ids_with_alert_words = set()

        if realm_alert_words is not None:
            for user_id, words in realm_alert_words.items():
                if user_id in message_user_ids:
                    if set(words).intersection(message.alert_words):
                        message.user_ids_with_alert_words.add(user_id)

    return rendered_content
示例#37
0
    def build_message_dict(
            apply_markdown,
            message,
            message_id,
            last_edit_time,
            edit_history,
            content,
            subject,
            pub_date,
            rendered_content,
            rendered_content_version,
            sender_id,
            sender_email,
            sender_realm_id,
            sender_realm_str,
            sender_avatar_source,
            sender_avatar_version,
            sender_is_mirror_dummy,
            sending_client_name,
            recipient_id,
            recipient_type,
            recipient_type_id,
            reactions
    ):
        # type: (bool, Optional[Message], int, Optional[datetime.datetime], Optional[Text], Text, Text, datetime.datetime, Optional[Text], Optional[int], int, Text, int, Text, Text, int, bool, Text, int, int, int, List[Dict[str, Any]]) -> Dict[str, Any]

        avatar_url = avatar_url_from_dict(dict(
            avatar_source=sender_avatar_source,
            avatar_version=sender_avatar_version,
            email=sender_email,
            id=sender_id,
            realm_id=sender_realm_id))

        obj = dict(
            id                = message_id,
            sender_email      = sender_email,
            sender_realm_str  = sender_realm_str,
            sender_id         = sender_id,
            recipient_type_id = recipient_type_id,
            recipient_type    = recipient_type,
            recipient_id      = recipient_id,
            subject           = subject,
            timestamp         = datetime_to_timestamp(pub_date),
            avatar_url        = avatar_url,
            client            = sending_client_name)

        obj['raw_display_recipient'] = get_display_recipient_by_id(
            recipient_id,
            recipient_type,
            recipient_type_id
        )

        obj['sender_is_mirror_dummy'] = sender_is_mirror_dummy

        obj['subject_links'] = bugdown.subject_links(sender_realm_id, subject)

        if last_edit_time is not None:
            obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
            assert edit_history is not None
            obj['edit_history'] = ujson.loads(edit_history)

        if apply_markdown:
            if Message.need_to_render_content(rendered_content, rendered_content_version, bugdown.version):
                if message is None:
                    # We really shouldn't be rendering objects in this method, but there is
                    # a scenario where we upgrade the version of bugdown and fail to run
                    # management commands to re-render historical messages, and then we
                    # need to have side effects.  This method is optimized to not need full
                    # blown ORM objects, but the bugdown renderer is unfortunately highly
                    # coupled to Message, and we also need to persist the new rendered content.
                    # If we don't have a message object passed in, we get one here.  The cost
                    # of going to the DB here should be overshadowed by the cost of rendering
                    # and updating the row.
                    # TODO: see #1379 to eliminate bugdown dependencies
                    message = Message.objects.select_related().get(id=message_id)

                assert message is not None  # Hint for mypy.
                # It's unfortunate that we need to have side effects on the message
                # in some cases.
                rendered_content = render_markdown(message, content, realm=message.get_realm())
                message.rendered_content = rendered_content
                message.rendered_content_version = bugdown.version
                message.save_rendered_content()

            if rendered_content is not None:
                obj['content'] = rendered_content
            else:
                obj['content'] = u'<p>[Zulip note: Sorry, we could not understand the formatting of your message]</p>'

            obj['content_type'] = 'text/html'
        else:
            obj['content'] = content
            obj['content_type'] = 'text/x-markdown'

        if rendered_content is not None:
            obj['is_me_message'] = Message.is_status_message(content, rendered_content)
        else:
            obj['is_me_message'] = False

        obj['reactions'] = [ReactionDict.build_dict_from_raw_db_row(reaction)
                            for reaction in reactions]
        return obj
示例#38
0
def save_message_rendered_content(message: Message, content: str) -> str:
    rendered_content = render_markdown(message, content, realm=message.get_realm())
    message.rendered_content = rendered_content
    message.rendered_content_version = bugdown.version
    message.save_rendered_content()
    return rendered_content
示例#39
0
def filter_by_exact_message_topic(query: QuerySet,
                                  message: Message) -> QuerySet:
    topic_name = message.topic_name()
    return query.filter(subject=topic_name)
示例#40
0
def save_message_rendered_content(message: Message, content: str) -> str:
    rendered_content = render_markdown(message, content, realm=message.get_realm())
    message.rendered_content = rendered_content
    message.rendered_content_version = bugdown.version
    message.save_rendered_content()
    return rendered_content
示例#41
0
    def build_message_dict(
            message: Optional[Message],
            message_id: int,
            last_edit_time: Optional[datetime.datetime],
            edit_history: Optional[str],
            content: str,
            topic_name: str,
            pub_date: datetime.datetime,
            rendered_content: Optional[str],
            rendered_content_version: Optional[int],
            sender_id: int,
            sender_realm_id: int,
            sending_client_name: str,
            recipient_id: int,
            recipient_type: int,
            recipient_type_id: int,
            reactions: List[Dict[str, Any]],
            submessages: List[Dict[str, Any]]
    ) -> Dict[str, Any]:

        obj = dict(
            id                = message_id,
            sender_id         = sender_id,
            content           = content,
            recipient_type_id = recipient_type_id,
            recipient_type    = recipient_type,
            recipient_id      = recipient_id,
            timestamp         = datetime_to_timestamp(pub_date),
            client            = sending_client_name)

        obj[TOPIC_NAME] = topic_name
        obj['sender_realm_id'] = sender_realm_id

        obj['raw_display_recipient'] = get_display_recipient_by_id(
            recipient_id,
            recipient_type,
            recipient_type_id
        )

        obj[TOPIC_LINKS] = bugdown.topic_links(sender_realm_id, topic_name)

        if last_edit_time is not None:
            obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
            assert edit_history is not None
            obj['edit_history'] = ujson.loads(edit_history)

        if Message.need_to_render_content(rendered_content, rendered_content_version, bugdown.version):
            if message is None:
                # We really shouldn't be rendering objects in this method, but there is
                # a scenario where we upgrade the version of bugdown and fail to run
                # management commands to re-render historical messages, and then we
                # need to have side effects.  This method is optimized to not need full
                # blown ORM objects, but the bugdown renderer is unfortunately highly
                # coupled to Message, and we also need to persist the new rendered content.
                # If we don't have a message object passed in, we get one here.  The cost
                # of going to the DB here should be overshadowed by the cost of rendering
                # and updating the row.
                # TODO: see #1379 to eliminate bugdown dependencies
                message = Message.objects.select_related().get(id=message_id)

            assert message is not None  # Hint for mypy.
            # It's unfortunate that we need to have side effects on the message
            # in some cases.
            rendered_content = save_message_rendered_content(message, content)

        if rendered_content is not None:
            obj['rendered_content'] = rendered_content
        else:
            obj['rendered_content'] = ('<p>[Zulip note: Sorry, we could not ' +
                                       'understand the formatting of your message]</p>')

        if rendered_content is not None:
            obj['is_me_message'] = Message.is_status_message(content, rendered_content)
        else:
            obj['is_me_message'] = False

        obj['reactions'] = [ReactionDict.build_dict_from_raw_db_row(reaction)
                            for reaction in reactions]
        obj['submessages'] = submessages
        return obj
示例#42
0
def save_message_for_edit_use_case(message: Message) -> None:
    message.save(update_fields=[
        TOPIC_NAME, "content", "rendered_content", "rendered_content_version",
        "last_edit_time", "edit_history", "has_attachment", "has_image",
        "has_link", "recipient_id"
    ])
示例#43
0
    def build_message_dict(
            message: Optional[Message], message_id: int,
            last_edit_time: Optional[datetime.datetime],
            edit_history: Optional[str], content: str, topic_name: str,
            date_sent: datetime.datetime, rendered_content: Optional[str],
            rendered_content_version: Optional[int], sender_id: int,
            sender_realm_id: int, sending_client_name: str, recipient_id: int,
            recipient_type: int, recipient_type_id: int,
            reactions: List[Dict[str, Any]],
            submessages: List[Dict[str, Any]]) -> Dict[str, Any]:

        obj = dict(id=message_id,
                   sender_id=sender_id,
                   content=content,
                   recipient_type_id=recipient_type_id,
                   recipient_type=recipient_type,
                   recipient_id=recipient_id,
                   timestamp=datetime_to_timestamp(date_sent),
                   client=sending_client_name)

        obj[TOPIC_NAME] = topic_name
        obj['sender_realm_id'] = sender_realm_id

        # Render topic_links with the stream's realm instead of the
        # user's realm; this is important for messages sent by
        # cross-realm bots like NOTIFICATION_BOT.
        #
        # TODO: We could potentially avoid this database query in
        # common cases by optionally passing through the
        # stream_realm_id through the code path from do_send_messages
        # (where we've already fetched the data).  It would involve
        # somewhat messy plumbing, but would probably be worth it.
        rendering_realm_id = sender_realm_id
        if message and recipient_type == Recipient.STREAM:
            rendering_realm_id = Stream.objects.get(
                id=recipient_type_id).realm_id

        obj[TOPIC_LINKS] = bugdown.topic_links(rendering_realm_id, topic_name)

        if last_edit_time is not None:
            obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
            assert edit_history is not None
            obj['edit_history'] = ujson.loads(edit_history)

        if Message.need_to_render_content(rendered_content,
                                          rendered_content_version,
                                          bugdown.version):
            if message is None:
                # We really shouldn't be rendering objects in this method, but there is
                # a scenario where we upgrade the version of bugdown and fail to run
                # management commands to re-render historical messages, and then we
                # need to have side effects.  This method is optimized to not need full
                # blown ORM objects, but the bugdown renderer is unfortunately highly
                # coupled to Message, and we also need to persist the new rendered content.
                # If we don't have a message object passed in, we get one here.  The cost
                # of going to the DB here should be overshadowed by the cost of rendering
                # and updating the row.
                # TODO: see #1379 to eliminate bugdown dependencies
                message = Message.objects.select_related().get(id=message_id)

            assert message is not None  # Hint for mypy.
            # It's unfortunate that we need to have side effects on the message
            # in some cases.
            rendered_content = save_message_rendered_content(message, content)

        if rendered_content is not None:
            obj['rendered_content'] = rendered_content
        else:
            obj['rendered_content'] = (
                '<p>[Zulip note: Sorry, we could not ' +
                'understand the formatting of your message]</p>')

        if rendered_content is not None:
            obj['is_me_message'] = Message.is_status_message(
                content, rendered_content)
        else:
            obj['is_me_message'] = False

        obj['reactions'] = [
            ReactionDict.build_dict_from_raw_db_row(reaction)
            for reaction in reactions
        ]
        obj['submessages'] = submessages
        return obj
示例#44
0
    def build_message_dict(
            message: Optional[Message], message_id: int,
            last_edit_time: Optional[datetime.datetime],
            edit_history: Optional[str], content: str, subject: str,
            pub_date: datetime.datetime, rendered_content: Optional[str],
            rendered_content_version: Optional[int], sender_id: int,
            sender_realm_id: int, sending_client_name: str, recipient_id: int,
            recipient_type: int, recipient_type_id: int,
            reactions: List[Dict[str, Any]],
            submessages: List[Dict[str, Any]]) -> Dict[str, Any]:

        obj = dict(id=message_id,
                   sender_id=sender_id,
                   content=content,
                   recipient_type_id=recipient_type_id,
                   recipient_type=recipient_type,
                   recipient_id=recipient_id,
                   subject=subject,
                   timestamp=datetime_to_timestamp(pub_date),
                   client=sending_client_name)

        obj['sender_realm_id'] = sender_realm_id

        obj['raw_display_recipient'] = get_display_recipient_by_id(
            recipient_id, recipient_type, recipient_type_id)

        obj['subject_links'] = bugdown.topic_links(sender_realm_id, subject)

        if last_edit_time is not None:
            obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
            assert edit_history is not None
            obj['edit_history'] = ujson.loads(edit_history)

        if Message.need_to_render_content(rendered_content,
                                          rendered_content_version,
                                          bugdown.version):
            if message is None:
                # We really shouldn't be rendering objects in this method, but there is
                # a scenario where we upgrade the version of bugdown and fail to run
                # management commands to re-render historical messages, and then we
                # need to have side effects.  This method is optimized to not need full
                # blown ORM objects, but the bugdown renderer is unfortunately highly
                # coupled to Message, and we also need to persist the new rendered content.
                # If we don't have a message object passed in, we get one here.  The cost
                # of going to the DB here should be overshadowed by the cost of rendering
                # and updating the row.
                # TODO: see #1379 to eliminate bugdown dependencies
                message = Message.objects.select_related().get(id=message_id)

            assert message is not None  # Hint for mypy.
            # It's unfortunate that we need to have side effects on the message
            # in some cases.
            rendered_content = save_message_rendered_content(message, content)

        if rendered_content is not None:
            obj['rendered_content'] = rendered_content
        else:
            obj['rendered_content'] = (
                '<p>[Zulip note: Sorry, we could not ' +
                'understand the formatting of your message]</p>')

        if rendered_content is not None:
            obj['is_me_message'] = Message.is_status_message(
                content, rendered_content)
        else:
            obj['is_me_message'] = False

        obj['reactions'] = [
            ReactionDict.build_dict_from_raw_db_row(reaction)
            for reaction in reactions
        ]
        obj['submessages'] = submessages
        return obj
示例#45
0
文件: messages.py 项目: danshev/zulip
def get_old_messages_backend(request, user_profile,
                             anchor = REQ(converter=int),
                             num_before = REQ(converter=to_non_negative_int),
                             num_after = REQ(converter=to_non_negative_int),
                             narrow = REQ('narrow', converter=narrow_parameter, default=None),
                             use_first_unread_anchor = REQ(default=False, converter=ujson.loads),
                             apply_markdown=REQ(default=True,
                                                converter=ujson.loads)):
    include_history = ok_to_include_history(narrow, user_profile.realm)

    if include_history and not use_first_unread_anchor:
        query = select([column("id").label("message_id")], None, "zerver_message")
        inner_msg_id_col = literal_column("zerver_message.id")
    elif narrow is None:
        query = select([column("message_id"), column("flags")],
                       column("user_profile_id") == literal(user_profile.id),
                       "zerver_usermessage")
        inner_msg_id_col = column("message_id")
    else:
        # TODO: Don't do this join if we're not doing a search
        query = select([column("message_id"), column("flags")],
                       column("user_profile_id") == literal(user_profile.id),
                       join("zerver_usermessage", "zerver_message",
                            literal_column("zerver_usermessage.message_id") ==
                            literal_column("zerver_message.id")))
        inner_msg_id_col = column("message_id")

    num_extra_messages = 1
    is_search = False

    if narrow is not None:
        # Add some metadata to our logging data for narrows
        verbose_operators = []
        for term in narrow:
            if term['operator'] == "is":
                verbose_operators.append("is:" + term['operand'])
            else:
                verbose_operators.append(term['operator'])
        request._log_data['extra'] = "[%s]" % (",".join(verbose_operators),)

        # Build the query for the narrow
        num_extra_messages = 0
        builder = NarrowBuilder(user_profile, inner_msg_id_col)
        for term in narrow:
            if term['operator'] == 'search' and not is_search:
                query = query.column("subject").column("rendered_content")
                is_search = True
            query = builder.add_term(query, term)

    # We add 1 to the number of messages requested if no narrow was
    # specified to ensure that the resulting list always contains the
    # anchor message.  If a narrow was specified, the anchor message
    # might not match the narrow anyway.
    if num_after != 0:
        num_after += num_extra_messages
    else:
        num_before += num_extra_messages

    sa_conn = get_sqlalchemy_connection()
    if use_first_unread_anchor:
        condition = column("flags").op("&")(UserMessage.flags.read.mask) == 0

        # We exclude messages on muted topics when finding the first unread
        # message in this narrow
        muting_conditions = exclude_muting_conditions(user_profile, narrow)
        if muting_conditions:
            condition = and_(condition, *muting_conditions)

        first_unread_query = query.where(condition)
        first_unread_query = first_unread_query.order_by(inner_msg_id_col.asc()).limit(1)
        first_unread_result = list(sa_conn.execute(first_unread_query).fetchall())
        if len(first_unread_result) > 0:
            anchor = first_unread_result[0][0]
        else:
            anchor = 10000000000000000

    before_query = None
    after_query = None
    if num_before != 0:
        before_anchor = anchor
        if num_after != 0:
            # Don't include the anchor in both the before query and the after query
            before_anchor = anchor - 1
        before_query = query.where(inner_msg_id_col <= before_anchor) \
                            .order_by(inner_msg_id_col.desc()).limit(num_before)
    if num_after != 0:
        after_query = query.where(inner_msg_id_col >= anchor) \
                           .order_by(inner_msg_id_col.asc()).limit(num_after)

    if num_before == 0 and num_after == 0:
        # This can happen when a narrow is specified.
        after_query = query.where(inner_msg_id_col == anchor)

    if before_query is not None:
        if after_query is not None:
            query = union_all(before_query.self_group(), after_query.self_group())
        else:
            query = before_query
    else:
        query = after_query
    main_query = alias(query)
    query = select(main_query.c, None, main_query).order_by(column("message_id").asc())
    # This is a hack to tag the query we use for testing
    query = query.prefix_with("/* get_old_messages */")
    query_result = list(sa_conn.execute(query).fetchall())

    # The following is a little messy, but ensures that the code paths
    # are similar regardless of the value of include_history.  The
    # 'user_messages' dictionary maps each message to the user's
    # UserMessage object for that message, which we will attach to the
    # rendered message dict before returning it.  We attempt to
    # bulk-fetch rendered message dicts from remote cache using the
    # 'messages' list.
    search_fields = dict()
    message_ids = []
    user_message_flags = {}
    if include_history:
        message_ids = [row[0] for row in query_result]

        # TODO: This could be done with an outer join instead of two queries
        user_message_flags = dict((user_message.message_id, user_message.flags_list()) for user_message in
                                  UserMessage.objects.filter(user_profile=user_profile,
                                                             message__id__in=message_ids))
        for row in query_result:
            message_id = row[0]
            if user_message_flags.get(message_id) is None:
                user_message_flags[message_id] = ["read", "historical"]
            if is_search:
                (_, subject, rendered_content, content_matches, subject_matches) = row
                search_fields[message_id] = get_search_fields(rendered_content, subject,
                                                              content_matches, subject_matches)
    else:
        for row in query_result:
            message_id = row[0]
            flags = row[1]
            user_message_flags[message_id] = parse_usermessage_flags(flags)

            message_ids.append(message_id)

            if is_search:
                (_, _, subject, rendered_content, content_matches, subject_matches) = row
                search_fields[message_id] = get_search_fields(rendered_content, subject,
                                                              content_matches, subject_matches)

    cache_transformer = lambda row: Message.build_dict_from_raw_db_row(row, apply_markdown)
    id_fetcher = lambda row: row['id']

    message_dicts = generic_bulk_cached_fetch(lambda message_id: to_dict_cache_key_id(message_id, apply_markdown),
                                              Message.get_raw_db_rows,
                                              message_ids,
                                              id_fetcher=id_fetcher,
                                              cache_transformer=cache_transformer,
                                              extractor=extract_message_dict,
                                              setter=stringify_message_dict)

    message_list = []
    for message_id in message_ids:
        msg_dict = message_dicts[message_id]
        msg_dict.update({"flags": user_message_flags[message_id]})
        msg_dict.update(search_fields.get(message_id, {}))
        message_list.append(msg_dict)

    statsd.incr('loaded_old_messages', len(message_list))
    ret = {'messages': message_list,
           "result": "success",
           "msg": ""}
    return json_success(ret)
示例#46
0
 def do_test_subject(self, msg: Message, expected_subject: Optional[Text]) -> None:
     if expected_subject is not None:
         self.assertEqual(msg.topic_name(), expected_subject)
示例#47
0
def generate_and_send_messages(
    data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
                Callable[[str], Any], int]
) -> int:
    (tot_messages, personals_pairs, options, output, random_seed) = data
    random.seed(random_seed)

    with open(
            os.path.join(get_or_create_dev_uuid_var_path('test-backend'),
                         "test_messages.json"), "r") as infile:
        dialog = ujson.load(infile)
    random.shuffle(dialog)
    texts = itertools.cycle(dialog)

    recipient_streams = [
        klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
    ]  # type: List[int]
    recipient_huddles = [
        h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
    ]  # type: List[int]

    huddle_members = {}  # type: Dict[int, List[int]]
    for h in recipient_huddles:
        huddle_members[h] = [
            s.user_profile.id
            for s in Subscription.objects.filter(recipient_id=h)
        ]

    message_batch_size = options['batch_size']
    num_messages = 0
    random_max = 1000000
    recipients = {}  # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
    messages = []
    while num_messages < tot_messages:
        saved_data = {}  # type: Dict[str, Any]
        message = Message()
        message.sending_client = get_client('populate_db')

        message.content = next(texts)

        randkey = random.randint(1, random_max)
        if (num_messages > 0
                and random.randint(1, random_max) * 100. / random_max <
                options["stickyness"]):
            # Use an old recipient
            message_type, recipient_id, saved_data = recipients[num_messages -
                                                                1]
            if message_type == Recipient.PERSONAL:
                personals_pair = saved_data['personals_pair']
                random.shuffle(personals_pair)
            elif message_type == Recipient.STREAM:
                message.subject = saved_data['subject']
                message.recipient = get_recipient_by_id(recipient_id)
            elif message_type == Recipient.HUDDLE:
                message.recipient = get_recipient_by_id(recipient_id)
        elif (randkey <= random_max * options["percent_huddles"] / 100.):
            message_type = Recipient.HUDDLE
            message.recipient = get_recipient_by_id(
                random.choice(recipient_huddles))
        elif (randkey <= random_max *
              (options["percent_huddles"] + options["percent_personals"]) /
              100.):
            message_type = Recipient.PERSONAL
            personals_pair = random.choice(personals_pairs)
            random.shuffle(personals_pair)
        elif (randkey <= random_max * 1.0):
            message_type = Recipient.STREAM
            message.recipient = get_recipient_by_id(
                random.choice(recipient_streams))

        if message_type == Recipient.HUDDLE:
            sender_id = random.choice(huddle_members[message.recipient.id])
            message.sender = get_user_profile_by_id(sender_id)
        elif message_type == Recipient.PERSONAL:
            message.recipient = Recipient.objects.get(
                type=Recipient.PERSONAL, type_id=personals_pair[0])
            message.sender = get_user_profile_by_id(personals_pair[1])
            saved_data['personals_pair'] = personals_pair
        elif message_type == Recipient.STREAM:
            stream = Stream.objects.get(id=message.recipient.type_id)
            # Pick a random subscriber to the stream
            message.sender = random.choice(
                Subscription.objects.filter(
                    recipient=message.recipient)).user_profile
            message.subject = stream.name + str(random.randint(1, 3))
            saved_data['subject'] = message.subject

        message.date_sent = choose_date_sent(num_messages, tot_messages,
                                             options['threads'])
        messages.append(message)

        recipients[num_messages] = (message_type, message.recipient.id,
                                    saved_data)
        num_messages += 1

        if (num_messages % message_batch_size) == 0:
            # Send the batch and empty the list:
            send_messages(messages)
            messages = []

    if len(messages) > 0:
        # If there are unsent messages after exiting the loop, send them:
        send_messages(messages)

    return tot_messages
示例#48
0
文件: message.py 项目: krtkmj/zulip
    def build_message_dict(
            apply_markdown,
            message,
            message_id,
            last_edit_time,
            edit_history,
            content,
            subject,
            pub_date,
            rendered_content,
            rendered_content_version,
            sender_id,
            sender_email,
            sender_realm_domain,
            sender_full_name,
            sender_short_name,
            sender_avatar_source,
            sender_is_mirror_dummy,
            sending_client_name,
            recipient_id,
            recipient_type,
            recipient_type_id,
    ):
        # type: (bool, Message, int, datetime.datetime, text_type, text_type, text_type, datetime.datetime, text_type, Optional[int], int, text_type, text_type, text_type, text_type, text_type, bool, text_type, int, int, int) -> Dict[str, Any]

        avatar_url = get_avatar_url(sender_avatar_source, sender_email)

        display_recipient = get_display_recipient_by_id(
                recipient_id,
                recipient_type,
                recipient_type_id
        )

        if recipient_type == Recipient.STREAM:
            display_type = "stream"
        elif recipient_type in (Recipient.HUDDLE, Recipient.PERSONAL):
            assert not isinstance(display_recipient, text_type)
            display_type = "private"
            if len(display_recipient) == 1:
                # add the sender in if this isn't a message between
                # someone and his self, preserving ordering
                recip = {'email': sender_email,
                         'domain': sender_realm_domain,
                         'full_name': sender_full_name,
                         'short_name': sender_short_name,
                         'id': sender_id,
                         'is_mirror_dummy': sender_is_mirror_dummy}
                if recip['email'] < display_recipient[0]['email']:
                    display_recipient = [recip, display_recipient[0]]
                elif recip['email'] > display_recipient[0]['email']:
                    display_recipient = [display_recipient[0], recip]

        obj = dict(
            id                = message_id,
            sender_email      = sender_email,
            sender_full_name  = sender_full_name,
            sender_short_name = sender_short_name,
            sender_domain     = sender_realm_domain,
            sender_id         = sender_id,
            type              = display_type,
            display_recipient = display_recipient,
            recipient_id      = recipient_id,
            subject           = subject,
            timestamp         = datetime_to_timestamp(pub_date),
            gravatar_hash     = gravatar_hash(sender_email), # Deprecated June 2013
            avatar_url        = avatar_url,
            client            = sending_client_name)

        obj['subject_links'] = bugdown.subject_links(sender_realm_domain.lower(), subject)

        if last_edit_time != None:
            obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
            obj['edit_history'] = ujson.loads(edit_history)

        if apply_markdown:
            if Message.need_to_render_content(rendered_content, rendered_content_version, bugdown.version):
                if message is None:
                    # We really shouldn't be rendering objects in this method, but there is
                    # a scenario where we upgrade the version of bugdown and fail to run
                    # management commands to re-render historical messages, and then we
                    # need to have side effects.  This method is optimized to not need full
                    # blown ORM objects, but the bugdown renderer is unfortunately highly
                    # coupled to Message, and we also need to persist the new rendered content.
                    # If we don't have a message object passed in, we get one here.  The cost
                    # of going to the DB here should be overshadowed by the cost of rendering
                    # and updating the row.
                    # TODO: see #1379 to eliminate bugdown dependencies
                    message = Message.objects.select_related().get(id=message_id)

                # It's unfortunate that we need to have side effects on the message
                # in some cases.
                rendered_content = render_markdown(message, content, sender_realm_domain)
                message.rendered_content = rendered_content
                message.rendered_content_version = bugdown.version
                message.save_rendered_content()

            if rendered_content is not None:
                obj['content'] = rendered_content
            else:
                obj['content'] = u'<p>[Zulip note: Sorry, we could not understand the formatting of your message]</p>'

            obj['content_type'] = 'text/html'
        else:
            obj['content'] = content
            obj['content_type'] = 'text/x-markdown'

        return obj
示例#49
0
def fill_edit_history_entries(
        raw_edit_history: List[EditHistoryEvent],
        message: Message) -> List[FormattedEditHistoryEvent]:
    """
    This fills out the message edit history entries from the database
    to have the current topic + content as of that time, plus data on
    whatever changed. This makes it much simpler to do future
    processing.
    """
    prev_content = message.content
    prev_rendered_content = message.rendered_content
    prev_topic = message.topic_name()

    # Make sure that the latest entry in the history corresponds to the
    # message's last edit time
    if len(raw_edit_history) > 0:
        assert message.last_edit_time is not None
        assert datetime_to_timestamp(
            message.last_edit_time) == raw_edit_history[0]["timestamp"]

    formatted_edit_history: List[FormattedEditHistoryEvent] = []
    for edit_history_event in raw_edit_history:
        formatted_entry: FormattedEditHistoryEvent = {
            "content": prev_content,
            "rendered_content": prev_rendered_content,
            "timestamp": edit_history_event["timestamp"],
            "topic": prev_topic,
            "user_id": edit_history_event["user_id"],
        }

        if "prev_topic" in edit_history_event:
            prev_topic = edit_history_event["prev_topic"]
            formatted_entry["prev_topic"] = prev_topic

        # Fill current values for content/rendered_content.
        if "prev_content" in edit_history_event:
            formatted_entry["prev_content"] = edit_history_event[
                "prev_content"]
            prev_content = formatted_entry["prev_content"]
            formatted_entry["prev_rendered_content"] = edit_history_event[
                "prev_rendered_content"]
            prev_rendered_content = formatted_entry["prev_rendered_content"]
            assert prev_rendered_content is not None
            rendered_content = formatted_entry["rendered_content"]
            assert rendered_content is not None
            formatted_entry["content_html_diff"] = highlight_html_differences(
                prev_rendered_content, rendered_content, message.id)

        if "prev_stream" in edit_history_event:
            formatted_entry["prev_stream"] = edit_history_event["prev_stream"]
            formatted_entry["stream"] = edit_history_event["stream"]

        formatted_edit_history.append(formatted_entry)

    initial_message_history: FormattedEditHistoryEvent = {
        "content": prev_content,
        "rendered_content": prev_rendered_content,
        "timestamp": datetime_to_timestamp(message.date_sent),
        "topic": prev_topic,
        "user_id": message.sender_id,
    }

    formatted_edit_history.append(initial_message_history)

    return formatted_edit_history
示例#50
0
def generate_and_send_messages(
    data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
                Callable[[str], Any], int]
) -> int:
    (tot_messages, personals_pairs, options, output, random_seed) = data
    random.seed(random_seed)

    with open(
            os.path.join(get_or_create_dev_uuid_var_path("test-backend"),
                         "test_messages.json"), "rb") as infile:
        dialog = orjson.loads(infile.read())
    random.shuffle(dialog)
    texts = itertools.cycle(dialog)

    # We need to filter out streams from the analytics realm as we don't want to generate
    # messages to its streams - and they might also have no subscribers, which would break
    # our message generation mechanism below.
    stream_ids = Stream.objects.filter(realm=get_realm("zulip")).values_list(
        "id", flat=True)
    recipient_streams: List[int] = [
        recipient.id
        for recipient in Recipient.objects.filter(type=Recipient.STREAM,
                                                  type_id__in=stream_ids)
    ]
    recipient_huddles: List[int] = [
        h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
    ]

    huddle_members: Dict[int, List[int]] = {}
    for h in recipient_huddles:
        huddle_members[h] = [
            s.user_profile.id
            for s in Subscription.objects.filter(recipient_id=h)
        ]

    # Generate different topics for each stream
    possible_topics = {}
    for stream_id in recipient_streams:
        possible_topics[stream_id] = generate_topics(options["max_topics"])

    message_batch_size = options["batch_size"]
    num_messages = 0
    random_max = 1000000
    recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {}
    messages: List[Message] = []
    while num_messages < tot_messages:
        saved_data: Dict[str, Any] = {}
        message = Message()
        message.sending_client = get_client("populate_db")

        message.content = next(texts)

        randkey = random.randint(1, random_max)
        if (num_messages > 0
                and random.randint(1, random_max) * 100.0 / random_max <
                options["stickyness"]):
            # Use an old recipient
            message_type, recipient_id, saved_data = recipients[num_messages -
                                                                1]
            if message_type == Recipient.PERSONAL:
                personals_pair = saved_data["personals_pair"]
                random.shuffle(personals_pair)
            elif message_type == Recipient.STREAM:
                message.subject = saved_data["subject"]
                message.recipient = get_recipient_by_id(recipient_id)
            elif message_type == Recipient.HUDDLE:
                message.recipient = get_recipient_by_id(recipient_id)
        elif randkey <= random_max * options["percent_huddles"] / 100.0:
            message_type = Recipient.HUDDLE
            message.recipient = get_recipient_by_id(
                random.choice(recipient_huddles))
        elif (randkey <= random_max *
              (options["percent_huddles"] + options["percent_personals"]) /
              100.0):
            message_type = Recipient.PERSONAL
            personals_pair = random.choice(personals_pairs)
            random.shuffle(personals_pair)
        elif randkey <= random_max * 1.0:
            message_type = Recipient.STREAM
            message.recipient = get_recipient_by_id(
                random.choice(recipient_streams))

        if message_type == Recipient.HUDDLE:
            sender_id = random.choice(huddle_members[message.recipient.id])
            message.sender = get_user_profile_by_id(sender_id)
        elif message_type == Recipient.PERSONAL:
            message.recipient = Recipient.objects.get(
                type=Recipient.PERSONAL, type_id=personals_pair[0])
            message.sender = get_user_profile_by_id(personals_pair[1])
            saved_data["personals_pair"] = personals_pair
        elif message_type == Recipient.STREAM:
            # Pick a random subscriber to the stream
            message.sender = random.choice(
                Subscription.objects.filter(
                    recipient=message.recipient)).user_profile
            message.subject = random.choice(
                possible_topics[message.recipient.id])
            saved_data["subject"] = message.subject

        message.date_sent = choose_date_sent(num_messages, tot_messages,
                                             options["threads"])
        messages.append(message)

        recipients[num_messages] = (message_type, message.recipient.id,
                                    saved_data)
        num_messages += 1

        if (num_messages % message_batch_size) == 0:
            # Send the batch and empty the list:
            send_messages(messages)
            messages = []

    if len(messages) > 0:
        # If there are unsent messages after exiting the loop, send them:
        send_messages(messages)

    return tot_messages
示例#51
0
    def build_message_dict(
        message_id: int,
        last_edit_time: Optional[datetime.datetime],
        edit_history: Optional[str],
        content: str,
        topic_name: str,
        date_sent: datetime.datetime,
        rendered_content: Optional[str],
        rendered_content_version: Optional[int],
        sender_id: int,
        sender_realm_id: int,
        sending_client_name: str,
        rendering_realm_id: int,
        recipient_id: int,
        recipient_type: int,
        recipient_type_id: int,
        reactions: List[RawReactionRow],
        submessages: List[Dict[str, Any]],
    ) -> Dict[str, Any]:

        obj = dict(
            id=message_id,
            sender_id=sender_id,
            content=content,
            recipient_type_id=recipient_type_id,
            recipient_type=recipient_type,
            recipient_id=recipient_id,
            timestamp=datetime_to_timestamp(date_sent),
            client=sending_client_name,
        )

        obj[TOPIC_NAME] = topic_name
        obj["sender_realm_id"] = sender_realm_id

        # Render topic_links with the stream's realm instead of the
        # sender's realm; this is important for messages sent by
        # cross-realm bots like NOTIFICATION_BOT.
        obj[TOPIC_LINKS] = topic_links(rendering_realm_id, topic_name)

        if last_edit_time is not None:
            obj["last_edit_timestamp"] = datetime_to_timestamp(last_edit_time)
            assert edit_history is not None
            obj["edit_history"] = orjson.loads(edit_history)

        if Message.need_to_render_content(rendered_content,
                                          rendered_content_version,
                                          markdown_version):
            # We really shouldn't be rendering objects in this method, but there is
            # a scenario where we upgrade the version of Markdown and fail to run
            # management commands to re-render historical messages, and then we
            # need to have side effects.  This method is optimized to not need full
            # blown ORM objects, but the Markdown renderer is unfortunately highly
            # coupled to Message, and we also need to persist the new rendered content.
            # If we don't have a message object passed in, we get one here.  The cost
            # of going to the DB here should be overshadowed by the cost of rendering
            # and updating the row.
            # TODO: see #1379 to eliminate Markdown dependencies
            message = Message.objects.select_related().get(id=message_id)

            assert message is not None  # Hint for mypy.
            # It's unfortunate that we need to have side effects on the message
            # in some cases.
            rendered_content = save_message_rendered_content(message, content)

        if rendered_content is not None:
            obj["rendered_content"] = rendered_content
        else:
            obj["rendered_content"] = (
                "<p>[Zulip note: Sorry, we could not " +
                "understand the formatting of your message]</p>")

        if rendered_content is not None:
            obj["is_me_message"] = Message.is_status_message(
                content, rendered_content)
        else:
            obj["is_me_message"] = False

        obj["reactions"] = [
            ReactionDict.build_dict_from_raw_db_row(reaction)
            for reaction in reactions
        ]
        obj["submessages"] = submessages
        return obj
示例#52
0
 def do_test_topic(self, msg: Message, expected_topic: Optional[str]) -> None:
     if expected_topic is not None:
         self.assertEqual(msg.topic_name(), expected_topic)
示例#53
0
def get_old_messages_backend(request, user_profile,
                             anchor = REQ(converter=int),
                             num_before = REQ(converter=to_non_negative_int),
                             num_after = REQ(converter=to_non_negative_int),
                             narrow = REQ('narrow', converter=narrow_parameter, default=None),
                             use_first_unread_anchor = REQ(default=False, converter=ujson.loads),
                             apply_markdown=REQ(default=True,
                                                converter=ujson.loads)):
    include_history = ok_to_include_history(narrow, user_profile.realm)

    if include_history and not use_first_unread_anchor:
        query = select([column("id").label("message_id")], None, "zerver_message")
        inner_msg_id_col = literal_column("zerver_message.id")
    elif narrow is None:
        query = select([column("message_id"), column("flags")],
                       column("user_profile_id") == literal(user_profile.id),
                       "zerver_usermessage")
        inner_msg_id_col = column("message_id")
    else:
        # TODO: Don't do this join if we're not doing a search
        query = select([column("message_id"), column("flags")],
                       column("user_profile_id") == literal(user_profile.id),
                       join("zerver_usermessage", "zerver_message",
                            literal_column("zerver_usermessage.message_id") ==
                            literal_column("zerver_message.id")))
        inner_msg_id_col = column("message_id")

    num_extra_messages = 1
    is_search = False

    if narrow is not None:
        # Add some metadata to our logging data for narrows
        verbose_operators = []
        for term in narrow:
            if term['operator'] == "is":
                verbose_operators.append("is:" + term['operand'])
            else:
                verbose_operators.append(term['operator'])
        request._log_data['extra'] = "[%s]" % (",".join(verbose_operators),)

        # Build the query for the narrow
        num_extra_messages = 0
        builder = NarrowBuilder(user_profile, inner_msg_id_col)
        for term in narrow:
            if term['operator'] == 'search' and not is_search:
                query = query.column("subject").column("rendered_content")
                is_search = True
            query = builder.add_term(query, term)

    # We add 1 to the number of messages requested if no narrow was
    # specified to ensure that the resulting list always contains the
    # anchor message.  If a narrow was specified, the anchor message
    # might not match the narrow anyway.
    if num_after != 0:
        num_after += num_extra_messages
    else:
        num_before += num_extra_messages

    sa_conn = get_sqlalchemy_connection()
    if use_first_unread_anchor:
        condition = column("flags").op("&")(UserMessage.flags.read.mask) == 0

        # We exclude messages on muted topics when finding the first unread
        # message in this narrow
        muting_conditions = exclude_muting_conditions(user_profile, narrow)
        if muting_conditions:
            condition = and_(condition, *muting_conditions)

        first_unread_query = query.where(condition)
        first_unread_query = first_unread_query.order_by(inner_msg_id_col.asc()).limit(1)
        first_unread_result = list(sa_conn.execute(first_unread_query).fetchall())
        if len(first_unread_result) > 0:
            anchor = first_unread_result[0][0]
        else:
            anchor = 10000000000000000

    before_query = None
    after_query = None
    if num_before != 0:
        before_anchor = anchor
        if num_after != 0:
            # Don't include the anchor in both the before query and the after query
            before_anchor = anchor - 1
        before_query = query.where(inner_msg_id_col <= before_anchor) \
                            .order_by(inner_msg_id_col.desc()).limit(num_before)
    if num_after != 0:
        after_query = query.where(inner_msg_id_col >= anchor) \
                           .order_by(inner_msg_id_col.asc()).limit(num_after)

    if num_before == 0 and num_after == 0:
        # This can happen when a narrow is specified.
        after_query = query.where(inner_msg_id_col == anchor)

    if before_query is not None:
        if after_query is not None:
            query = union_all(before_query.self_group(), after_query.self_group())
        else:
            query = before_query
    else:
        query = after_query
    main_query = alias(query)
    query = select(main_query.c, None, main_query).order_by(column("message_id").asc())
    # This is a hack to tag the query we use for testing
    query = query.prefix_with("/* get_old_messages */")
    query_result = list(sa_conn.execute(query).fetchall())

    # The following is a little messy, but ensures that the code paths
    # are similar regardless of the value of include_history.  The
    # 'user_messages' dictionary maps each message to the user's
    # UserMessage object for that message, which we will attach to the
    # rendered message dict before returning it.  We attempt to
    # bulk-fetch rendered message dicts from remote cache using the
    # 'messages' list.
    search_fields = dict() # type: Dict[int, Dict[str, str]]
    message_ids = [] # type: List[int]
    user_message_flags = {} # type: Dict[int, List[str]]
    if include_history:
        message_ids = [row[0] for row in query_result]

        # TODO: This could be done with an outer join instead of two queries
        user_message_flags = dict((user_message.message_id, user_message.flags_list()) for user_message in
                                  UserMessage.objects.filter(user_profile=user_profile,
                                                             message__id__in=message_ids))
        for row in query_result:
            message_id = row[0]
            if user_message_flags.get(message_id) is None:
                user_message_flags[message_id] = ["read", "historical"]
            if is_search:
                (_, subject, rendered_content, content_matches, subject_matches) = row
                search_fields[message_id] = get_search_fields(rendered_content, subject,
                                                              content_matches, subject_matches)
    else:
        for row in query_result:
            message_id = row[0]
            flags = row[1]
            user_message_flags[message_id] = parse_usermessage_flags(flags)

            message_ids.append(message_id)

            if is_search:
                (_, _, subject, rendered_content, content_matches, subject_matches) = row
                search_fields[message_id] = get_search_fields(rendered_content, subject,
                                                              content_matches, subject_matches)

    cache_transformer = lambda row: Message.build_dict_from_raw_db_row(row, apply_markdown)
    id_fetcher = lambda row: row['id']

    message_dicts = generic_bulk_cached_fetch(lambda message_id: to_dict_cache_key_id(message_id, apply_markdown),
                                              Message.get_raw_db_rows,
                                              message_ids,
                                              id_fetcher=id_fetcher,
                                              cache_transformer=cache_transformer,
                                              extractor=extract_message_dict,
                                              setter=stringify_message_dict)

    message_list = []
    for message_id in message_ids:
        msg_dict = message_dicts[message_id]
        msg_dict.update({"flags": user_message_flags[message_id]})
        msg_dict.update(search_fields.get(message_id, {}))
        message_list.append(msg_dict)

    statsd.incr('loaded_old_messages', len(message_list))
    ret = {'messages': message_list,
           "result": "success",
           "msg": ""}
    return json_success(ret)
示例#54
0
def save_message_for_edit_use_case(message: Message) -> None:
    message.save(update_fields=[
        "subject", "content", "rendered_content", "rendered_content_version",
        "last_edit_time", "edit_history"
    ])
示例#55
0
def send_messages(data):
    # type: (Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int]) -> int
    (tot_messages, personals_pairs, options, output, random_seed) = data
    random.seed(random_seed)

    with open("var/test_messages.json", "r") as infile:
        dialog = ujson.load(infile)
    random.shuffle(dialog)
    texts = itertools.cycle(dialog)

    recipient_streams = [
        klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
    ]  # type: List[int]
    recipient_huddles = [
        h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
    ]  # type: List[int]

    huddle_members = {}  # type: Dict[int, List[int]]
    for h in recipient_huddles:
        huddle_members[h] = [
            s.user_profile.id
            for s in Subscription.objects.filter(recipient_id=h)
        ]

    num_messages = 0
    random_max = 1000000
    recipients = {}  # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
    while num_messages < tot_messages:
        saved_data = {}  # type: Dict[str, Any]
        message = Message()
        message.sending_client = get_client('populate_db')

        message.content = next(texts)

        randkey = random.randint(1, random_max)
        if (num_messages > 0
                and random.randint(1, random_max) * 100. / random_max <
                options["stickyness"]):
            # Use an old recipient
            message_type, recipient_id, saved_data = recipients[num_messages -
                                                                1]
            if message_type == Recipient.PERSONAL:
                personals_pair = saved_data['personals_pair']
                random.shuffle(personals_pair)
            elif message_type == Recipient.STREAM:
                message.subject = saved_data['subject']
                message.recipient = get_recipient_by_id(recipient_id)
            elif message_type == Recipient.HUDDLE:
                message.recipient = get_recipient_by_id(recipient_id)
        elif (randkey <= random_max * options["percent_huddles"] / 100.):
            message_type = Recipient.HUDDLE
            message.recipient = get_recipient_by_id(
                random.choice(recipient_huddles))
        elif (randkey <= random_max *
              (options["percent_huddles"] + options["percent_personals"]) /
              100.):
            message_type = Recipient.PERSONAL
            personals_pair = random.choice(personals_pairs)
            random.shuffle(personals_pair)
        elif (randkey <= random_max * 1.0):
            message_type = Recipient.STREAM
            message.recipient = get_recipient_by_id(
                random.choice(recipient_streams))

        if message_type == Recipient.HUDDLE:
            sender_id = random.choice(huddle_members[message.recipient.id])
            message.sender = get_user_profile_by_id(sender_id)
        elif message_type == Recipient.PERSONAL:
            message.recipient = Recipient.objects.get(
                type=Recipient.PERSONAL, type_id=personals_pair[0])
            message.sender = get_user_profile_by_id(personals_pair[1])
            saved_data['personals_pair'] = personals_pair
        elif message_type == Recipient.STREAM:
            stream = Stream.objects.get(id=message.recipient.type_id)
            # Pick a random subscriber to the stream
            message.sender = random.choice(
                Subscription.objects.filter(
                    recipient=message.recipient)).user_profile
            message.subject = stream.name + Text(random.randint(1, 3))
            saved_data['subject'] = message.subject

        message.pub_date = timezone_now()
        do_send_messages([{'message': message}])

        recipients[num_messages] = (message_type, message.recipient.id,
                                    saved_data)
        num_messages += 1
    return tot_messages
示例#56
0
def send_messages(
    data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
                Callable[[str], Any], int]
) -> int:
    (tot_messages, personals_pairs, options, output, random_seed) = data
    random.seed(random_seed)

    with open("var/test_messages.json", "r") as infile:
        dialog = ujson.load(infile)
    random.shuffle(dialog)
    texts = itertools.cycle(dialog)

    recipient_streams = [
        klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
    ]  # type: List[int]
    recipient_huddles = [
        h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
    ]  # type: List[int]

    huddle_members = {}  # type: Dict[int, List[int]]
    for h in recipient_huddles:
        huddle_members[h] = [
            s.user_profile.id
            for s in Subscription.objects.filter(recipient_id=h)
        ]

    num_messages = 0
    random_max = 1000000
    recipients = {}  # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
    while num_messages < tot_messages:
        saved_data = {}  # type: Dict[str, Any]
        message = Message()
        message.sending_client = get_client('populate_db')

        message.content = next(texts)

        randkey = random.randint(1, random_max)
        if (num_messages > 0
                and random.randint(1, random_max) * 100. / random_max <
                options["stickyness"]):
            # Use an old recipient
            message_type, recipient_id, saved_data = recipients[num_messages -
                                                                1]
            if message_type == Recipient.PERSONAL:
                personals_pair = saved_data['personals_pair']
                random.shuffle(personals_pair)
            elif message_type == Recipient.STREAM:
                message.subject = saved_data['subject']
                message.recipient = get_recipient_by_id(recipient_id)
            elif message_type == Recipient.HUDDLE:
                message.recipient = get_recipient_by_id(recipient_id)
        elif (randkey <= random_max * options["percent_huddles"] / 100.):
            message_type = Recipient.HUDDLE
            message.recipient = get_recipient_by_id(
                random.choice(recipient_huddles))
        elif (randkey <= random_max *
              (options["percent_huddles"] + options["percent_personals"]) /
              100.):
            message_type = Recipient.PERSONAL
            personals_pair = random.choice(personals_pairs)
            random.shuffle(personals_pair)
        elif (randkey <= random_max * 1.0):
            message_type = Recipient.STREAM
            message.recipient = get_recipient_by_id(
                random.choice(recipient_streams))

        if message_type == Recipient.HUDDLE:
            sender_id = random.choice(huddle_members[message.recipient.id])
            message.sender = get_user_profile_by_id(sender_id)
        elif message_type == Recipient.PERSONAL:
            message.recipient = Recipient.objects.get(
                type=Recipient.PERSONAL, type_id=personals_pair[0])
            message.sender = get_user_profile_by_id(personals_pair[1])
            saved_data['personals_pair'] = personals_pair
        elif message_type == Recipient.STREAM:
            stream = Stream.objects.get(id=message.recipient.type_id)
            # Pick a random subscriber to the stream
            message.sender = random.choice(
                Subscription.objects.filter(
                    recipient=message.recipient)).user_profile
            message.subject = stream.name + str(random.randint(1, 3))
            saved_data['subject'] = message.subject

        # Spoofing time not supported with threading
        if options['threads'] != 1:
            message.pub_date = timezone_now()
        else:
            # Distrubutes 80% of messages starting from 5 days ago, over a period
            # of 3 days. Then, distributes remaining messages over past 24 hours.
            spoofed_date = timezone_now() - timezone_timedelta(days=5)
            if (num_messages < tot_messages * 0.8):
                # Maximum of 3 days ahead, convert to minutes
                time_ahead = 3 * 24 * 60
                time_ahead //= int(tot_messages * 0.8)
            else:
                time_ahead = 24 * 60
                time_ahead //= int(tot_messages * 0.2)

            spoofed_minute = random.randint(time_ahead * num_messages,
                                            time_ahead * (num_messages + 1))
            spoofed_date += timezone_timedelta(minutes=spoofed_minute)
            message.pub_date = spoofed_date

        # We disable USING_RABBITMQ here, so that deferred work is
        # executed in do_send_message_messages, rather than being
        # queued.  This is important, because otherwise, if run-dev.py
        # wasn't running when populate_db was run, a developer can end
        # up with queued events that reference objects from a previous
        # life of the database, which naturally throws exceptions.
        settings.USING_RABBITMQ = False
        do_send_messages([{'message': message}])
        settings.USING_RABBITMQ = True

        recipients[num_messages] = (message_type, message.recipient.id,
                                    saved_data)
        num_messages += 1
    return tot_messages
示例#57
0
def do_update_message(
    user_profile: UserProfile,
    target_message: Message,
    new_stream: Optional[Stream],
    topic_name: Optional[str],
    propagate_mode: Optional[str],
    send_notification_to_old_thread: bool,
    send_notification_to_new_thread: bool,
    content: Optional[str],
    rendering_result: Optional[MessageRenderingResult],
    prior_mention_user_ids: Set[int],
    mention_data: Optional[MentionData] = None,
) -> int:
    """
    The main function for message editing.  A message edit event can
    modify:
    * the message's content (in which case the caller will have
      set both content and rendered_content),
    * the topic, in which case the caller will have set topic_name
    * or both message's content and the topic
    * or stream and/or topic, in which case the caller will have set
        new_stream and/or topic_name.

    With topic edits, propagate_mode determines whether other message
    also have their topics edited.
    """
    timestamp = timezone_now()
    target_message.last_edit_time = timestamp

    event: Dict[str, Any] = {
        "type": "update_message",
        "user_id": user_profile.id,
        "edit_timestamp": datetime_to_timestamp(timestamp),
        "message_id": target_message.id,
        "rendering_only": False,
    }

    edit_history_event: EditHistoryEvent = {
        "user_id": user_profile.id,
        "timestamp": event["edit_timestamp"],
    }

    changed_messages = [target_message]

    realm = user_profile.realm

    stream_being_edited = None
    if target_message.is_stream_message():
        stream_id = target_message.recipient.type_id
        stream_being_edited = get_stream_by_id_in_realm(stream_id, realm)
        event["stream_name"] = stream_being_edited.name
        event["stream_id"] = stream_being_edited.id

    ums = UserMessage.objects.filter(message=target_message.id)

    if content is not None:
        assert rendering_result is not None

        # mention_data is required if there's a content edit.
        assert mention_data is not None

        # add data from group mentions to mentions_user_ids.
        for group_id in rendering_result.mentions_user_group_ids:
            members = mention_data.get_group_members(group_id)
            rendering_result.mentions_user_ids.update(members)

        update_user_message_flags(rendering_result, ums)

        # One could imagine checking realm.allow_edit_history here and
        # modifying the events based on that setting, but doing so
        # doesn't really make sense.  We need to send the edit event
        # to clients regardless, and a client already had access to
        # the original/pre-edit content of the message anyway.  That
        # setting must be enforced on the client side, and making a
        # change here simply complicates the logic for clients parsing
        # edit history events.
        event["orig_content"] = target_message.content
        event["orig_rendered_content"] = target_message.rendered_content
        edit_history_event["prev_content"] = target_message.content
        edit_history_event[
            "prev_rendered_content"] = target_message.rendered_content
        edit_history_event[
            "prev_rendered_content_version"] = target_message.rendered_content_version
        target_message.content = content
        target_message.rendered_content = rendering_result.rendered_content
        target_message.rendered_content_version = markdown_version
        event["content"] = content
        event["rendered_content"] = rendering_result.rendered_content
        event[
            "prev_rendered_content_version"] = target_message.rendered_content_version
        event["is_me_message"] = Message.is_status_message(
            content, rendering_result.rendered_content)

        # target_message.has_image and target_message.has_link will have been
        # already updated by Markdown rendering in the caller.
        target_message.has_attachment = check_attachment_reference_change(
            target_message, rendering_result)

        if target_message.is_stream_message():
            if topic_name is not None:
                new_topic_name = topic_name
            else:
                new_topic_name = target_message.topic_name()

            stream_topic: Optional[StreamTopicTarget] = StreamTopicTarget(
                stream_id=stream_id,
                topic_name=new_topic_name,
            )
        else:
            stream_topic = None

        info = get_recipient_info(
            realm_id=realm.id,
            recipient=target_message.recipient,
            sender_id=target_message.sender_id,
            stream_topic=stream_topic,
            possible_wildcard_mention=mention_data.message_has_wildcards(),
        )

        event["online_push_user_ids"] = list(info["online_push_user_ids"])
        event["pm_mention_push_disabled_user_ids"] = list(
            info["pm_mention_push_disabled_user_ids"])
        event["pm_mention_email_disabled_user_ids"] = list(
            info["pm_mention_email_disabled_user_ids"])
        event["stream_push_user_ids"] = list(info["stream_push_user_ids"])
        event["stream_email_user_ids"] = list(info["stream_email_user_ids"])
        event["muted_sender_user_ids"] = list(info["muted_sender_user_ids"])
        event["prior_mention_user_ids"] = list(prior_mention_user_ids)
        event["presence_idle_user_ids"] = filter_presence_idle_user_ids(
            info["active_user_ids"])
        event["all_bot_user_ids"] = list(info["all_bot_user_ids"])
        if rendering_result.mentions_wildcard:
            event["wildcard_mention_user_ids"] = list(
                info["wildcard_mention_user_ids"])
        else:
            event["wildcard_mention_user_ids"] = []

        do_update_mobile_push_notification(
            target_message,
            prior_mention_user_ids,
            rendering_result.mentions_user_ids,
            info["stream_push_user_ids"],
        )

    if topic_name is not None or new_stream is not None:
        assert propagate_mode is not None
        orig_topic_name = target_message.topic_name()
        event["propagate_mode"] = propagate_mode

    if new_stream is not None:
        assert content is None
        assert target_message.is_stream_message()
        assert stream_being_edited is not None

        edit_history_event["prev_stream"] = stream_being_edited.id
        edit_history_event["stream"] = new_stream.id
        event[ORIG_TOPIC] = orig_topic_name
        assert new_stream.recipient_id is not None
        target_message.recipient_id = new_stream.recipient_id

        event["new_stream_id"] = new_stream.id
        event["propagate_mode"] = propagate_mode

        # When messages are moved from one stream to another, some
        # users may lose access to those messages, including guest
        # users and users not subscribed to the new stream (if it is a
        # private stream).  For those users, their experience is as
        # though the messages were deleted, and we should send a
        # delete_message event to them instead.

        subs_to_old_stream = get_active_subscriptions_for_stream_id(
            stream_id,
            include_deactivated_users=True).select_related("user_profile")
        subs_to_new_stream = list(
            get_active_subscriptions_for_stream_id(
                new_stream.id,
                include_deactivated_users=True).select_related("user_profile"))

        old_stream_sub_ids = [
            user.user_profile_id for user in subs_to_old_stream
        ]
        new_stream_sub_ids = [
            user.user_profile_id for user in subs_to_new_stream
        ]

        # Get users who aren't subscribed to the new_stream.
        subs_losing_usermessages = [
            sub for sub in subs_to_old_stream
            if sub.user_profile_id not in new_stream_sub_ids
        ]
        # Users who can longer access the message without some action
        # from administrators.
        subs_losing_access = [
            sub for sub in subs_losing_usermessages
            if sub.user_profile.is_guest or not new_stream.is_public()
        ]
        ums = ums.exclude(user_profile_id__in=[
            sub.user_profile_id for sub in subs_losing_usermessages
        ])

        subs_gaining_usermessages = []
        if not new_stream.is_history_public_to_subscribers():
            # For private streams, with history not public to subscribers,
            # We find out users who are not present in the msgs' old stream
            # and create new UserMessage for these users so that they can
            # access this message.
            subs_gaining_usermessages += [
                user_id for user_id in new_stream_sub_ids
                if user_id not in old_stream_sub_ids
            ]

    if topic_name is not None:
        topic_name = truncate_topic(topic_name)
        target_message.set_topic_name(topic_name)

        # These fields have legacy field names.
        event[ORIG_TOPIC] = orig_topic_name
        event[TOPIC_NAME] = topic_name
        event[TOPIC_LINKS] = topic_links(target_message.sender.realm_id,
                                         topic_name)
        edit_history_event["prev_topic"] = orig_topic_name
        edit_history_event["topic"] = topic_name

    update_edit_history(target_message, timestamp, edit_history_event)

    delete_event_notify_user_ids: List[int] = []
    if propagate_mode in ["change_later", "change_all"]:
        assert topic_name is not None or new_stream is not None
        assert stream_being_edited is not None

        # Other messages should only get topic/stream fields in their edit history.
        topic_only_edit_history_event: EditHistoryEvent = {
            "user_id": edit_history_event["user_id"],
            "timestamp": edit_history_event["timestamp"],
        }
        if topic_name is not None:
            topic_only_edit_history_event["prev_topic"] = edit_history_event[
                "prev_topic"]
            topic_only_edit_history_event["topic"] = edit_history_event[
                "topic"]
        if new_stream is not None:
            topic_only_edit_history_event["prev_stream"] = edit_history_event[
                "prev_stream"]
            topic_only_edit_history_event["stream"] = edit_history_event[
                "stream"]

        messages_list = update_messages_for_topic_edit(
            acting_user=user_profile,
            edited_message=target_message,
            propagate_mode=propagate_mode,
            orig_topic_name=orig_topic_name,
            topic_name=topic_name,
            new_stream=new_stream,
            old_stream=stream_being_edited,
            edit_history_event=topic_only_edit_history_event,
            last_edit_time=timestamp,
        )
        changed_messages += messages_list

        if new_stream is not None:
            assert stream_being_edited is not None
            changed_message_ids = [msg.id for msg in changed_messages]

            if subs_gaining_usermessages:
                ums_to_create = []
                for message_id in changed_message_ids:
                    for user_profile_id in subs_gaining_usermessages:
                        # The fact that the user didn't have a UserMessage originally means we can infer that the user
                        # was not mentioned in the original message (even if mention syntax was present, it would not
                        # take effect for a user who was not subscribed). If we were editing the message's content, we
                        # would rerender the message and then use the new stream's data to determine whether this is
                        # a mention of a subscriber; but as we are not doing so, we choose to preserve the "was this
                        # mention syntax an actual mention" decision made during the original rendering for implementation
                        # simplicity. As a result, the only flag to consider applying here is read.
                        um = UserMessageLite(
                            user_profile_id=user_profile_id,
                            message_id=message_id,
                            flags=UserMessage.flags.read,
                        )
                        ums_to_create.append(um)
                bulk_insert_ums(ums_to_create)

            # Delete UserMessage objects for users who will no
            # longer have access to these messages.  Note: This could be
            # very expensive, since it's N guest users x M messages.
            UserMessage.objects.filter(
                user_profile_id__in=[
                    sub.user_profile_id for sub in subs_losing_usermessages
                ],
                message_id__in=changed_message_ids,
            ).delete()

            delete_event: DeleteMessagesEvent = {
                "type": "delete_message",
                "message_ids": changed_message_ids,
                "message_type": "stream",
                "stream_id": stream_being_edited.id,
                "topic": orig_topic_name,
            }
            delete_event_notify_user_ids = [
                sub.user_profile_id for sub in subs_losing_access
            ]
            send_event(user_profile.realm, delete_event,
                       delete_event_notify_user_ids)

            # Reset the Attachment.is_*_public caches for all messages
            # moved to another stream with different access permissions.
            if new_stream.invite_only != stream_being_edited.invite_only:
                Attachment.objects.filter(
                    messages__in=changed_message_ids).update(
                        is_realm_public=None, )
                ArchivedAttachment.objects.filter(
                    messages__in=changed_message_ids).update(
                        is_realm_public=None, )

            if new_stream.is_web_public != stream_being_edited.is_web_public:
                Attachment.objects.filter(
                    messages__in=changed_message_ids).update(
                        is_web_public=None, )
                ArchivedAttachment.objects.filter(
                    messages__in=changed_message_ids).update(
                        is_web_public=None, )

    # This does message.save(update_fields=[...])
    save_message_for_edit_use_case(message=target_message)

    realm_id: Optional[int] = None
    if stream_being_edited is not None:
        realm_id = stream_being_edited.realm_id

    event["message_ids"] = update_to_dict_cache(changed_messages, realm_id)

    def user_info(um: UserMessage) -> Dict[str, Any]:
        return {
            "id": um.user_profile_id,
            "flags": um.flags_list(),
        }

    # The following blocks arranges that users who are subscribed to a
    # stream and can see history from before they subscribed get
    # live-update when old messages are edited (e.g. if the user does
    # a topic edit themself).
    #
    # We still don't send an update event to users who are not
    # subscribed to this stream and don't have a UserMessage row. This
    # means if a non-subscriber is viewing the narrow, they won't get
    # a real-time updates. This is a balance between sending
    # message-edit notifications for every public stream to every user
    # in the organization (too expansive, and also not what we do for
    # newly sent messages anyway) and having magical live-updates
    # where possible.
    users_to_be_notified = list(map(user_info, ums))
    if stream_being_edited is not None:
        if stream_being_edited.is_history_public_to_subscribers():
            subscriptions = get_active_subscriptions_for_stream_id(
                stream_id, include_deactivated_users=False)
            # We exclude long-term idle users, since they by
            # definition have no active clients.
            subscriptions = subscriptions.exclude(
                user_profile__long_term_idle=True)
            # Remove duplicates by excluding the id of users already
            # in users_to_be_notified list.  This is the case where a
            # user both has a UserMessage row and is a current
            # Subscriber
            subscriptions = subscriptions.exclude(
                user_profile_id__in=[um.user_profile_id for um in ums])

            if new_stream is not None:
                assert delete_event_notify_user_ids is not None
                subscriptions = subscriptions.exclude(
                    user_profile_id__in=delete_event_notify_user_ids)

            # All users that are subscribed to the stream must be
            # notified when a message is edited
            subscriber_ids = set(
                subscriptions.values_list("user_profile_id", flat=True))

            if new_stream is not None:
                # TODO: Guest users don't see the new moved topic
                # unless breadcrumb message for new stream is
                # enabled. Excluding these users from receiving this
                # event helps us avoid a error traceback for our
                # clients. We should figure out a way to inform the
                # guest users of this new topic if sending a 'message'
                # event for these messages is not an option.
                #
                # Don't send this event to guest subs who are not
                # subscribers of the old stream but are subscribed to
                # the new stream; clients will be confused.
                old_stream_unsubbed_guests = [
                    sub for sub in subs_to_new_stream
                    if sub.user_profile.is_guest
                    and sub.user_profile_id not in subscriber_ids
                ]
                subscriptions = subscriptions.exclude(user_profile_id__in=[
                    sub.user_profile_id for sub in old_stream_unsubbed_guests
                ])
                subscriber_ids = set(
                    subscriptions.values_list("user_profile_id", flat=True))

            users_to_be_notified += list(
                map(subscriber_info, sorted(list(subscriber_ids))))

    # UserTopic updates and the content of notifications depend on
    # whether we've moved the entire topic, or just part of it. We
    # make that determination here.
    moved_all_visible_messages = False
    if topic_name is not None or new_stream is not None:
        assert stream_being_edited is not None

        if propagate_mode == "change_all":
            moved_all_visible_messages = True
        else:
            # With other propagate modes, if the user in fact moved
            # all messages in the stream, we want to explain it was a
            # full-topic move.
            #
            # For security model reasons, we don't want to allow a
            # user to take any action that would leak information
            # about older messages they cannot access (E.g. the only
            # remaining messages are in a stream without shared
            # history). The bulk_access_messages call below addresses
            # that concern.
            #
            # bulk_access_messages is inefficient for this task, since
            # we just want to do the exists() version of this
            # query. But it's nice to reuse code, and this bulk
            # operation is likely cheaper than a `GET /messages`
            # unless the topic has thousands of messages of history.
            assert stream_being_edited.recipient_id is not None
            unmoved_messages = messages_for_topic(
                stream_being_edited.recipient_id,
                orig_topic_name,
            )
            visible_unmoved_messages = bulk_access_messages(
                user_profile, unmoved_messages, stream=stream_being_edited)
            moved_all_visible_messages = len(visible_unmoved_messages) == 0

    # Migrate muted topic configuration in the following circumstances:
    #
    # * If propagate_mode is change_all, do so unconditionally.
    #
    # * If propagate_mode is change_later or change_one, do so when
    #   the acting user has moved the entire topic (as visible to them).
    #
    # This rule corresponds to checking moved_all_visible_messages.
    #
    # We may want more complex behavior in cases where one appears to
    # be merging topics (E.g. there are existing messages in the
    # target topic).
    if moved_all_visible_messages:
        assert stream_being_edited is not None
        assert topic_name is not None or new_stream is not None

        for muting_user in get_users_muting_topic(stream_being_edited.id,
                                                  orig_topic_name):
            # TODO: Ideally, this would be a bulk update operation,
            # because we are doing database operations in a loop here.
            #
            # This loop is only acceptable in production because it is
            # rare for more than a few users to have muted an
            # individual topic that is being moved; as of this
            # writing, no individual topic in Zulip Cloud had been
            # muted by more than 100 users.

            if new_stream is not None and muting_user.id in delete_event_notify_user_ids:
                # If the messages are being moved to a stream the user
                # cannot access, then we treat this as the
                # messages/topic being deleted for this user. This is
                # important for security reasons; we don't want to
                # give users a UserTopic row in a stream they cannot
                # access.  Unmute the topic for such users.
                do_unmute_topic(muting_user, stream_being_edited,
                                orig_topic_name)
            else:
                # Otherwise, we move the muted topic record for the user.
                # We call remove_topic_mute rather than do_unmute_topic to
                # avoid sending two events with new muted topics in
                # immediate succession; this is correct only because
                # muted_topics events always send the full set of topics.
                remove_topic_mute(muting_user, stream_being_edited.id,
                                  orig_topic_name)
                do_mute_topic(
                    muting_user,
                    new_stream
                    if new_stream is not None else stream_being_edited,
                    topic_name if topic_name is not None else orig_topic_name,
                    ignore_duplicate=True,
                )

    send_event(user_profile.realm, event, users_to_be_notified)

    if len(
            changed_messages
    ) > 0 and new_stream is not None and stream_being_edited is not None:
        # Notify users that the topic was moved.
        changed_messages_count = len(changed_messages)

        old_thread_notification_string = None
        if send_notification_to_old_thread:
            if moved_all_visible_messages:
                old_thread_notification_string = gettext_lazy(
                    "This topic was moved to {new_location} by {user}.")
            elif changed_messages_count == 1:
                old_thread_notification_string = gettext_lazy(
                    "A message was moved from this topic to {new_location} by {user}."
                )
            else:
                old_thread_notification_string = gettext_lazy(
                    "{changed_messages_count} messages were moved from this topic to {new_location} by {user}."
                )

        new_thread_notification_string = None
        if send_notification_to_new_thread:
            if moved_all_visible_messages:
                new_thread_notification_string = gettext_lazy(
                    "This topic was moved here from {old_location} by {user}.")
            elif changed_messages_count == 1:
                new_thread_notification_string = gettext_lazy(
                    "A message was moved here from {old_location} by {user}.")
            else:
                new_thread_notification_string = gettext_lazy(
                    "{changed_messages_count} messages were moved here from {old_location} by {user}."
                )

        send_message_moved_breadcrumbs(
            user_profile,
            stream_being_edited,
            orig_topic_name,
            old_thread_notification_string,
            new_stream,
            topic_name,
            new_thread_notification_string,
            changed_messages_count,
        )

    if (topic_name is not None and new_stream is None and content is None
            and len(changed_messages) > 0):
        assert stream_being_edited is not None
        maybe_send_resolve_topic_notifications(
            user_profile=user_profile,
            stream=stream_being_edited,
            old_topic=orig_topic_name,
            new_topic=topic_name,
            changed_messages=changed_messages,
        )

    return len(changed_messages)
示例#58
0
 def do_test_topic(self, msg: Message,
                   expected_topic: Optional[str]) -> None:
     if expected_topic is not None:
         self.assertEqual(msg.topic_name(), expected_topic)
示例#59
0
 def handle(self, *args, **kwargs):
     get_UserMessages_to_expunge().delete()
     Message.remove_unreachable()