Exemplo n.º 1
0
 def test_empty_reference(self):
     msg = Message()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["References"] = " "
     msg.set_payload("Dummy message")
     try:
         utils.get_ref(msg)
     except IndexError:
         self.fail("Empty 'References' tag should be handled")
Exemplo n.º 2
0
 def test_empty_reference(self):
     msg = Message()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["References"] = " "
     msg.set_payload("Dummy message")
     try:
         utils.get_ref(msg)
     except IndexError:
         self.fail("Empty 'References' tag should be handled")
Exemplo n.º 3
0
 def test_ref_parsing(self):
     with open(
             get_test_file("strange-in-reply-to-header.txt")
             ) as email_file:
         msg = message_from_file(email_file)
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, "*****@*****.**")
Exemplo n.º 4
0
 def test_in_reply_to(self):
     msg = Message()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["In-Reply-To"] = " <ref-1> "
     msg.set_payload("Dummy message")
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, "ref-1")
Exemplo n.º 5
0
 def test_multiple_reference(self):
     msg = Message()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["References"] = " <ref-1> <ref-2> "
     msg.set_payload("Dummy message")
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, "ref-2")
Exemplo n.º 6
0
 def test_reference_no_brackets(self):
     msg = Message()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["References"] = "ref-1"
     msg.set_payload("Dummy message")
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, "ref-1")
Exemplo n.º 7
0
 def test_in_reply_to(self):
     msg = Message()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["In-Reply-To"] = " <ref-1> "
     msg.set_payload("Dummy message")
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, "ref-1")
Exemplo n.º 8
0
 def test_reference_no_brackets(self):
     msg = Message()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["References"] = "ref-1"
     msg.set_payload("Dummy message")
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, "ref-1")
Exemplo n.º 9
0
 def test_multiple_reference(self):
     msg = Message()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["References"] = " <ref-1> <ref-2> "
     msg.set_payload("Dummy message")
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, "ref-2")
Exemplo n.º 10
0
 def test_single_reference(self):
     msg = EmailMessage()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["References"] = " <ref-1> "
     msg.set_content("Dummy message")
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, "ref-1")
Exemplo n.º 11
0
 def test_in_reply_to_and_reference(self):
     """The In-Reply-To header should win over References"""
     msg = Message()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["In-Reply-To"] = " <ref-1> "
     msg["References"] = " <ref-2> "
     msg.set_payload("Dummy message")
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, "ref-1")
Exemplo n.º 12
0
 def test_in_reply_to_and_reference(self):
     """The In-Reply-To header should win over References"""
     msg = Message()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["In-Reply-To"] = " <ref-1> "
     msg["References"] = " <ref-2> "
     msg.set_payload("Dummy message")
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, "ref-1")
Exemplo n.º 13
0
 def test_non_ascii_ref(self):
     msg = Message()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["In-Reply-To"] = u"<ref-\xed>".encode('utf-8')
     msg.set_payload("Dummy message")
     try:
         ref_id = utils.get_ref(msg)
     except UnicodeEncodeError as e:
         self.fail(e)
     self.assertEqual(ref_id, "ref-")
Exemplo n.º 14
0
 def test_non_ascii_ref(self):
     msg = Message()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["In-Reply-To"] = "<ref-\xed>".encode("utf-8")
     msg.set_payload("Dummy message")
     try:
         ref_id = utils.get_ref(msg)
     except UnicodeEncodeError as e:
         self.fail(e)
     self.assertEqual(ref_id, "ref-")
Exemplo n.º 15
0
 def test_non_ascii_ref(self):
     msg = EmailMessage()
     msg["From"] = "*****@*****.**"
     msg["Message-ID"] = "<dummy>"
     msg["In-Reply-To"] = "<ref-\xed>"
     msg.set_content("Dummy message")
     try:
         ref_id = utils.get_ref(msg)
     except UnicodeEncodeError as e:
         self.fail(e)
     # utf-8 characters are perfectly legitimate here (RFC 6532) and
     # stripping it here makes no sense at all
     self.assertEqual(ref_id, "ref-\xed")
Exemplo n.º 16
0
def add_to_list(list_name, message):
    # timeit("1 start")
    mlist = MailingList.objects.get_or_create(name=list_name)[0]
    if not getattr(settings, "HYPERKITTY_BATCH_MODE", False):
        mlist.update_from_mailman()
    mlist.save()
    if mlist.archive_policy == ArchivePolicy.never.value:
        logger.info("Archiving disabled by list policy for %s", list_name)
        return
    if "Message-Id" not in message:
        raise ValueError("No 'Message-Id' header in email", message)
    # timeit("2 after ml, before checking email & sender")
    msg_id = get_message_id(message)
    if Email.objects.filter(mailinglist=mlist, message_id=msg_id).exists():
        raise DuplicateMessage(msg_id)
    email = Email(mailinglist=mlist, message_id=msg_id)
    email.in_reply_to = get_ref(message)  # Find thread id
    if message.get_unixfrom() is not None:
        mo = UNIXFROM_DATE_RE.match(message.get_unixfrom())
        if mo:
            archived_date = parsedate(mo.group(1))
            if archived_date is not None:
                email.archived_date = archived_date

    # Sender
    try:
        from_name, from_email = parseaddr(message['From'])
        from_name = header_to_unicode(from_name).strip()
        sender_address = from_email.decode("ascii").strip()
    except (UnicodeDecodeError, UnicodeEncodeError):
        raise ValueError("Non-ascii sender address", message)
    if not sender_address:
        if from_name:
            sender_address = re.sub("[^a-z0-9]", "", from_name.lower())
            if not sender_address:
                sender_address = "unknown"
            sender_address = "{}@example.com".format(sender_address)
        else:
            sender_address = "*****@*****.**"
    email.sender_name = from_name
    sender = Sender.objects.get_or_create(address=sender_address)[0]
    email.sender = sender
    if not getattr(settings, "HYPERKITTY_BATCH_MODE", False):
        try:
            sender.set_mailman_id()
        except MailmanConnectionError:
            pass
    # timeit("3 after sender, before email content")

    # Headers
    email.subject = header_to_unicode(message.get('Subject'))
    if email.subject is not None:
        # limit subject size to 512, it's a varchar field
        email.subject = email.subject[:512]
    msg_date = parsedate(message.get("Date"))
    if msg_date is None:
        # Absent or unparseable date
        msg_date = timezone.now()
    utcoffset = msg_date.utcoffset()
    if msg_date.tzinfo is not None:
        msg_date = msg_date.astimezone(timezone.utc)  # store in UTC
    email.date = msg_date
    if utcoffset is None:
        email.timezone = 0
    else:
        # in minutes
        email.timezone = int(
            ((utcoffset.days * 24 * 60 * 60) + utcoffset.seconds) / 60)

    # Content
    scrubber = Scrubber(message)
    # warning: scrubbing modifies the msg in-place
    email.content, attachments = scrubber.scrub()
    # timeit("4 after email content, before signals")

    # TODO: detect category?

    # Set or create the Thread
    if email.in_reply_to is not None:
        try:
            ref_msg = Email.objects.get(mailinglist=email.mailinglist,
                                        message_id=email.in_reply_to)
        except Email.DoesNotExist:
            # the parent may not be archived (on partial imports), create a new
            # thread for now.
            pass
        else:
            # re-use parent's thread-id
            email.parent = ref_msg
            email.thread_id = ref_msg.thread_id
            thread = ref_msg.thread

    thread_created = False
    if email.thread_id is None:
        # Create the thread if not found
        thread, thread_created = Thread.objects.get_or_create(
            mailinglist=email.mailinglist, thread_id=email.message_id_hash)
        email.thread = thread

    email.save()  # must save before setting the thread.starting_email

    thread.date_active = email.date
    if thread_created:
        thread.starting_email = email
    thread.save()
    if thread_created:
        new_thread.send("Mailman", thread=thread)
        # signal_results = new_thread.send_robust("Mailman", thread=thread)
        # for receiver, result in signal_results:
        #     if isinstance(result, Exception):
        #         logger.warning(
        #             "Signal 'new_thread' to {} raised an "
        #             "exception: {}".format(
        #             receiver.func_name, result))

    # Signals
    new_email.send("Mailman", email=email)
    # signal_results = new_email.send_robust("Mailman", email=email)
    # for receiver, result in signal_results:
    #     if isinstance(result, Exception):
    #         logger.warning(
    #             "Signal 'new_email' to {} raised an exception: {}".format(
    #             receiver.func_name, result))
    #         #logger.exception(result)
    #         #from traceback import print_exc; print_exc(result)
    # timeit("5 after signals, before save")
    # timeit("6 after save")
    #  compute thread props here because email must have been saved before
    # (there will be DB queries in this function)
    if not getattr(settings, "HYPERKITTY_BATCH_MODE", False):
        compute_thread_order_and_depth(email.thread)

    # Attachments (email must have been saved before)
    for attachment in attachments:
        counter, name, content_type, encoding, content = attachment
        if Attachment.objects.filter(email=email, counter=counter).exists():
            continue
        Attachment.objects.create(email=email,
                                  counter=counter,
                                  name=name,
                                  content_type=content_type,
                                  encoding=encoding,
                                  content=content)

    return email.message_id_hash
Exemplo n.º 17
0
def add_to_list(list_name, message):
    # timeit("1 start")
    mlist = MailingList.objects.get_or_create(name=list_name)[0]
    if not getattr(settings, "HYPERKITTY_BATCH_MODE", False):
        mlist.update_from_mailman()
    mlist.save()
    if mlist.archive_policy == ArchivePolicy.never.value:
        logger.info("Archiving disabled by list policy for %s", list_name)
        return
    if not message.has_key("Message-Id"):
        raise ValueError("No 'Message-Id' header in email", message)
    # timeit("2 after ml, before checking email & sender")
    msg_id = get_message_id(message)
    if Email.objects.filter(mailinglist=mlist, message_id=msg_id).exists():
        raise DuplicateMessage(msg_id)
    email = Email(mailinglist=mlist, message_id=msg_id)
    email.in_reply_to = get_ref(message)  # Find thread id

    # Sender
    try:
        from_name, from_email = parseaddr(message["From"])
        from_name = header_to_unicode(from_name).strip()
        sender_address = from_email.decode("ascii").strip()
    except (UnicodeDecodeError, UnicodeEncodeError):
        raise ValueError("Non-ascii sender address", message)
    if not sender_address:
        if from_name:
            sender_address = re.sub("[^a-z0-9]", "", from_name.lower())
            if not sender_address:
                sender_address = "unknown"
            sender_address = "{}@example.com".format(sender_address)
        else:
            sender_address = "*****@*****.**"
    sender = Sender.objects.get_or_create(address=sender_address)[0]
    sender.name = from_name  # update the name if needed
    sender.save()
    email.sender = sender
    if not getattr(settings, "HYPERKITTY_BATCH_MODE", False):
        set_sender_mailman_id(sender)
    # timeit("3 after sender, before email content")

    # Headers
    email.subject = header_to_unicode(message.get("Subject"))
    if email.subject is not None:
        # limit subject size to 512, it's a varchar field
        email.subject = email.subject[:512]
    msg_date = parsedate(message.get("Date"))
    if msg_date is None:
        # Absent or unparseable date
        msg_date = timezone.now()
    utcoffset = msg_date.utcoffset()
    if msg_date.tzinfo is not None:
        msg_date = msg_date.astimezone(timezone.utc)  # store in UTC
    email.date = msg_date
    if utcoffset is None:
        email.timezone = 0
    else:
        # in minutes
        email.timezone = int(((utcoffset.days * 24 * 60 * 60) + utcoffset.seconds) / 60)

    # Content
    scrubber = Scrubber(list_name, message)
    # warning: scrubbing modifies the msg in-place
    email.content, attachments = scrubber.scrub()
    # timeit("4 after email content, before signals")

    # TODO: detect category?

    # Set or create the Thread
    if email.in_reply_to is not None:
        try:
            ref_msg = Email.objects.get(mailinglist=email.mailinglist, message_id=email.in_reply_to)
        except Email.DoesNotExist:
            # the parent may not be archived (on partial imports), create a new
            # thread for now.
            pass
        else:
            # re-use parent's thread-id
            email.parent = ref_msg
            email.thread_id = ref_msg.thread_id
            ref_msg.thread.date_active = email.date
            ref_msg.thread.save()

    thread_created = False
    if email.thread_id is None:
        # Create the thread if not found
        thread = Thread.objects.create(
            mailinglist=email.mailinglist, thread_id=email.message_id_hash, date_active=email.date
        )
        thread_created = True
        email.thread = thread

    email.save()  # must save before setting the thread.starting_email

    if thread_created:
        thread.starting_email = email
        thread.save()
        new_thread.send("Mailman", thread=thread)
        # signal_results = new_thread.send_robust("Mailman", thread=thread)
        # for receiver, result in signal_results:
        #    if isinstance(result, Exception):
        #        logger.warning(
        #            "Signal 'new_thread' to {} raised an exception: {}".format(
        #            receiver.func_name, result))

    # Signals
    new_email.send("Mailman", email=email)
    # signal_results = new_email.send_robust("Mailman", email=email)
    # for receiver, result in signal_results:
    #    if isinstance(result, Exception):
    #        logger.warning(
    #            "Signal 'new_email' to {} raised an exception: {}".format(
    #            receiver.func_name, result))
    #        #logger.exception(result)
    #        #from traceback import print_exc; print_exc(result)
    # timeit("5 after signals, before save")
    # timeit("6 after save")
    # compute thread props here because email must have been saved before
    # (there will be DB queries in this function)
    if not getattr(settings, "HYPERKITTY_BATCH_MODE", False):
        compute_thread_order_and_depth(email.thread)

    # Attachments (email must have been saved before)
    for attachment in attachments:
        counter, name, content_type, encoding, content = attachment
        if Attachment.objects.filter(email=email, counter=counter).exists():
            continue
        Attachment.objects.create(
            email=email, counter=counter, name=name, content_type=content_type, encoding=encoding, content=content
        )

    return email.message_id_hash
Exemplo n.º 18
0
 def test_ref_parsing(self):
     with open(get_test_file("strange-in-reply-to-header.txt")) as email_file:
         msg = message_from_file(email_file)
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, "*****@*****.**")
Exemplo n.º 19
0
 def test_wrong_reply_to_format(self):
     with open(get_test_file("wrong-in-reply-to-header.txt")) as email_file:
         msg = message_from_file(email_file)
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, None)
def add_to_list(list_name, message):
    assert isinstance(message, EmailMessage)
    # timeit("1 start")
    mlist = MailingList.objects.get_or_create(name=list_name)[0]
    if not getattr(settings, "HYPERKITTY_BATCH_MODE", False):
        update_from_mailman(mlist.name)
    mlist.save()
    if mlist.archive_policy == ArchivePolicy.never.value:
        logger.info("Archiving disabled by list policy for %s", list_name)
        return
    if "Message-Id" not in message:
        raise ValueError("No 'Message-Id' header in email", message)
    # timeit("2 after ml, before checking email & sender")
    msg_id = get_message_id(message)
    if Email.objects.filter(mailinglist=mlist, message_id=msg_id).exists():
        raise DuplicateMessage(msg_id)
    email = Email(mailinglist=mlist, message_id=msg_id)
    email.in_reply_to = get_ref(message)  # Find thread id
    if message.get_unixfrom() is not None:
        mo = UNIXFROM_DATE_RE.match(message.get_unixfrom())
        if mo:
            archived_date = parsedate(mo.group(1))
            if archived_date is not None:
                email.archived_date = archived_date

    # Sender
    try:
        from_str = header_to_unicode(message['From'])
        from_name, from_email = parseaddr(from_str)
        from_name = from_name.strip()
        sender_address = from_email.encode('ascii').decode("ascii").strip()
    except (UnicodeDecodeError, UnicodeEncodeError):
        raise ValueError("Non-ascii sender address", message)
    if not sender_address:
        if from_name:
            sender_address = re.sub("[^a-z0-9]", "", from_name.lower())
            if not sender_address:
                sender_address = "unknown"
            sender_address = "{}@example.com".format(sender_address)
        else:
            sender_address = "*****@*****.**"
    email.sender_name = from_name
    sender = Sender.objects.get_or_create(address=sender_address)[0]
    email.sender = sender
    if not getattr(settings, "HYPERKITTY_BATCH_MODE", False):
        sender_mailman_id(sender.pk)
    # timeit("3 after sender, before email content")

    # Headers
    email.subject = header_to_unicode(message.get('Subject'))
    if email.subject is not None:
        # limit subject size to 512, it's a varchar field
        email.subject = email.subject[:512]
    msg_date = parsedate(message.get("Date"))
    if msg_date is None:
        # Absent or unparseable date
        msg_date = timezone.now()
    utcoffset = msg_date.utcoffset()
    if msg_date.tzinfo is not None:
        msg_date = msg_date.astimezone(timezone.utc)  # store in UTC
    email.date = msg_date
    if utcoffset is None:
        email.timezone = 0
    else:
        # in minutes
        email.timezone = int(
            ((utcoffset.days * 24 * 60 * 60) + utcoffset.seconds) / 60)

    # Content
    scrubber = Scrubber(message)
    # warning: scrubbing modifies the msg in-place
    email.content, attachments = scrubber.scrub()
    # timeit("4 after email content, before signals")

    # TODO: detect category?

    # Find the parent email.
    # This can't be moved to Email.on_pre_save() because Email.set_parent()
    # needs to be free to change the parent independently from the in_reply_to
    # property, and will save() the instance.
    # This, along with some of the work done in Email.on_pre_save(), could be
    # moved to an async task, but the rest of the app must be able to cope with
    # emails lacking this data, and email being process randomly (child before
    # parent). The work in Email.on_post_created() also depends on it, so be
    # careful with task dependencies if you ever do this.
    # Plus, it has "premature optimization" written all over it.
    if email.in_reply_to is not None:
        try:
            ref_msg = Email.objects.get(mailinglist=email.mailinglist,
                                        message_id=email.in_reply_to)
        except Email.DoesNotExist:
            # the parent may not be archived (on partial imports), create a new
            # thread for now.
            pass
        else:
            # re-use parent's thread-id
            email.parent = ref_msg
            email.thread_id = ref_msg.thread_id

    try:
        email.save()
    except DataError as e:
        raise ValueError(str(e))

    # Attachments (email must have been saved before)
    for attachment in attachments:
        counter, name, content_type, encoding, content = attachment
        if Attachment.objects.filter(email=email, counter=counter).exists():
            continue
        att = Attachment.objects.create(email=email,
                                        counter=counter,
                                        name=name,
                                        content_type=content_type,
                                        encoding=encoding)
        att.set_content(content)
        att.save()

    return email.message_id_hash
Exemplo n.º 21
0
 def test_wrong_reply_to_format(self):
     with open(get_test_file("wrong-in-reply-to-header.txt")) as email_file:
         msg = message_from_file(email_file)
     ref_id = utils.get_ref(msg)
     self.assertEqual(ref_id, None)