def test_empty_reference(self): msg = Message() msg["From"] = "*****@*****.**" msg["Message-ID"] = "<dummy>" msg["References"] = " " msg.set_payload("Dummy message") try: utils.get_ref(msg) except IndexError: self.fail("Empty 'References' tag should be handled")
def test_ref_parsing(self): with open( get_test_file("strange-in-reply-to-header.txt") ) as email_file: msg = message_from_file(email_file) ref_id = utils.get_ref(msg) self.assertEqual(ref_id, "*****@*****.**")
def test_in_reply_to(self): msg = Message() msg["From"] = "*****@*****.**" msg["Message-ID"] = "<dummy>" msg["In-Reply-To"] = " <ref-1> " msg.set_payload("Dummy message") ref_id = utils.get_ref(msg) self.assertEqual(ref_id, "ref-1")
def test_multiple_reference(self): msg = Message() msg["From"] = "*****@*****.**" msg["Message-ID"] = "<dummy>" msg["References"] = " <ref-1> <ref-2> " msg.set_payload("Dummy message") ref_id = utils.get_ref(msg) self.assertEqual(ref_id, "ref-2")
def test_reference_no_brackets(self): msg = Message() msg["From"] = "*****@*****.**" msg["Message-ID"] = "<dummy>" msg["References"] = "ref-1" msg.set_payload("Dummy message") ref_id = utils.get_ref(msg) self.assertEqual(ref_id, "ref-1")
def test_single_reference(self): msg = EmailMessage() msg["From"] = "*****@*****.**" msg["Message-ID"] = "<dummy>" msg["References"] = " <ref-1> " msg.set_content("Dummy message") ref_id = utils.get_ref(msg) self.assertEqual(ref_id, "ref-1")
def test_in_reply_to_and_reference(self): """The In-Reply-To header should win over References""" msg = Message() msg["From"] = "*****@*****.**" msg["Message-ID"] = "<dummy>" msg["In-Reply-To"] = " <ref-1> " msg["References"] = " <ref-2> " msg.set_payload("Dummy message") ref_id = utils.get_ref(msg) self.assertEqual(ref_id, "ref-1")
def test_non_ascii_ref(self): msg = Message() msg["From"] = "*****@*****.**" msg["Message-ID"] = "<dummy>" msg["In-Reply-To"] = u"<ref-\xed>".encode('utf-8') msg.set_payload("Dummy message") try: ref_id = utils.get_ref(msg) except UnicodeEncodeError as e: self.fail(e) self.assertEqual(ref_id, "ref-")
def test_non_ascii_ref(self): msg = Message() msg["From"] = "*****@*****.**" msg["Message-ID"] = "<dummy>" msg["In-Reply-To"] = "<ref-\xed>".encode("utf-8") msg.set_payload("Dummy message") try: ref_id = utils.get_ref(msg) except UnicodeEncodeError as e: self.fail(e) self.assertEqual(ref_id, "ref-")
def test_non_ascii_ref(self): msg = EmailMessage() msg["From"] = "*****@*****.**" msg["Message-ID"] = "<dummy>" msg["In-Reply-To"] = "<ref-\xed>" msg.set_content("Dummy message") try: ref_id = utils.get_ref(msg) except UnicodeEncodeError as e: self.fail(e) # utf-8 characters are perfectly legitimate here (RFC 6532) and # stripping it here makes no sense at all self.assertEqual(ref_id, "ref-\xed")
def add_to_list(list_name, message): # timeit("1 start") mlist = MailingList.objects.get_or_create(name=list_name)[0] if not getattr(settings, "HYPERKITTY_BATCH_MODE", False): mlist.update_from_mailman() mlist.save() if mlist.archive_policy == ArchivePolicy.never.value: logger.info("Archiving disabled by list policy for %s", list_name) return if "Message-Id" not in message: raise ValueError("No 'Message-Id' header in email", message) # timeit("2 after ml, before checking email & sender") msg_id = get_message_id(message) if Email.objects.filter(mailinglist=mlist, message_id=msg_id).exists(): raise DuplicateMessage(msg_id) email = Email(mailinglist=mlist, message_id=msg_id) email.in_reply_to = get_ref(message) # Find thread id if message.get_unixfrom() is not None: mo = UNIXFROM_DATE_RE.match(message.get_unixfrom()) if mo: archived_date = parsedate(mo.group(1)) if archived_date is not None: email.archived_date = archived_date # Sender try: from_name, from_email = parseaddr(message['From']) from_name = header_to_unicode(from_name).strip() sender_address = from_email.decode("ascii").strip() except (UnicodeDecodeError, UnicodeEncodeError): raise ValueError("Non-ascii sender address", message) if not sender_address: if from_name: sender_address = re.sub("[^a-z0-9]", "", from_name.lower()) if not sender_address: sender_address = "unknown" sender_address = "{}@example.com".format(sender_address) else: sender_address = "*****@*****.**" email.sender_name = from_name sender = Sender.objects.get_or_create(address=sender_address)[0] email.sender = sender if not getattr(settings, "HYPERKITTY_BATCH_MODE", False): try: sender.set_mailman_id() except MailmanConnectionError: pass # timeit("3 after sender, before email content") # Headers email.subject = header_to_unicode(message.get('Subject')) if email.subject is not None: # limit subject size to 512, it's a varchar field email.subject = email.subject[:512] msg_date = parsedate(message.get("Date")) if msg_date is None: # Absent or unparseable date msg_date = timezone.now() utcoffset = msg_date.utcoffset() if msg_date.tzinfo is not None: msg_date = msg_date.astimezone(timezone.utc) # store in UTC email.date = msg_date if utcoffset is None: email.timezone = 0 else: # in minutes email.timezone = int( ((utcoffset.days * 24 * 60 * 60) + utcoffset.seconds) / 60) # Content scrubber = Scrubber(message) # warning: scrubbing modifies the msg in-place email.content, attachments = scrubber.scrub() # timeit("4 after email content, before signals") # TODO: detect category? # Set or create the Thread if email.in_reply_to is not None: try: ref_msg = Email.objects.get(mailinglist=email.mailinglist, message_id=email.in_reply_to) except Email.DoesNotExist: # the parent may not be archived (on partial imports), create a new # thread for now. pass else: # re-use parent's thread-id email.parent = ref_msg email.thread_id = ref_msg.thread_id thread = ref_msg.thread thread_created = False if email.thread_id is None: # Create the thread if not found thread, thread_created = Thread.objects.get_or_create( mailinglist=email.mailinglist, thread_id=email.message_id_hash) email.thread = thread email.save() # must save before setting the thread.starting_email thread.date_active = email.date if thread_created: thread.starting_email = email thread.save() if thread_created: new_thread.send("Mailman", thread=thread) # signal_results = new_thread.send_robust("Mailman", thread=thread) # for receiver, result in signal_results: # if isinstance(result, Exception): # logger.warning( # "Signal 'new_thread' to {} raised an " # "exception: {}".format( # receiver.func_name, result)) # Signals new_email.send("Mailman", email=email) # signal_results = new_email.send_robust("Mailman", email=email) # for receiver, result in signal_results: # if isinstance(result, Exception): # logger.warning( # "Signal 'new_email' to {} raised an exception: {}".format( # receiver.func_name, result)) # #logger.exception(result) # #from traceback import print_exc; print_exc(result) # timeit("5 after signals, before save") # timeit("6 after save") # compute thread props here because email must have been saved before # (there will be DB queries in this function) if not getattr(settings, "HYPERKITTY_BATCH_MODE", False): compute_thread_order_and_depth(email.thread) # Attachments (email must have been saved before) for attachment in attachments: counter, name, content_type, encoding, content = attachment if Attachment.objects.filter(email=email, counter=counter).exists(): continue Attachment.objects.create(email=email, counter=counter, name=name, content_type=content_type, encoding=encoding, content=content) return email.message_id_hash
def add_to_list(list_name, message): # timeit("1 start") mlist = MailingList.objects.get_or_create(name=list_name)[0] if not getattr(settings, "HYPERKITTY_BATCH_MODE", False): mlist.update_from_mailman() mlist.save() if mlist.archive_policy == ArchivePolicy.never.value: logger.info("Archiving disabled by list policy for %s", list_name) return if not message.has_key("Message-Id"): raise ValueError("No 'Message-Id' header in email", message) # timeit("2 after ml, before checking email & sender") msg_id = get_message_id(message) if Email.objects.filter(mailinglist=mlist, message_id=msg_id).exists(): raise DuplicateMessage(msg_id) email = Email(mailinglist=mlist, message_id=msg_id) email.in_reply_to = get_ref(message) # Find thread id # Sender try: from_name, from_email = parseaddr(message["From"]) from_name = header_to_unicode(from_name).strip() sender_address = from_email.decode("ascii").strip() except (UnicodeDecodeError, UnicodeEncodeError): raise ValueError("Non-ascii sender address", message) if not sender_address: if from_name: sender_address = re.sub("[^a-z0-9]", "", from_name.lower()) if not sender_address: sender_address = "unknown" sender_address = "{}@example.com".format(sender_address) else: sender_address = "*****@*****.**" sender = Sender.objects.get_or_create(address=sender_address)[0] sender.name = from_name # update the name if needed sender.save() email.sender = sender if not getattr(settings, "HYPERKITTY_BATCH_MODE", False): set_sender_mailman_id(sender) # timeit("3 after sender, before email content") # Headers email.subject = header_to_unicode(message.get("Subject")) if email.subject is not None: # limit subject size to 512, it's a varchar field email.subject = email.subject[:512] msg_date = parsedate(message.get("Date")) if msg_date is None: # Absent or unparseable date msg_date = timezone.now() utcoffset = msg_date.utcoffset() if msg_date.tzinfo is not None: msg_date = msg_date.astimezone(timezone.utc) # store in UTC email.date = msg_date if utcoffset is None: email.timezone = 0 else: # in minutes email.timezone = int(((utcoffset.days * 24 * 60 * 60) + utcoffset.seconds) / 60) # Content scrubber = Scrubber(list_name, message) # warning: scrubbing modifies the msg in-place email.content, attachments = scrubber.scrub() # timeit("4 after email content, before signals") # TODO: detect category? # Set or create the Thread if email.in_reply_to is not None: try: ref_msg = Email.objects.get(mailinglist=email.mailinglist, message_id=email.in_reply_to) except Email.DoesNotExist: # the parent may not be archived (on partial imports), create a new # thread for now. pass else: # re-use parent's thread-id email.parent = ref_msg email.thread_id = ref_msg.thread_id ref_msg.thread.date_active = email.date ref_msg.thread.save() thread_created = False if email.thread_id is None: # Create the thread if not found thread = Thread.objects.create( mailinglist=email.mailinglist, thread_id=email.message_id_hash, date_active=email.date ) thread_created = True email.thread = thread email.save() # must save before setting the thread.starting_email if thread_created: thread.starting_email = email thread.save() new_thread.send("Mailman", thread=thread) # signal_results = new_thread.send_robust("Mailman", thread=thread) # for receiver, result in signal_results: # if isinstance(result, Exception): # logger.warning( # "Signal 'new_thread' to {} raised an exception: {}".format( # receiver.func_name, result)) # Signals new_email.send("Mailman", email=email) # signal_results = new_email.send_robust("Mailman", email=email) # for receiver, result in signal_results: # if isinstance(result, Exception): # logger.warning( # "Signal 'new_email' to {} raised an exception: {}".format( # receiver.func_name, result)) # #logger.exception(result) # #from traceback import print_exc; print_exc(result) # timeit("5 after signals, before save") # timeit("6 after save") # compute thread props here because email must have been saved before # (there will be DB queries in this function) if not getattr(settings, "HYPERKITTY_BATCH_MODE", False): compute_thread_order_and_depth(email.thread) # Attachments (email must have been saved before) for attachment in attachments: counter, name, content_type, encoding, content = attachment if Attachment.objects.filter(email=email, counter=counter).exists(): continue Attachment.objects.create( email=email, counter=counter, name=name, content_type=content_type, encoding=encoding, content=content ) return email.message_id_hash
def test_ref_parsing(self): with open(get_test_file("strange-in-reply-to-header.txt")) as email_file: msg = message_from_file(email_file) ref_id = utils.get_ref(msg) self.assertEqual(ref_id, "*****@*****.**")
def test_wrong_reply_to_format(self): with open(get_test_file("wrong-in-reply-to-header.txt")) as email_file: msg = message_from_file(email_file) ref_id = utils.get_ref(msg) self.assertEqual(ref_id, None)
def add_to_list(list_name, message): assert isinstance(message, EmailMessage) # timeit("1 start") mlist = MailingList.objects.get_or_create(name=list_name)[0] if not getattr(settings, "HYPERKITTY_BATCH_MODE", False): update_from_mailman(mlist.name) mlist.save() if mlist.archive_policy == ArchivePolicy.never.value: logger.info("Archiving disabled by list policy for %s", list_name) return if "Message-Id" not in message: raise ValueError("No 'Message-Id' header in email", message) # timeit("2 after ml, before checking email & sender") msg_id = get_message_id(message) if Email.objects.filter(mailinglist=mlist, message_id=msg_id).exists(): raise DuplicateMessage(msg_id) email = Email(mailinglist=mlist, message_id=msg_id) email.in_reply_to = get_ref(message) # Find thread id if message.get_unixfrom() is not None: mo = UNIXFROM_DATE_RE.match(message.get_unixfrom()) if mo: archived_date = parsedate(mo.group(1)) if archived_date is not None: email.archived_date = archived_date # Sender try: from_str = header_to_unicode(message['From']) from_name, from_email = parseaddr(from_str) from_name = from_name.strip() sender_address = from_email.encode('ascii').decode("ascii").strip() except (UnicodeDecodeError, UnicodeEncodeError): raise ValueError("Non-ascii sender address", message) if not sender_address: if from_name: sender_address = re.sub("[^a-z0-9]", "", from_name.lower()) if not sender_address: sender_address = "unknown" sender_address = "{}@example.com".format(sender_address) else: sender_address = "*****@*****.**" email.sender_name = from_name sender = Sender.objects.get_or_create(address=sender_address)[0] email.sender = sender if not getattr(settings, "HYPERKITTY_BATCH_MODE", False): sender_mailman_id(sender.pk) # timeit("3 after sender, before email content") # Headers email.subject = header_to_unicode(message.get('Subject')) if email.subject is not None: # limit subject size to 512, it's a varchar field email.subject = email.subject[:512] msg_date = parsedate(message.get("Date")) if msg_date is None: # Absent or unparseable date msg_date = timezone.now() utcoffset = msg_date.utcoffset() if msg_date.tzinfo is not None: msg_date = msg_date.astimezone(timezone.utc) # store in UTC email.date = msg_date if utcoffset is None: email.timezone = 0 else: # in minutes email.timezone = int( ((utcoffset.days * 24 * 60 * 60) + utcoffset.seconds) / 60) # Content scrubber = Scrubber(message) # warning: scrubbing modifies the msg in-place email.content, attachments = scrubber.scrub() # timeit("4 after email content, before signals") # TODO: detect category? # Find the parent email. # This can't be moved to Email.on_pre_save() because Email.set_parent() # needs to be free to change the parent independently from the in_reply_to # property, and will save() the instance. # This, along with some of the work done in Email.on_pre_save(), could be # moved to an async task, but the rest of the app must be able to cope with # emails lacking this data, and email being process randomly (child before # parent). The work in Email.on_post_created() also depends on it, so be # careful with task dependencies if you ever do this. # Plus, it has "premature optimization" written all over it. if email.in_reply_to is not None: try: ref_msg = Email.objects.get(mailinglist=email.mailinglist, message_id=email.in_reply_to) except Email.DoesNotExist: # the parent may not be archived (on partial imports), create a new # thread for now. pass else: # re-use parent's thread-id email.parent = ref_msg email.thread_id = ref_msg.thread_id try: email.save() except DataError as e: raise ValueError(str(e)) # Attachments (email must have been saved before) for attachment in attachments: counter, name, content_type, encoding, content = attachment if Attachment.objects.filter(email=email, counter=counter).exists(): continue att = Attachment.objects.create(email=email, counter=counter, name=name, content_type=content_type, encoding=encoding) att.set_content(content) att.save() return email.message_id_hash