def test_cant_write_error(self):
# This message throws an exception which is caught by the catchall
# except clause. Ensure we can then write the error message.
with open(get_test_file("cant_write_error_message.txt")) as email_file:
msg = message_from_file(email_file)
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
# Add a scond message because we need to archive something.
msg = EmailMessage()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg2>"
msg["Date"] = "01 Feb 2015 12:00:00"
msg.set_payload("msg2")
mbox.add(msg)
mbox.close()
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
# Message 1 must have been rejected, but no crash
self.assertIn("failed to import, skipping", output.getvalue())
# Message 2 must have been accepted
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
def test_ref_parsing(self):
with open(
get_test_file("strange-in-reply-to-header.txt")
) as email_file:
msg = message_from_file(email_file)
ref_id = utils.get_ref(msg)
self.assertEqual(ref_id, "*****@*****.**")
def test_bad_content_type(self):
# Content-Type: binary/octet-stream throws KeyError.
with open(get_test_file("bad_content_type.txt")) as email_file:
msg = message_from_file(email_file)
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
# Second message
msg = EmailMessage()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg1>"
msg["Date"] = "01 Feb 2015 12:00:00"
msg.set_payload("msg1")
mbox.add(msg)
mbox.close()
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
# Message 1 must have been rejected, but no crash
self.assertIn("Failed adding message <msg@id>:", output.getvalue())
# Message 2 must have been accepted
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
def test_another_unconvertable_message(self):
# This message can't be converted to an email.message.EmailMessage.
# This fails with Python>=3.7.5 because
# https://bugs.python.org/issue37491 is fixed.
if sys.hexversion >= 0x30705f0:
raise SkipTest
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
# We have to do it this way to see the exception.
with open(get_test_file("unconvertable_msg-2.txt"), "rb") as em_file:
with open(os.path.join(self.tmpdir, "test.mbox"), "wb") as mb_file:
mb_file.write(em_file.read())
# Add a scond message because we need to archive something.
msg = EmailMessage()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg2>"
msg["Date"] = "01 Feb 2015 12:00:00"
msg.set_payload("msg2")
mbox.add(msg)
mbox.close()
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
# Message 1 must have been rejected, but no crash
self.assertIn("Failed to convert n/a to email", output.getvalue())
# Message 2 must have been accepted
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
def test_unknown_encoding(self):
# Spam messages have been seen with bogus charset= encodings which
# throw LookupError.
with open(get_test_file("unknown-charset.txt")) as email_file:
msg = message_from_file(email_file)
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
# Second message
msg = EmailMessage()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg2>"
msg["Date"] = "01 Feb 2015 12:00:00"
msg.set_payload("msg2")
mbox.add(msg)
mbox.close()
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
# Message 1 must have been rejected, but no crash
self.assertIn("Failed adding message <msg@id>:", output.getvalue())
# Message 2 must have been accepted
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
def test_ungetable_message(self):
# This mbox message can't be converted to bytes.
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
# We have to do it this way to see the exception.
with open(get_test_file("unicode_issue.txt"), "rb") as em_file:
with open(os.path.join(self.tmpdir, "test.mbox"), "wb") as mb_file:
mb_file.write(em_file.read())
# Add a scond message because we need to archive something.
msg = EmailMessage()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg2>"
msg["Date"] = "01 Feb 2015 12:00:00"
msg.set_payload("msg2")
mbox.add(msg)
mbox.close()
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
# Message 1 must have been rejected, but no crash
self.assertIn("Failed to convert n/a to bytes", output.getvalue())
# Message 2 must have been accepted
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
def test_remove_next_part_from_content(self):
with open(get_test_file("pipermail_nextpart.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents = scrubber.scrub()[0]
self.failIf("-------------- next part --------------" in contents)
def test_remove_next_part_from_content(self):
with open(get_test_file("pipermail_nextpart.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents = scrubber.scrub()[0]
self.failIf("-------------- next part --------------" in contents)
def test_another_wrong_encoding(self):
# This is gb2312 with a bad character. It seems to fail before
# getting to the back end.
with open(get_test_file("another-wrong-encoding.txt")) as email_file:
msg = message_from_file(email_file)
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
# Second message
msg = EmailMessage()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg1>"
msg["Date"] = "01 Feb 2015 12:00:00"
msg.set_payload("msg1")
mbox.add(msg)
mbox.close()
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
# Message 1 must have been rejected, but no crash
self.assertIn("Failed adding message <msg@id>:", output.getvalue())
# Message 2 must have been accepted
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
def test_wrong_encoding(self):
"""badly encoded message, only fails on PostgreSQL"""
db_engine = settings.DATABASES[DEFAULT_DB_ALIAS]["ENGINE"]
if db_engine == "django.db.backends.sqlite3":
raise SkipTest # SQLite will accept anything
with open(get_test_file("payload-utf8-wrong.txt")) as email_file:
msg = message_from_file(email_file)
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
# Second message
msg = EmailMessage()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg1>"
msg["Date"] = "01 Feb 2015 12:00:00"
msg.set_payload("msg1")
mbox.add(msg)
mbox.close()
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
# Message 1 must have been rejected, but no crash
self.assertIn("Message wrong.encoding failed to import, skipping",
output.getvalue())
# Message 2 must have been accepted
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
def test_wrong_encoding(self):
"""badly encoded message, only fails on PostgreSQL"""
db_engine = settings.DATABASES[DEFAULT_DB_ALIAS]["ENGINE"]
if db_engine == "django.db.backends.sqlite3":
raise SkipTest # SQLite will accept anything
with open(get_test_file("payload-utf8-wrong.txt")) as email_file:
msg = message_from_file(email_file)
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
# Second message
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<msg1>"
msg["Date"] = "2015-02-01 12:00:00"
msg.set_payload("msg1")
mbox.add(msg)
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
self.command.execute(os.path.join(self.tmpdir, "test.mbox"), **kw)
# Message 1 must have been rejected, but no crash
self.assertIn("Message wrong.encoding failed to import, skipping",
output.getvalue())
# Message 2 must have been accepted
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
def test_attachment_insert_order(self):
"""Attachments must not be inserted in the DB before the email"""
with open(get_test_file("attachment-1.txt")) as email_file:
msg = message_from_file(email_file)
try:
add_to_list("example-list", msg)
except IntegrityError, e:
self.fail(e)
def test_html_only_email(self):
# This email only has an HTML part, thus the scrubbed content will be
# empty. It should be an unicode empty string, not str.
with open(get_test_file("html-email-2.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents = scrubber.scrub()[0]
self.assertTrue(isinstance(contents, unicode),
u"Scrubbed content should always be unicode")
def test_html_only_email(self):
# This email only has an HTML part, thus the scrubbed content will be
# empty. It should be an unicode empty string, not str.
with open(get_test_file("html-email-2.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents = scrubber.scrub()[0]
self.assertTrue(isinstance(contents, unicode),
u"Scrubbed content should always be unicode")
def test_string_no_cset_attachment(self):
"""Some attachments have content as str with no specified encoding."""
with open(get_test_file("attachment-3.txt")) as email_file:
msg = message_from_file(email_file, EmailMessage, policy=default)
try:
add_to_list("example-list", msg)
except IntegrityError as e:
self.fail(e)
self.assertEqual(Attachment.objects.count(), 1)
def test_non_ascii_payload(self):
"""Scrubber must handle non-ascii messages"""
for enc in ["utf8", "iso8859"]:
with open(get_test_file("payload-%s.txt" % enc)) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents = scrubber.scrub()[0]
self.assertTrue(isinstance(contents, unicode))
self.assertEqual(contents, u'This message contains non-ascii '
u'characters:\n\xe9 \xe8 \xe7 \xe0 \xee \xef \xeb \u20ac\n')
def test_name_unicode(self):
for num in range(1, 6):
with open(get_test_file("attachment-%d.txt" % num)) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
attachments = scrubber.scrub()[1]
for attachment in attachments:
name = attachment[1]
self.assertTrue(isinstance(name, unicode),
"attachment %r must be unicode" % name)
def test_attachment_insert_order(self):
"""Attachments must not be inserted in the DB before the email"""
with open(get_test_file("attachment-1.txt")) as email_file:
msg = message_from_file(email_file, EmailMessage, policy=default)
try:
add_to_list("example-list", msg)
except IntegrityError as e:
self.fail(e)
self.assertEqual(Email.objects.count(), 1)
self.assertEqual(Attachment.objects.count(), 1)
def test_name_unicode(self):
for num in range(1, 6):
with open(get_test_file("attachment-%d.txt" % num)) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
attachments = scrubber.scrub()[1]
for attachment in attachments:
name = attachment[1]
self.assertTrue(isinstance(name, unicode),
"attachment %r must be unicode" % name)
def test_bad_content_type(self):
"""Scrubber must handle unknown content-types"""
with open(get_test_file("payload-unknown.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
try:
contents = scrubber.scrub()[0]
except LookupError, e:
import traceback
print(traceback.format_exc())
self.fail(e) # codec not found
def test_bad_content_type(self):
"""Scrubber must handle unknown content-types"""
with open(get_test_file("payload-unknown.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
try:
contents = scrubber.scrub()[0]
except LookupError, e:
import traceback
print(traceback.format_exc())
self.fail(e) # codec not found
def test_non_ascii_payload(self):
"""Scrubber must handle non-ascii messages"""
for enc in ["utf8", "iso8859"]:
with open(get_test_file("payload-%s.txt" % enc)) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents = scrubber.scrub()[0]
self.assertTrue(isinstance(contents, unicode))
self.assertEqual(
contents, u'This message contains non-ascii '
u'characters:\n\xe9 \xe8 \xe7 \xe0 \xee \xef \xeb \u20ac\n')
def test_attachment_1(self):
with open(get_test_file("attachment-1.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
self.assertEqual(attachments[0],
(2, u'puntogil.vcf', u'text/x-vcard', u"utf-8",
'begin:vcard\r\nfn:gil\r\nn:;gil\r\nversion:2.1\r\n'
'end:vcard\r\n\r\n'))
self.assertEqual(
contents, "This is a test message.\r\n\r\n"
"\n-- \ndevel mailing list\[email protected]\n"
"https://admin.fedoraproject.org/mailman/listinfo/devel\n")
def test_attachment_5(self):
with open(get_test_file("attachment-5.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
# text attachment
self.assertEqual(attachments[0][0:4],
(2, u"todo-djeuner.txt", "text/plain", "utf-8"))
self.assertEqual(len(attachments[0][4]), 112)
# Scrubbed content
self.assertEqual(contents, u'This is a test, HTML message with '
u'accented letters : \xe9 \xe8 \xe7 \xe0.\r\nAnd an '
u'attachment with an accented filename\r\n\r\n\r\n\r\n')
def test_html_email_1(self):
with open(get_test_file("html-email-1.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
# HTML part
self._check_html_attachment(
attachments[0], (2, u"attachment.html", "text/html", "iso-8859-1"))
self.assertEqual(len(attachments[0][4]), 2723)
# Scrubbed content
self.assertEqual(
contents, u"This is a test message\r\n"
u"Non-ASCII chars: r\xe9ponse fran\xe7ais \n")
def test_html_email_1(self):
with open(get_test_file("html-email-1.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
# HTML part
self._check_html_attachment(attachments[0],
(2, u"attachment.html", "text/html", "iso-8859-1"))
self.assertEqual(len(attachments[0][4]), 2723)
# Scrubbed content
self.assertEqual(contents,
u"This is a test message\r\n"
u"Non-ASCII chars: r\xe9ponse fran\xe7ais \n")
def test_attachment_1(self):
with open(get_test_file("attachment-1.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
self.assertEqual(attachments[0], (
2, u'puntogil.vcf', u'text/x-vcard', u"utf-8",
'begin:vcard\r\nfn:gil\r\nn:;gil\r\nversion:2.1\r\n'
'end:vcard\r\n\r\n'))
self.assertEqual(contents,
"This is a test message.\r\n\r\n"
"\n-- \ndevel mailing list\[email protected]\n"
"https://admin.fedoraproject.org/mailman/listinfo/devel\n"
)
def test_another_wrong_encoding_part_two(self):
# This is gb2312 with a bad character. Since above failure was fixed,
# test current behavior.
with open(get_test_file("another-wrong-encoding.txt")) as email_file:
msg = message_from_file(email_file)
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
def test_bad_content_type_part_two(self):
# Content-Type: binary/octet-stream. Since above failure was fixed,
# test current behavior.
with open(get_test_file("bad_content_type.txt")) as email_file:
msg = message_from_file(email_file)
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
def test_attachment_5(self):
with open(get_test_file("attachment-5.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
# text attachment
self.assertEqual(attachments[0][0:4],
(2, u"todo-djeuner.txt", "text/plain", "utf-8"))
self.assertEqual(len(attachments[0][4]), 112)
# Scrubbed content
self.assertEqual(
contents, u'This is a test, HTML message with '
u'accented letters : \xe9 \xe8 \xe7 \xe0.\r\nAnd an '
u'attachment with an accented filename\r\n\r\n\r\n\r\n')
def test_attachment_3(self):
with open(get_test_file("attachment-3.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 2)
# HTML part
self._check_html_attachment(
attachments[0], (3, u"attachment.html", "text/html", "iso-8859-1"))
self.assertEqual(len(attachments[0][4]), 3134)
# Image attachment
self.assertEqual(attachments[1][0:4],
(4, u"GeoffreyRoucourt.jpg", "image/jpeg", None))
self.assertEqual(len(attachments[1][4]), 282180)
# Scrubbed content
self.assertEqual(contents, u"This is a test message\r\n")
def test_non_ascii_text_attachment_declared_as_ascii(self):
# Some defective mail has a text attachment declared as ascii but
# containing non-ascii. We should just replace the non-ascii character.
with open(get_test_file("attachment-4.txt")) as email_file:
msg = message_from_file(email_file, EmailMessage, policy=default)
try:
add_to_list("example-list", msg)
except Exception as e:
self.fail(e)
self.assertEqual(Attachment.objects.count(), 1)
self.assertEqual(
Attachment.objects.all()[0].content,
b'All votes are reported in the form "*Y-N-A*" '
b'(*in favor-Y???opposed-N???abstentions-A*; e.g. '
b'"5-1-2" means "5 in favor, 1 opposed, and 2 '
b'abstentions").\n')
def test_ungettable_date(self):
# Certain bad Date: headers will throw TypeError on msg.get('date').
# Test that we handle that.
# For this test we use the testdata mbox directly to avoid a parse
# error in message_from_file().
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
kw["verbosity"] = 2
call_command('hyperkitty_import',
get_test_file("non-ascii-date-header.txt"), **kw)
# The message should be archived.
self.assertEqual(Email.objects.count(), 1)
# But there should be an error message.
self.assertIn("Can't get date header in message", output.getvalue())
def test_attachment_3(self):
with open(get_test_file("attachment-3.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 2)
# HTML part
self._check_html_attachment(attachments[0],
(3, u"attachment.html", "text/html", "iso-8859-1"))
self.assertEqual(len(attachments[0][4]), 3134)
# Image attachment
self.assertEqual(attachments[1][0:4],
(4, u"GeoffreyRoucourt.jpg", "image/jpeg", None))
self.assertEqual(len(attachments[1][4]), 282180)
# Scrubbed content
self.assertEqual(contents, u"This is a test message\r\n")
def test_attachment_2(self):
with open(get_test_file("attachment-2.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
self.assertEqual(attachments[0], (
3, u'signature.asc', u'application/pgp-signature', None,
'-----BEGIN PGP SIGNATURE-----\r\nVersion: GnuPG v1.4.12 '
'(GNU/Linux)\r\nComment: Using GnuPG with Mozilla - '
'http://www.enigmail.net/\r\n\r\niEYEARECAAYFAlBhm3oACgkQhmBj'
'z394AnmMnQCcC+6tWcqE1dPQmIdRbLXgKGVp\r\nEeUAn2OqtaXaXaQV7rx+'
'SmOldmSzcFw4\r\n=OEJv\r\n-----END PGP SIGNATURE-----\r\n'))
self.assertEqual(contents,
u"This is a test message\r\nNon-ascii chars: Hofm\xfchlgasse\r\n"
u"\n-- \ndevel mailing list\[email protected]\n"
u"https://admin.fedoraproject.org/mailman/listinfo/devel\n"
)
def test_attachment_4(self):
with open(get_test_file("attachment-4.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 2)
# HTML part
self._check_html_attachment(attachments[0],
(3, u"attachment.html", "text/html", "iso-8859-1"))
self.assertEqual(len(attachments[0][4]), 114)
# text attachment
self.assertEqual(attachments[1][0:4],
#(4, u"todo-déjeuner.txt", "text/plain", "utf-8"))
(4, u"todo-djeuner.txt", "text/plain", "utf-8"))
self.assertEqual(len(attachments[1][4]), 112)
# Scrubbed content
self.assertEqual(contents, u'This is a test, HTML message with '
u'accented letters : \xe9 \xe8 \xe7 \xe0.\r\nAnd an '
u'attachment with an accented filename\r\n')
def test_bad_subject_header(self):
# This message has a Subject: header with an encoded word that
# contains \x85 which becomes a unicode next line.
with open(get_test_file("bad-subject-header.txt")) as email_file:
msg = message_from_file(email_file)
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
mbox.close()
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
# Message must have been accepted
self.assertEqual(MailingList.objects.count(), 1)
self.assertEqual(Email.objects.count(), 1)
self.assertEqual(Email.objects.first().subject,
'(et en plus, en local\x85)')
def test_bad_date_tz_and_no_resent_date(self):
# If the Dete: header is bad and no Resent-Date: header, fall back
# to the unixfrom date.
with open(get_test_file("bad_date_tz.txt")) as email_file:
msg = message_from_file(email_file)
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
mbox.close()
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
# The message should be archived.
self.assertEqual(Email.objects.count(), 1)
# The archived_date should be Dec 1 00:56:19 1999 1999
self.assertEqual(Email.objects.all()[0].date,
datetime(1999, 12, 1, 0, 56, 19, tzinfo=utc))
def test_attachment_2(self):
with open(get_test_file("attachment-2.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
self.assertEqual(
attachments[0],
(3, u'signature.asc', u'application/pgp-signature', None,
'-----BEGIN PGP SIGNATURE-----\r\nVersion: GnuPG v1.4.12 '
'(GNU/Linux)\r\nComment: Using GnuPG with Mozilla - '
'http://www.enigmail.net/\r\n\r\niEYEARECAAYFAlBhm3oACgkQhmBj'
'z394AnmMnQCcC+6tWcqE1dPQmIdRbLXgKGVp\r\nEeUAn2OqtaXaXaQV7rx+'
'SmOldmSzcFw4\r\n=OEJv\r\n-----END PGP SIGNATURE-----\r\n'))
self.assertEqual(
contents,
u"This is a test message\r\nNon-ascii chars: Hofm\xfchlgasse\r\n"
u"\n-- \ndevel mailing list\[email protected]\n"
u"https://admin.fedoraproject.org/mailman/listinfo/devel\n")
def test_no_date_but_resent_date(self):
# If there's no Dete: header, fall back to Resent-Date:.
with open(get_test_file("resent-date.txt")) as email_file:
msg = message_from_file(email_file)
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
mbox.close()
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
# The message should be archived.
self.assertEqual(Email.objects.count(), 1)
# The archived_date should be 8 Nov 1999 20:53:05 -0600 which is
# 9 Nov 1999 02:53:05 UTC.
self.assertEqual(Email.objects.all()[0].date,
datetime(1999, 11, 9, 2, 53, 5, tzinfo=utc))
def test_attachment_4(self):
with open(get_test_file("attachment-4.txt")) as email_file:
msg = message_from_file(email_file)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 2)
# HTML part
self._check_html_attachment(
attachments[0], (3, u"attachment.html", "text/html", "iso-8859-1"))
self.assertEqual(len(attachments[0][4]), 114)
# text attachment
self.assertEqual(
attachments[1][0:4],
#(4, u"todo-déjeuner.txt", "text/plain", "utf-8"))
(4, u"todo-djeuner.txt", "text/plain", "utf-8"))
self.assertEqual(len(attachments[1][4]), 112)
# Scrubbed content
self.assertEqual(
contents, u'This is a test, HTML message with '
u'accented letters : \xe9 \xe8 \xe7 \xe0.\r\nAnd an '
u'attachment with an accented filename\r\n')
def test_folding_with_cr(self):
# See https://gitlab.com/mailman/hyperkitty/-/issues/280 for the
# issue.
with open(get_test_file("bad_folding.txt")) as email_file:
msg = message_from_file(email_file)
mbox = mailbox.mbox(os.path.join(self.tmpdir, "test.mbox"))
mbox.add(msg)
mbox.close()
# do the import
output = StringIO()
kw = self.common_cmd_args.copy()
kw["stdout"] = kw["stderr"] = output
call_command('hyperkitty_import', os.path.join(self.tmpdir,
"test.mbox"), **kw)
# The message should be archived.
self.assertEqual(Email.objects.count(), 1)
# The subject should be ???
self.assertEqual(
Email.objects.all()[0].subject,
'[<redacted>] Sicherheit 2005: Stichworte und '
'Vorschlag PC-Mitglieder; Ergänzung!')
def test_attachment_local_storage_bad_list_name(self):
# The HYPERKITTY_ATTACHMENT_FOLDER config allows usage of a local
# folder for attachments. Verify that bad list names don't crash the
# app.
with open(get_test_file("attachment-1.txt")) as email_file:
msg = message_from_file(email_file, EmailMessage, policy=default)
attachment_folder = os.path.join(self.tmpdir, "attachments")
with self.settings(HYPERKITTY_ATTACHMENT_FOLDER=attachment_folder):
add_to_list("list.example.com", msg)
add_to_list("list@[email protected]", msg)
email1 = Email.objects.filter(
mailinglist__name="list.example.com").first()
email2 = Email.objects.filter(
mailinglist__name="list@[email protected]").first()
self.assertTrue(
os.path.exists(
os.path.join(
attachment_folder,
"list.example.com",
"none",
"E3",
"YP",
"52",
str(email1.id),
"2",
)))
self.assertTrue(
os.path.exists(
os.path.join(
attachment_folder,
"example.com",
"list@local",
"E3",
"YP",
"52",
str(email2.id),
"2",
)))
def test_wrong_reply_to_format(self):
with open(get_test_file("wrong-in-reply-to-header.txt")) as email_file:
msg = message_from_file(email_file)
ref_id = utils.get_ref(msg)
self.assertEqual(ref_id, None)
def test_ref_parsing(self):
with open(get_test_file("strange-in-reply-to-header.txt")) as email_file:
msg = message_from_file(email_file)
ref_id = utils.get_ref(msg)
self.assertEqual(ref_id, "*****@*****.**")
