def test_remove_next_part_from_content(self):
with open(get_test_file("pipermail_nextpart.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.failIf("-------------- next part --------------" in contents)
def test_remove_next_part_from_content(self):
with open(get_test_file("pipermail_nextpart.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.failIf("-------------- next part --------------" in contents)
def test_attachment_2(self):
with open(get_test_file("attachment-2.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
self.assertEqual(
attachments[0],
(
3,
"signature.asc",
"application/pgp-signature",
None,
"-----BEGIN PGP SIGNATURE-----\r\nVersion: GnuPG v1.4.12 "
"(GNU/Linux)\r\nComment: Using GnuPG with Mozilla - "
"http://www.enigmail.net/\r\n\r\niEYEARECAAYFAlBhm3oACgkQhmBj"
"z394AnmMnQCcC+6tWcqE1dPQmIdRbLXgKGVp\r\nEeUAn2OqtaXaXaQV7rx+"
"SmOldmSzcFw4\r\n=OEJv\r\n-----END PGP SIGNATURE-----\r\n",
),
)
self.assertEqual(
contents,
u"This is a test message\r\nNon-ascii chars: Hofm\xfchlgasse\r\n"
u"\n-- \ndevel mailing list\[email protected]\n"
u"https://admin.fedoraproject.org/mailman/listinfo/devel\n",
)
def test_html_only_email(self):
# This email only has an HTML part, thus the scrubbed content will be
# empty. It should be an unicode empty string, not str.
with open(get_test_file("html-email-2.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertTrue(isinstance(contents, unicode),
u"Scrubbed content should always be unicode")
def test_html_only_email(self):
# This email only has an HTML part, thus the scrubbed content will be
# empty. It should be an unicode empty string, not str.
with open(get_test_file("html-email-2.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertTrue(isinstance(contents, unicode),
u"Scrubbed content should always be unicode")
def test_name_unicode(self):
for num in range(1, 6):
with open(get_test_file("attachment-%d.txt" % num)) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
for attachment in attachments:
name = attachment[1]
self.assertTrue(isinstance(name, unicode),
"attachment %r must be unicode" % name)
def test_non_ascii_payload(self):
"""Scrubber must handle non-ascii messages"""
for enc in ["utf8", "iso8859"]:
with open(get_test_file("payload-%s.txt" % enc)) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertTrue(isinstance(contents, unicode))
self.assertEqual(contents, u'This message contains non-ascii '
u'characters:\n\xe9 \xe8 \xe7 \xe0 \xee \xef \xeb \u20ac\n')
def test_non_ascii_payload(self):
"""Scrubber must handle non-ascii messages"""
for enc in ["utf8", "iso8859"]:
with open(get_test_file("payload-%s.txt" % enc)) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertTrue(isinstance(contents, unicode))
self.assertEqual(contents, u'This message contains non-ascii '
u'characters:\n\xe9 \xe8 \xe7 \xe0 \xee \xef \xeb \u20ac\n')
def test_bad_content_type(self):
"""Scrubber must handle unknown content-types"""
with open(get_test_file("payload-unknown.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
try:
contents, attachments = scrubber.scrub()
except LookupError, e:
import traceback;
print(traceback.format_exc())
self.fail(e) # codec not found
def test_html_email_1(self):
with open(get_test_file("html-email-1.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
# HTML part
self.assertEqual(attachments[0][0:4], (2, "attachment.html", "text/html", "iso-8859-1"))
self.assertEqual(len(attachments[0][4]), 2723)
# Scrubbed content
self.assertEqual(contents, u"This is a test message\r\n" u"Non-ASCII chars: r\xe9ponse fran\xe7ais \n")
def test_bad_content_type(self):
"""Scrubber must handle unknown content-types"""
with open(get_test_file("payload-unknown.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
try:
contents, attachments = scrubber.scrub()
except LookupError, e:
import traceback
print traceback.format_exc()
self.fail(e) # codec not found
def test_attachment_insert_order(self):
"""Attachments must not be inserted in the DB before the email"""
# Re-activate foreign key support in sqlite
self.store.db._connection._raw_connection.isolation_level = 'IMMEDIATE'
self.store.db.execute("PRAGMA foreign_keys = ON")
self.store.db._connection._raw_connection.execute("PRAGMA foreign_keys = ON")
#print "*"*10, list(self.store.db.execute("PRAGMA foreign_keys"))
#self.store = get_storm_store("postgres://*****:*****@localhost/kittystore_test")
with open(get_test_file("attachment-1.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
try:
self.store.add_to_list(FakeList("example-list"), msg)
except IntegrityError, e:
self.fail(e)
def test_html_email_1(self):
with open(get_test_file("html-email-1.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
# HTML part
self.assertEqual(attachments[0][0:4],
(2, "attachment.html", "text/html", "iso-8859-1"))
self.assertEqual(len(attachments[0][4]), 2723)
# Scrubbed content
self.assertEqual(contents,
u"This is a test message\r\n"
u"Non-ASCII chars: r\xe9ponse fran\xe7ais \n")
def test_attachment_1(self):
with open(get_test_file("attachment-1.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
self.assertEqual(attachments[0],
(2, 'puntogil.vcf', 'text/x-vcard', "utf-8",
'begin:vcard\r\nfn:gil\r\nn:;gil\r\nversion:2.1\r\n'
'end:vcard\r\n\r\n'))
self.assertEqual(
contents, "This is a test message.\r\n\r\n"
"\n-- \ndevel mailing list\[email protected]\n"
"https://admin.fedoraproject.org/mailman/listinfo/devel\n")
def test_attachment_5(self):
with open(get_test_file("attachment-5.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
# text attachment
self.assertEqual(attachments[0][0:4],
(2, u"todo-djeuner.txt", "text/plain", "utf-8"))
self.assertEqual(len(attachments[0][4]), 112)
# Scrubbed content
self.assertEqual(contents, u'This is a test, HTML message with '
u'accented letters : \xe9 \xe8 \xe7 \xe0.\r\nAnd an '
u'attachment with an accented filename\r\n\r\n\r\n\r\n')
def test_attachment_3(self):
with open(get_test_file("attachment-3.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 2)
# HTML part
self.assertEqual(attachments[0][0:4], (3, "attachment.html", "text/html", "iso-8859-1"))
self.assertEqual(len(attachments[0][4]), 3134)
# Image attachment
self.assertEqual(attachments[1][0:4], (4, "GeoffreyRoucourt.jpg", "image/jpeg", None))
self.assertEqual(len(attachments[1][4]), 282180)
# Scrubbed content
self.assertEqual(contents, u"This is a test message\r\n")
def test_attachment_insert_order(self):
"""Attachments must not be inserted in the DB before the email"""
# Re-activate foreign key support in sqlite
self.store.db._connection._raw_connection.isolation_level = 'IMMEDIATE'
self.store.db.execute("PRAGMA foreign_keys = ON")
self.store.db._connection._raw_connection.execute(
"PRAGMA foreign_keys = ON")
#print "*"*10, list(self.store.db.execute("PRAGMA foreign_keys"))
#self.store = get_storm_store("postgres://*****:*****@localhost/kittystore_test")
with open(get_test_file("attachment-1.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
try:
self.store.add_to_list(FakeList("example-list"), msg)
except IntegrityError, e:
self.fail(e)
def test_attachment_5(self):
with open(get_test_file("attachment-5.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
# text attachment
self.assertEqual(attachments[0][0:4],
#(2, u"todo-déjeuner.txt", "text/plain", "utf-8"))
(2, u"attachment.bin", "text/plain", "utf-8"))
self.assertEqual(len(attachments[0][4]), 112)
# Scrubbed content
self.assertEqual(contents, u'This is a test, HTML message with '
u'accented letters : \xe9 \xe8 \xe7 \xe0.\r\nAnd an '
u'attachment with an accented filename\r\n\r\n\r\n\r\n')
def test_attachment_1(self):
with open(get_test_file("attachment-1.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
self.assertEqual(attachments[0], (
2, 'puntogil.vcf', 'text/x-vcard', "utf-8",
'begin:vcard\r\nfn:gil\r\nn:;gil\r\nversion:2.1\r\n'
'end:vcard\r\n\r\n'))
self.assertEqual(contents,
"This is a test message.\r\n\r\n"
"\n-- \ndevel mailing list\[email protected]\n"
"https://admin.fedoraproject.org/mailman/listinfo/devel\n"
)
def test_attachment_3(self):
with open(get_test_file("attachment-3.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 2)
# HTML part
self.assertEqual(attachments[0][0:4],
(3, "attachment.html", "text/html", "iso-8859-1"))
self.assertEqual(len(attachments[0][4]), 3134)
# Image attachment
self.assertEqual(attachments[1][0:4],
(4, "GeoffreyRoucourt.jpg", "image/jpeg", None))
self.assertEqual(len(attachments[1][4]), 282180)
# Scrubbed content
self.assertEqual(contents, u"This is a test message\r\n")
def test_attachment_2(self):
with open(get_test_file("attachment-2.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 1)
self.assertEqual(attachments[0], (
3, 'signature.asc', 'application/pgp-signature', None,
'-----BEGIN PGP SIGNATURE-----\r\nVersion: GnuPG v1.4.12 '
'(GNU/Linux)\r\nComment: Using GnuPG with Mozilla - '
'http://www.enigmail.net/\r\n\r\niEYEARECAAYFAlBhm3oACgkQhmBj'
'z394AnmMnQCcC+6tWcqE1dPQmIdRbLXgKGVp\r\nEeUAn2OqtaXaXaQV7rx+'
'SmOldmSzcFw4\r\n=OEJv\r\n-----END PGP SIGNATURE-----\r\n'))
self.assertEqual(contents,
u"This is a test message\r\nNon-ascii chars: Hofm\xfchlgasse\r\n"
u"\n-- \ndevel mailing list\[email protected]\n"
u"https://admin.fedoraproject.org/mailman/listinfo/devel\n"
)
def test_attachment_4(self):
with open(get_test_file("attachment-4.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
scrubber = Scrubber("*****@*****.**", msg)
contents, attachments = scrubber.scrub()
self.assertEqual(len(attachments), 2)
# HTML part
self.assertEqual(attachments[0][0:4], (3, "attachment.html", "text/html", "iso-8859-1"))
self.assertEqual(len(attachments[0][4]), 114)
# text attachment
self.assertEqual(
attachments[1][0:4],
# (4, u"todo-déjeuner.txt", "text/plain", "utf-8"))
(4, u"todo-djeuner.txt", "text/plain", "utf-8"),
)
self.assertEqual(len(attachments[1][4]), 112)
# Scrubbed content
self.assertEqual(
contents,
u"This is a test, HTML message with "
u"accented letters : \xe9 \xe8 \xe7 \xe0.\r\nAnd an "
u"attachment with an accented filename\r\n",
)
def test_ref_parsing(self):
with open(get_test_file("strange-in-reply-to-header.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
ref_id = kittystore.utils.get_ref(msg)
self.assertEqual(ref_id, "*****@*****.**")
def test_wrong_reply_to_format(self):
with open(get_test_file("wrong-in-reply-to-header.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
ref_id = kittystore.utils.get_ref(msg)
self.assertEqual(ref_id, None)
def test_ref_parsing(self):
with open(
get_test_file("strange-in-reply-to-header.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
ref_id = kittystore.utils.get_ref(msg)
self.assertEqual(ref_id, "*****@*****.**")
def test_wrong_reply_to_format(self):
with open(get_test_file("wrong-in-reply-to-header.txt")) as email_file:
msg = email.message_from_file(email_file, _class=Message)
ref_id = kittystore.utils.get_ref(msg)
self.assertEqual(ref_id, None)
