def test_attachment_name_redaction(self):
request = FoiRequest.objects.get_by_secret_mail(
"*****@*****.**")
user = factories.UserFactory.create(last_name='Username')
profile = user.get_profile()
profile.private = True
profile.save()
request.user = user
request.save()
with file(p("test_mail_06.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(content)
self.assertEqual(len(mail['attachments']), 1)
self.assertEqual(
mail['attachments'][0].name,
u'Eingangsbestätigung und Hinweis auf Unzustellbarkeit - Username.pdf'
)
request.add_message_from_email(mail, content)
messages = request.foimessage_set.all()
self.assertEqual(len(messages), 2)
mes = messages[1]
self.assertEqual(
mes.attachments[0].name,
u'EingangsbesttigungundHinweisaufUnzustellbarkeit-NAME.pdf')
def test_recipient_parsing(self):
with open(p("test_mail_05.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertEqual(len(mail['cc']), 2)
self.assertEqual(len(mail['to']), 3) # contains X-Original-To as well
def _process_mail(mail_string, mail_type=None):
parser = EmailParser()
if mail_type is None:
email = parser.parse(BytesIO(mail_string))
elif mail_type == 'postmark':
email = parser.parse_postmark(json.loads(mail_string.decode('utf-8')))
return _deliver_mail(email, mail_string=mail_string)
def test_attachment_name_broken_encoding(self):
with open(p("test_mail_06.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertEqual(len(mail['attachments']), 1)
self.assertEqual(mail['attachments'][0].name, u'Eingangsbestätigung und Hinweis auf Unzustellbarkeit - Username.pdf')
def test_eml_attachments(self):
with open(p("test_mail_08.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
subject = 'WG: Disziplinarverfahren u.a. gegen Bürgermeister/Hauptverwaltungsbeamte/Amtsdirektoren/ehrenamtliche Bürgermeister/Ortsvorsteher/Landräte im Land Brandenburg in den letzten Jahren [#5617]'
self.assertEqual(mail.attachments[0].name, '%s.eml' % subject[:45])
def test_long_attachment_names(self):
request = FoiRequest.objects.get_by_secret_mail(self.secret_address)
with open(p("test_mail_04.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertEqual(
mail['subject'], 'Kooperationen des Ministerium für Schule und '
'Weiterbildung des Landes Nordrhein-Westfalen mit außerschulischen Partnern'
)
self.assertEqual(
mail['attachments'][0].name, 'Kooperationen des MSW, Antrag '
'nach Informationsfreiheitsgesetz NRW, Stefan Safario vom 06.12.2012 - AW vom '
'08.01.2013 - RS.pdf')
request.add_message_from_email(mail, content)
request = FoiRequest.objects.get_by_secret_mail(self.secret_address)
messages = request.foimessage_set.all()
self.assertEqual(len(messages), 2)
self.assertEqual(messages[1].subject, mail['subject'])
self.assertEqual(len(messages[1].attachments), 2)
names = set([a.name for a in messages[1].attachments])
self.assertEqual(
names,
set([
'KooperationendesMSWAntragnachInformationsfreiheitsgesetzNRWStefanSafariovom06.12.2012-Anlage.pdf',
"KooperationendesMSWAntragnachInformationsfreiheitsgesetzNRWStefanSafariovom06.12.2012-AWvom08.01.2013-RS.pdf"
]))
def test_attachment_name_parsing(self):
with open(p("test_mail_07.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertEqual(len(mail['attachments']), 1)
self.assertEqual(mail['attachments'][0].name, 'Bescheid Fäker.pdf')
def test_attachment_name_parsing_header(self):
with open(p("test_mail_10.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertEqual(len(mail['attachments']), 1)
self.assertEqual(mail['attachments'][0].name, 'Eingangsbestätigung Akteneinsicht.doc')
def test_long_attachment_names(self):
request = FoiRequest.objects.get_by_secret_mail(
"*****@*****.**")
with file(p("test_mail_04.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(content)
self.assertEqual(
mail['subject'], u'Kooperationen des Ministerium für Schule und '
u'Weiterbildung des Landes Nordrhein-Westfalen mit außerschulischen Partnern'
)
self.assertEqual(
mail['attachments'][0].name, u'Kooperationen des MSW, Antrag '
'nach Informationsfreiheitsgesetz NRW, Stefan Safario vom 06.12.2012 - AW vom '
'08.01.2013 - RS.pdf')
request.add_message_from_email(mail, content)
request = FoiRequest.objects.get_by_secret_mail(
"*****@*****.**")
messages = request.foimessage_set.all()
self.assertEqual(len(messages), 2)
self.assertEqual(messages[1].subject, mail['subject'])
self.assertEqual(len(messages[1].attachments), 2)
self.assertEqual(
messages[1].attachments[0].name,
u"KooperationendesMSWAntragnachInformationsfreiheitsgesetzNRWStefanSafariovom06.12.2012-AWvom08.01.2013-RS.pdf"
)
def _process_mail(mail_bytes, mail_type=None, manual=False):
parser = EmailParser()
if mail_type is None:
email = parser.parse(BytesIO(mail_bytes))
elif mail_type == 'postmark':
email = parser.parse_postmark(json.loads(mail_bytes.decode('utf-8')))
return _deliver_mail(email, mail_bytes=mail_bytes, manual=manual)
def test_attachment_name_parsing_2(self):
with open(p("test_mail_11.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertEqual(mail.subject, 'Bescheid zu Ihrer ergänzten IFG-Anfrage Bestellung Infomaterial, Broschüren... [#32154]')
self.assertEqual(len(mail.attachments), 1)
self.assertEqual(mail.attachments[0].name, '20180904_Bescheid Broschüren.pdf')
def test_recipient_parsing(self):
with open(p("test_mail_05.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertEqual(len(mail.cc), 2)
self.assertEqual(len(mail.to), 3) # contains X-Original-To as well
self.assertTrue(mail.is_auto_reply)
def deferred_fill_sender(apps, schema_editor):
parser = EmailParser()
DeferredMessage = apps.get_model('foirequest', 'DeferredMessage')
for deferred in DeferredMessage.objects.all():
print(deferred.pk)
email = parser.parse(BytesIO(base64.b64decode(deferred.mail)))
deferred.sender = email.from_[1]
deferred.save()
def test_borked_subject(self):
''' Subject completly borked '''
with open(p("test_mail_09.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertIn('Unterlagen nach', mail['subject'])
self.assertIn('E-Mail Empfangsbest', mail['subject'])
def test_attachment_name_broken_encoding(self):
with open(p("test_mail_06.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertEqual(len(mail.attachments), 2)
self.assertEqual(mail.attachments[0].name, 'usernameEingangsbestätigung und Hinweis auf Unzustellbarkeit - Username.pdf')
self.assertEqual(mail.attachments[1].name, '15-725_002 II_0367.pdf')
def test_attachment_name_parsing(self):
with open(p("test_mail_07.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertEqual(mail.subject, 'Anfrage nach dem Informationsfreiheitsgesetz; Gespräch damaliger BM Steinmeier Matthias Müller VW AG; Vg. 069-2018')
self.assertEqual(len(mail.attachments), 2)
self.assertEqual(mail.attachments[0].name, 'Bescheid Fäker.pdf')
self.assertEqual(mail.attachments[1].name, '180328 Schreiben an Antragstellerin; Vg. 069-2018.pdf')
def test_html_only_mail(self):
with open(p("test_mail_13.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertTrue(len(mail['body']) > 10)
# Markdown like links are rendered
self.assertIn('](https://', mail['body'])
self.assertIn('*[email protected]*', mail['body'])
def test_bounce_parsing_2(self):
parser = EmailParser()
with open(p("bounce_002.txt"), 'rb') as f:
email = parser.parse(f)
bounce_address = make_bounce_address(self.email)
email.to = [('', bounce_address)]
bounce_info = email.bounce_info
self.assertTrue(bounce_info.is_bounce)
self.assertEqual(bounce_info.bounce_type, 'hard')
self.assertEqual(bounce_info.status, (5, 1, 1))
def redeliver_subject(self, request, queryset):
parser = EmailParser()
for deferred in queryset:
email = parser.parse(BytesIO(deferred.encoded_mail()))
match = SUBJECT_REQUEST_ID.search(email['subject'])
if match is not None:
try:
req = FoiRequest.objects.get(pk=match.group(1))
deferred.redeliver(req)
except FoiRequest.DoesNotExist:
continue
def test_attachment_name_parsing(self):
"""
FIXME: This test is broken on Python 2 due to different header parsing
"""
with open(p("test_mail_07.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertEqual(mail['subject'], 'Anfrage nach dem Informationsfreiheitsgesetz; Gespräch damaliger BM Steinmeier Matthias Müller VW AG; Vg. 069-2018')
self.assertEqual(len(mail['attachments']), 2)
self.assertEqual(mail['attachments'][0].name, 'Bescheid Fäker.pdf')
self.assertEqual(mail['attachments'][1].name, '180328 Schreiben an Antragstellerin; Vg. 069-2018.pdf')
def test_strip_html(self):
request = FoiRequest.objects.get_by_secret_mail("*****@*****.**")
with file(p("test_mail_05.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(content)
request.add_message_from_email(mail, content)
messages = request.foimessage_set.all()
self.assertEqual(len(messages), 2)
mes = messages[1]
self.assertTrue(len(mes.plaintext_redacted) > 0)
self.assertTrue(len(mes.plaintext) > 0)
def process_bounce_mail(mail_bytes):
parser = EmailParser()
with closing(BytesIO(mail_bytes)) as stream:
email = parser.parse(stream)
bounce_info = email.bounce_info
if bounce_info.is_bounce:
add_bounce_mail(email)
else:
if email.is_auto_reply:
return
mail_managers('No bounce detected in bounce mailbox', email.subject)
def test_strip_html(self):
request = FoiRequest.objects.get_by_secret_mail(self.secret_address)
with open(p("test_mail_05.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
request.add_message_from_email(mail, content)
messages = request.foimessage_set.all()
self.assertEqual(len(messages), 2)
mes = messages[1]
self.assertTrue(len(mes.plaintext_redacted) > 0)
self.assertTrue(len(mes.plaintext) > 0)
def _process_mail(mail_string):
from .models import FoiRequest, DeferredMessage
parser = EmailParser()
email = parser.parse(mail_string)
received_list = email['to'] + email['cc'] \
+ email['resent_to'] + email['resent_cc']
# TODO: BCC?
mail_filter = lambda x: x[1].endswith("@%s" % settings.FOI_EMAIL_DOMAIN)
received_list = filter(mail_filter, received_list)
# make original mail storeable as unicode
b64_encoded = False
try:
mail_string = mail_string.decode("utf-8")
except UnicodeDecodeError:
b64_encoded = True
mail_string = base64.b64encode(mail_string).decode("utf-8")
already = set()
for received in received_list:
secret_mail = received[1]
if secret_mail in already:
continue
already.add(secret_mail)
if not secret_mail.endswith('@%s' % settings.FOI_EMAIL_DOMAIN):
continue
try:
foi_request = FoiRequest.objects.get_by_secret_mail(secret_mail)
except FoiRequest.DoesNotExist:
try:
deferred = DeferredMessage.objects.get(recipient=secret_mail,
request__isnull=False)
foi_request = deferred.request
except DeferredMessage.DoesNotExist:
if not b64_encoded:
mail_string = base64.b64encode(mail_string).decode("utf-8")
DeferredMessage.objects.create(
recipient=secret_mail,
mail=mail_string,
)
mail_managers(
_('Unknown FoI-Mail Recipient'),
unknown_foimail_message % {
'address':
secret_mail,
'url':
settings.SITE_URL +
reverse('admin:foirequest_deferredmessage_changelist')
})
continue
foi_request.add_message_from_email(email, mail_string)
def _process_mail(mail_bytes, mail_uid=None, mail_type=None, manual=False):
parser = EmailParser()
email = None
if mail_type is None:
with closing(BytesIO(mail_bytes)) as stream:
email = parser.parse(stream)
elif mail_type == 'postmark':
email = parser.parse_postmark(json.loads(mail_bytes.decode('utf-8')))
assert email is not None
_deliver_mail(email, mail_bytes=mail_bytes, manual=manual)
# Unflag mail after delivery is complete
if mail_uid is not None:
with get_foi_mail_client() as mailbox:
unflag_mail(mailbox, mail_uid)
def test_bounce_parsing(self):
parser = EmailParser()
with open(p("bounce_001.txt"), 'rb') as f:
email = parser.parse(f)
bounce_address = make_bounce_address(self.email)
email.to = [('', bounce_address)]
bounce_info = email.bounce_info
self.assertTrue(bounce_info.is_bounce)
self.assertEqual(bounce_info.bounce_type, 'hard')
self.assertEqual(bounce_info.status, (5, 5, 3))
add_bounce_mail(email)
bounce = Bounce.objects.get(email=self.email)
self.assertEqual(bounce.email, self.email)
self.assertIsNone(bounce.user)
self.assertEqual(len(bounce.bounces), 1)
def auto_redeliver(self, request, queryset):
import json
parser = EmailParser()
for deferred in queryset:
logger.info("Encoded email: {0}".format(deferred.encoded_mail()))
email = parser.parse_postmark(
json.loads(deferred.encoded_mail().decode('utf-8')))
# email = parser.parse(BytesIO(deferred.encoded_mail()))
logger.info("Attempting to parse deferred email {0}".format(email))
if 'subject' in email:
logger.info("Email subject is {0}".format(email['subject']))
match = SUBJECT_REQUEST_ID.search(email['subject'])
if match is not None:
try:
req = FoiRequest.objects.get(pk=match.group(1))
deferred.redeliver(req)
except FoiRequest.DoesNotExist:
continue
def test_attachment_name_redaction(self):
request = FoiRequest.objects.get_by_secret_mail(self.secret_address)
user = factories.UserFactory.create(last_name='Username')
user.private = True
user.save()
request.user = user
request.save()
with open(p("test_mail_06.txt"), 'rb') as f:
parser = EmailParser()
content = f.read()
mail = parser.parse(BytesIO(content))
self.assertEqual(len(mail['attachments']), 2)
self.assertEqual(mail['attachments'][0].name, 'usernameEingangsbestätigung und Hinweis auf Unzustellbarkeit - Username.pdf')
add_message_from_email(request, mail)
messages = request.foimessage_set.all()
self.assertEqual(len(messages), 2)
mes = messages[1]
self.assertIn('NAMEEingangsbesttigungundHinweisaufUnzustellbarkeit-NAME.pdf', {a.name for a in mes.attachments})
def process_unsubscribe_mail(mail_bytes):
parser = EmailParser()
with closing(BytesIO(mail_bytes)) as stream:
email = parser.parse(stream)
recipient_list = list(set([
get_recipient_address_from_unsubscribe(addr) for name, addr in email.to
]))
if len(recipient_list) != 1:
return
recipient, status = recipient_list[0]
if not status:
return
subject = email.subject
if not subject.startswith(UNSUBSCRIBE_PREFIX):
return
_, reference = subject.split(UNSUBSCRIBE_PREFIX, 1)
email_unsubscribed.send(
sender=None, email=recipient, reference=reference
)
def test_inline_attachments(self):
parser = EmailParser()
with open(p("test_mail_03.txt"), 'rb') as f:
email = parser.parse(f)
self.assertEqual(len(email['attachments']), 1)