def _get_detached_message_for_person(self, sender):
# Return a signed message that contains a detached signature.
body = dedent("""\
This is a multi-line body.
Sincerely,
Your friendly tester.""")
to = self.factory.getUniqueEmailAddress()
msg = MIMEMultipart()
msg['Message-Id'] = make_msgid('launchpad')
msg['Date'] = formatdate()
msg['To'] = to
msg['From'] = sender.preferredemail.email
msg['Subject'] = 'Sample'
body_text = MIMEText(body)
msg.attach(body_text)
# A detached signature is calculated on the entire string content of
# the body message part.
key = import_secret_test_key()
gpghandler = getUtility(IGPGHandler)
signature = gpghandler.signContent(
canonicalise_line_endings(body_text.as_string()), key.fingerprint,
'test', gpgme.SIG_MODE_DETACH)
attachment = Message()
attachment.set_payload(signature)
attachment['Content-Type'] = 'application/pgp-signature'
msg.attach(attachment)
self.assertTrue(msg.is_multipart())
return signed_message_from_string(msg.as_string())
def test_dkim_signed_but_from_unverified_address(self):
"""Sent from trusted dkim address, but only the From address is known.
The sender is a known, but unverified address.
See https://bugs.launchpad.net/launchpad/+bug/925597
"""
from_address = "*****@*****.**"
sender_address = "*****@*****.**"
person = self.factory.makePerson(
email=from_address,
name='dkimtest',
displayname='DKIM Test')
self.factory.makeEmail(sender_address, person, EmailAddressStatus.NEW)
self.preload_dns_response()
tweaked_message = self.makeMessageText(
sender=sender_address,
from_address="DKIM Test <*****@*****.**>")
signed_message = self.fake_signing(tweaked_message)
principal = authenticateEmail(
signed_message_from_string(signed_message))
self.assertEqual(principal.person.preferredemail.email,
from_address)
self.assertWeaklyAuthenticated(principal, signed_message)
self.assertDkimLogContains(
'valid dkim signature, but not from an active email address')
def read_test_message(filename):
"""Reads a test message and returns it as ISignedMessage.
The test messages are located in lp/services/mail/tests/emails
"""
message_string = open(os.path.join(testmails_path, filename)).read()
return signed_message_from_string(message_string)
def _get_detached_message_for_person(self, sender):
# Return a signed message that contains a detached signature.
body = dedent("""\
This is a multi-line body.
Sincerely,
Your friendly tester.""")
to = self.factory.getUniqueEmailAddress()
msg = MIMEMultipart()
msg['Message-Id'] = make_msgid('launchpad')
msg['Date'] = formatdate()
msg['To'] = to
msg['From'] = sender.preferredemail.email
msg['Subject'] = 'Sample'
body_text = MIMEText(body)
msg.attach(body_text)
# A detached signature is calculated on the entire string content of
# the body message part.
key = import_secret_test_key()
gpghandler = getUtility(IGPGHandler)
signature = gpghandler.signContent(
canonicalise_line_endings(body_text.as_string()),
key.fingerprint, 'test', gpgme.SIG_MODE_DETACH)
attachment = Message()
attachment.set_payload(signature)
attachment['Content-Type'] = 'application/pgp-signature'
msg.attach(attachment)
self.assertTrue(msg.is_multipart())
return signed_message_from_string(msg.as_string())
def read_test_message(filename):
"""Reads a test message and returns it as ISignedMessage.
The test messages are located in lp/services/mail/tests/emails
"""
message_string = open(os.path.join(testmails_path, filename)).read()
return signed_message_from_string(message_string)
def test_dkim_valid(self):
signed_message = self.fake_signing(self.makeMessageText())
self.preload_dns_response()
principal = authenticateEmail(
signed_message_from_string(signed_message))
self.assertStronglyAuthenticated(principal, signed_message)
self.assertEqual(principal.person.preferredemail.email,
'*****@*****.**')
def test_dkim_garbage_pubkey(self):
signed_message = self.fake_signing(self.makeMessageText())
self.preload_dns_response('garbage')
principal = authenticateEmail(
signed_message_from_string(signed_message))
self.assertWeaklyAuthenticated(principal, signed_message)
self.assertEqual(principal.person.preferredemail.email,
'*****@*****.**')
self.assertDkimLogContains('invalid format in _domainkey txt record')
def test_dkim_nxdomain(self):
# If there's no DNS entry for the pubkey it should be handled
# decently.
signed_message = self.fake_signing(self.makeMessageText())
principal = authenticateEmail(
signed_message_from_string(signed_message))
self.assertWeaklyAuthenticated(principal, signed_message)
self.assertEqual(principal.person.preferredemail.email,
'*****@*****.**')
def test_dkim_message_unsigned(self):
# This is a degenerate case: a message with no signature is
# treated as weakly authenticated.
# The library doesn't log anything if there's no header at all.
principal = authenticateEmail(
signed_message_from_string(self.makeMessageText()))
self.assertWeaklyAuthenticated(principal, self.makeMessageText())
self.assertEqual(principal.person.preferredemail.email,
'*****@*****.**')
def test_unsigned_message(self):
# An unsigned message will not have a signature nor signed content,
# and generates a weakly authenticated principle.
sender = self.factory.makePerson()
email_message = self.factory.makeEmailMessage(sender=sender)
msg = signed_message_from_string(email_message.as_string())
self.assertIs(None, msg.signedContent)
self.assertIs(None, msg.signature)
principle = authenticateEmail(msg)
self.assertEqual(sender, principle.person)
self.assertTrue(IWeaklyAuthenticatedPrincipal.providedBy(principle))
self.assertIs(None, msg.signature)
def test_dkim_changed_from_realname(self):
# If the real name part of the message has changed, it's detected.
signed_message = self.fake_signing(self.makeMessageText())
self.preload_dns_response()
fiddled_message = signed_message.replace(
'From: Foo Bar <*****@*****.**>',
'From: Evil Foo <*****@*****.**>')
principal = authenticateEmail(
signed_message_from_string(fiddled_message))
# We don't care about the real name for determining the principal.
self.assertWeaklyAuthenticated(principal, fiddled_message)
self.assertEqual(principal.person.preferredemail.email,
'*****@*****.**')
def test_dkim_body_mismatch(self):
# The message has a syntactically valid DKIM signature that
# doesn't actually correspond to what was signed. We log
# something about this but we don't want to drop the message.
signed_message = self.fake_signing(self.makeMessageText())
signed_message += 'blah blah'
self.preload_dns_response()
principal = authenticateEmail(
signed_message_from_string(signed_message))
self.assertWeaklyAuthenticated(principal, signed_message)
self.assertEqual(principal.person.preferredemail.email,
'*****@*****.**')
self.assertDkimLogContains('body hash mismatch')
def test_unsigned_message(self):
# An unsigned message will not have a signature nor signed content,
# and generates a weakly authenticated principle.
sender = self.factory.makePerson()
email_message = self.factory.makeEmailMessage(sender=sender)
msg = signed_message_from_string(email_message.as_string())
self.assertIs(None, msg.signedContent)
self.assertIs(None, msg.signature)
principle = authenticateEmail(msg)
self.assertEqual(sender, principle.person)
self.assertTrue(
IWeaklyAuthenticatedPrincipal.providedBy(principle))
self.assertIs(None, msg.signature)
def test_dkim_broken_pubkey(self):
"""Handle a subtly-broken pubkey like qq.com, see bug 881237.
The message is not trusted but inbound message processing does not
abort either.
"""
signed_message = self.fake_signing(self.makeMessageText())
self.preload_dns_response('broken')
principal = authenticateEmail(
signed_message_from_string(signed_message))
self.assertWeaklyAuthenticated(principal, signed_message)
self.assertEqual(principal.person.preferredemail.email,
'*****@*****.**')
self.assertDkimLogContains('unexpected error in DKIM verification')
def test_dkim_signing_irrelevant(self):
# It's totally valid for a message to be signed by a domain other than
# that of the From-sender, if that domain is relaying the message.
# However, we shouldn't then trust the purported sender, because they
# might have just made it up rather than relayed it.
tweaked_message = self.makeMessageText(
from_address='*****@*****.**')
signed_message = self.fake_signing(tweaked_message)
self.preload_dns_response()
principal = authenticateEmail(
signed_message_from_string(signed_message))
self.assertWeaklyAuthenticated(principal, signed_message)
# should come from From, not the dkim signature
self.assertEqual(principal.person.preferredemail.email,
'*****@*****.**')
def test_dkim_changed_from_address(self):
# If the address part of the message has changed, it's detected.
# We still treat this as weakly authenticated by the purported
# From-header sender, though perhaps in future we would prefer
# to reject these messages.
signed_message = self.fake_signing(self.makeMessageText())
self.preload_dns_response()
fiddled_message = signed_message.replace(
'From: Foo Bar <*****@*****.**>',
'From: Carlos <*****@*****.**>')
principal = authenticateEmail(
signed_message_from_string(fiddled_message))
self.assertWeaklyAuthenticated(principal, fiddled_message)
# should come from From, not the dkim signature
self.assertEqual(principal.person.preferredemail.email,
'*****@*****.**')
def test_dkim_untrusted_signer(self):
# Valid signature from an untrusted domain -> untrusted
signed_message = self.fake_signing(self.makeMessageText())
self.preload_dns_response()
saved_domains = incoming._trusted_dkim_domains[:]
def restore():
incoming._trusted_dkim_domains = saved_domains
self.addCleanup(restore)
incoming._trusted_dkim_domains = []
principal = authenticateEmail(
signed_message_from_string(signed_message))
self.assertWeaklyAuthenticated(principal, signed_message)
self.assertEqual(principal.person.preferredemail.email,
'*****@*****.**')
def _get_clearsigned_for_person(self, sender, body=None):
# Create a signed message for the sender specified with the test
# secret key.
key = import_secret_test_key()
signing_context = GPGSigningContext(key.fingerprint, password='******')
if body is None:
body = dedent("""\
This is a multi-line body.
Sincerely,
Your friendly tester.
""")
msg = self.factory.makeSignedMessage(
email_address=sender.preferredemail.email,
body=body, signing_context=signing_context)
self.assertFalse(msg.is_multipart())
return signed_message_from_string(msg.as_string())
def _get_clearsigned_for_person(self, sender, body=None):
# Create a signed message for the sender specified with the test
# secret key.
key = import_secret_test_key()
signing_context = GPGSigningContext(key.fingerprint, password='******')
if body is None:
body = dedent("""\
This is a multi-line body.
Sincerely,
Your friendly tester.
""")
msg = self.factory.makeSignedMessage(
email_address=sender.preferredemail.email,
body=body,
signing_context=signing_context)
self.assertFalse(msg.is_multipart())
return signed_message_from_string(msg.as_string())
def test_require_strong_email_authentication_and_unsigned(self):
sender = getUtility(IPersonSet).getByEmail('*****@*****.**')
sender.require_strong_email_authentication = True
email_message = self.factory.makeEmailMessage(sender=sender)
msg = signed_message_from_string(email_message.as_string())
error = self.assertRaises(IncomingEmailError, authenticateEmail, msg)
expected_message = (
"Launchpad only accepts signed email from your address.\n\n"
"If you want to use the Launchpad email interface, you will need "
"to go here\n"
"to import your OpenPGP key and then use it to sign your "
"messages:\n\n"
" http://launchpad.dev/~%s/+editpgpkeys\n\n"
"If you did not knowingly send email to Launchpad, then spammers "
"may be\n"
"forging messages as if they were sent from your address, perhaps "
"due to\n"
"a compromised address book, and you can safely ignore this "
"message.\n" % sender.name)
self.assertEqual(expected_message, error.message)
def test_dkim_signed_by_other_address(self):
# If the message is From one of a person's addresses, and the Sender
# corresponds to another, and there is a DKIM signature for the Sender
# domain, this is valid - see bug 643223. For this to be a worthwhile
# test we need the two addresses to be in different domains. It
# will be signed by canonical.com, so make that the sender.
person = self.factory.makePerson(
email='*****@*****.**',
name='dkimtest',
displayname='DKIM Test')
self.factory.makeEmail(
person=person,
address='*****@*****.**')
self.preload_dns_response()
tweaked_message = self.makeMessageText(
sender="*****@*****.**",
from_address="DKIM Test <*****@*****.**>")
signed_message = self.fake_signing(tweaked_message)
principal = authenticateEmail(
signed_message_from_string(signed_message))
self.assertStronglyAuthenticated(principal, signed_message)
def main(self):
self.txn.begin()
# NB: This somewhat duplicates handleMail, but there it's mixed in
# with handling a mailbox, which we're avoiding here.
if len(self.args) >= 1:
from_file = file(self.args[0], 'rb')
else:
from_file = sys.stdin
self.logger.debug("reading message from %r" % (from_file,))
raw_mail = from_file.read()
self.logger.debug("got %d bytes" % len(raw_mail))
file_alias = save_mail_to_librarian(raw_mail)
self.logger.debug("saved to librarian as %r" % (file_alias,))
parsed_mail = signed_message_from_string(raw_mail)
# Kinda kludgey way to cause sendmail to just print it.
config.sendmail_to_stdout = True
handle_one_mail(
self.logger, parsed_mail,
file_alias, file_alias.http_url,
signature_timestamp_checker=None)
self.logger.debug("mail handling complete")
self.txn.commit()
def main(self):
self.txn.begin()
# NB: This somewhat duplicates handleMail, but there it's mixed in
# with handling a mailbox, which we're avoiding here.
if len(self.args) >= 1:
from_file = file(self.args[0], 'rb')
else:
from_file = sys.stdin
self.logger.debug("reading message from %r" % (from_file, ))
raw_mail = from_file.read()
self.logger.debug("got %d bytes" % len(raw_mail))
file_alias = save_mail_to_librarian(raw_mail)
self.logger.debug("saved to librarian as %r" % (file_alias, ))
parsed_mail = signed_message_from_string(raw_mail)
# Kinda kludgey way to cause sendmail to just print it.
config.sendmail_to_stdout = True
handle_one_mail(self.logger,
parsed_mail,
file_alias,
file_alias.http_url,
signature_timestamp_checker=None)
self.logger.debug("mail handling complete")
self.txn.commit()
def test_dkim_signed_from_person_without_account(self):
"""You can have a person with no account.
We don't accept mail from them.
See https://bugs.launchpad.net/launchpad/+bug/925597
"""
from_address = "*****@*****.**"
# This is not quite the same as having account=None, but it seems as
# close as the factory lets us get? -- mbp 2012-04-13
self.factory.makePerson(
email=from_address,
name='dkimtest',
displayname='DKIM Test',
account_status=AccountStatus.NOACCOUNT)
self.preload_dns_response()
message_text = self.makeMessageText(
sender=from_address,
from_address=from_address)
signed_message = self.fake_signing(message_text)
self.assertRaises(
InactiveAccount,
authenticateEmail,
signed_message_from_string(signed_message))
def handleMail(trans=transaction, signature_timestamp_checker=None):
log = logging.getLogger('process-mail')
mailbox = getUtility(IMailBox)
log.info("Opening the mail box.")
mailbox.open()
try:
for mail_id, raw_mail in mailbox.items():
log.info("Processing mail %s" % mail_id)
trans.begin()
try:
file_alias = save_mail_to_librarian(raw_mail)
# Let's save the url of the file alias, otherwise we might not
# be able to access it later if we get a DB exception.
file_alias_url = file_alias.http_url
log.debug('Uploaded mail to librarian %s' % (file_alias_url, ))
# If something goes wrong when handling the mail, the
# transaction will be aborted. Therefore we need to commit the
# transaction now, to ensure that the mail gets stored in the
# Librarian.
trans.commit()
except UploadFailed:
# Something went wrong in the Librarian. It could be that it's
# not running, but not necessarily. Log the error and skip the
# message, but don't delete it: retrying might help.
log.exception('Upload to Librarian failed')
continue
try:
mail = signed_message_from_string(raw_mail)
except email.errors.MessageError:
# If we can't parse the message, we can't send a reply back to
# the user, but logging an exception will let us investigate.
log.exception("Couldn't convert email to email.message: %s" %
(file_alias_url, ))
mailbox.delete(mail_id)
continue
try:
trans.begin()
handle_one_mail(log, mail, file_alias, file_alias_url,
signature_timestamp_checker)
trans.commit()
mailbox.delete(mail_id)
except (KeyboardInterrupt, SystemExit):
raise
except:
# This bare except is needed in order to prevent any bug
# in the email handling from causing the email interface
# to lock up. We simply log the error, then send an oops, and
# continue through the mailbox, so that it doesn't stop the
# rest of the emails from being processed.
log.exception(
"An exception was raised inside the handler:\n%s" %
(file_alias_url, ))
# Delete the troublesome email before attempting to send the
# OOPS in case something goes wrong. Retrying probably
# wouldn't work and we'd get stuck on the bad message.
mailbox.delete(mail_id)
_send_email_oops(trans, log, mail, "Unhandled exception",
file_alias_url)
finally:
log.info("Closing the mail box.")
mailbox.close()
def getOriginalEmail(self):
"""See `ICodeReviewComment`."""
if self.message.raw is None:
return None
return signed_message_from_string(self.message.raw.read())
def handleMail(trans=transaction, signature_timestamp_checker=None):
log = logging.getLogger('process-mail')
mailbox = getUtility(IMailBox)
log.info("Opening the mail box.")
mailbox.open()
try:
for mail_id, raw_mail in mailbox.items():
log.info("Processing mail %s" % mail_id)
trans.begin()
try:
file_alias = save_mail_to_librarian(raw_mail)
# Let's save the url of the file alias, otherwise we might not
# be able to access it later if we get a DB exception.
file_alias_url = file_alias.http_url
log.debug('Uploaded mail to librarian %s' % (file_alias_url,))
# If something goes wrong when handling the mail, the
# transaction will be aborted. Therefore we need to commit the
# transaction now, to ensure that the mail gets stored in the
# Librarian.
trans.commit()
except UploadFailed:
# Something went wrong in the Librarian. It could be that it's
# not running, but not necessarily. Log the error and skip the
# message, but don't delete it: retrying might help.
log.exception('Upload to Librarian failed')
continue
try:
mail = signed_message_from_string(raw_mail)
except email.Errors.MessageError:
# If we can't parse the message, we can't send a reply back to
# the user, but logging an exception will let us investigate.
log.exception(
"Couldn't convert email to email.Message: %s" % (
file_alias_url, ))
mailbox.delete(mail_id)
continue
try:
trans.begin()
handle_one_mail(log, mail, file_alias, file_alias_url,
signature_timestamp_checker)
trans.commit()
mailbox.delete(mail_id)
except (KeyboardInterrupt, SystemExit):
raise
except:
# This bare except is needed in order to prevent any bug
# in the email handling from causing the email interface
# to lock up. We simply log the error, then send an oops, and
# continue through the mailbox, so that it doesn't stop the
# rest of the emails from being processed.
log.exception(
"An exception was raised inside the handler:\n%s"
% (file_alias_url,))
# Delete the troublesome email before attempting to send the
# OOPS in case something goes wrong. Retrying probably
# wouldn't work and we'd get stuck on the bad message.
mailbox.delete(mail_id)
_send_email_oops(trans, log, mail,
"Unhandled exception", file_alias_url)
finally:
log.info("Closing the mail box.")
mailbox.close()
def getOriginalEmail(self):
"""See `ICodeReviewComment`."""
if self.message.raw is None:
return None
return signed_message_from_string(self.message.raw.read())
