def _dispose(self, mlist, msg, msgdata):
# List isn't doing bounce processing?
if not mlist.process_bounces:
return False
# Try VERP detection first, since it's quick and easy
context = BounceContext.normal
addresses = StandardVERP().get_verp(mlist, msg)
if len(addresses) > 0:
# Scan the message to see if it contained permanent or temporary
# failures. We'll ignore temporary failures, but even if there
# are no permanent failures, we'll assume VERP bounces are
# permanent.
temporary, permanent = all_failures(msg)
if len(temporary) > 0:
# This was a temporary failure, so just ignore it.
return False
else:
# See if this was a probe message.
addresses = ProbeVERP().get_verp(mlist, msg)
if len(addresses) > 0:
context = BounceContext.probe
else:
# That didn't give us anything useful, so try the old fashion
# bounce matching modules. Since Mailman currently doesn't
# score temporary failures, if we get no permanent failures,
# we're done, but we do need to check for temporary failures
# to know if the bounce was recognized.
temporary, addresses = all_failures(msg)
if len(addresses) == 0 and len(temporary) > 0:
# This is a recognized temp fail so ignore it.
return False
# If that still didn't return us any useful addresses, then send it on
# or discard it. The addresses will come back from flufl.bounce as
# bytes/8-bit strings, but we must store them as unicodes in the
# database. Assume utf-8 encoding, but be cautious.
if len(addresses) > 0:
for address in addresses:
if isinstance(address, bytes):
try:
address = address.decode('utf-8')
except UnicodeError:
log.exception('Ignoring non-UTF-8 encoded '
'address: {}'.format(address))
continue
self._processor.register(mlist, address, msg, context)
else:
log.info('Bounce message w/no discernable addresses: %s',
msg.get('message-id', 'n/a'))
maybe_forward(mlist, msg)
# Dequeue this message.
return False
def instanciate_answer(self, lines):
answer = self.answer_class()
msgtxt = ''.join(lines)
msg = email.message_from_string(msgtxt)
temporary, permanent = all_failures(msg)
if permanent:
answer.is_bounced = True
answer.email_from = scan_message(msg).pop()
elif temporary:
raise TemporaryFailure
else:
answer.email_from = msg["From"]
the_recipient = msg["To"]
answer.subject = msg["Subject"]
answer.when = msg["Date"]
answer.message_id = msg["Message-ID"]
the_recipient = re.sub(r"\n", "", the_recipient)
regex = re.compile(r".*[\+\-](.*)@.*")
the_match = regex.match(the_recipient)
if the_match is None:
raise CouldNotFindIdentifier
answer.email_to = the_recipient
answer.outbound_message_identifier = the_match.groups()[0]
logging.info("Reading the parts")
for part in msg.walk():
logging.info("Part of type " + part.get_content_type())
content_type_attr = self.content_types_attrs.get(
part.get_content_type())
if content_type_attr:
charset = part.get_content_charset() or "ISO-8859-1"
data = part.get_payload(decode=True).decode(charset)
setattr(
answer,
content_type_attr,
EmailReplyParser.parse_reply(data).strip(),
)
else:
self.handle_not_processed_part(part)
attachment = self.parse_attachment(part)
if attachment:
answer.add_attachment(attachment)
log = 'New incoming email from %(from)s sent on %(date)s with subject %(subject)s and content %(content)s'
log = log % {
'from': answer.email_from,
'date': answer.when,
'subject': answer.subject,
'content': answer.content_text,
}
logging.info(log)
return answer
def instanciate_answer(self, lines):
answer = self.answer_class()
msgtxt = ''.join(lines)
msg = email.message_from_string(msgtxt)
temporary, permanent = all_failures(msg)
if permanent:
answer.is_bounced = True
answer.email_from = scan_message(msg).pop()
elif temporary:
raise TemporaryFailure
else:
answer.email_from = msg["From"]
the_recipient = msg["To"]
answer.subject = msg["Subject"]
answer.when = msg["Date"]
answer.message_id = msg["Message-ID"]
the_recipient = re.sub(r"\n", "", the_recipient)
regex = re.compile(r".*[\+\-](.*)@.*")
the_match = regex.match(the_recipient)
if the_match is None:
raise CouldNotFindIdentifier
answer.email_to = the_recipient
answer.outbound_message_identifier = the_match.groups()[0]
logging.info("Reading the parts")
for part in msg.walk():
logging.info("Part of type " + part.get_content_type())
content_type_attr = self.content_types_attrs.get(part.get_content_type())
if content_type_attr:
charset = part.get_content_charset() or "ISO-8859-1"
data = part.get_payload(decode=True).decode(charset)
setattr(
answer,
content_type_attr,
EmailReplyParser.parse_reply(data).strip(),
)
else:
self.handle_not_processed_part(part)
attachment = self.parse_attachment(part)
if attachment:
answer.add_attachment(attachment)
log = 'New incoming email from %(from)s sent on %(date)s with subject %(subject)s and content %(content)s'
log = log % {
'from': answer.email_from,
'date': answer.when,
'subject': answer.subject,
'content': answer.content_text,
}
logging.info(log)
return answer
def _dispose(self, mlist, msg, msgdata):
# List isn't doing bounce processing?
if not mlist.process_bounces:
return False
# Try VERP detection first, since it's quick and easy
context = BounceContext.normal
addresses = StandardVERP().get_verp(mlist, msg)
if len(addresses) > 0:
# Scan the message to see if it contained permanent or temporary
# failures. We'll ignore temporary failures, but even if there
# are no permanent failures, we'll assume VERP bounces are
# permanent.
temporary, permanent = all_failures(msg)
if len(temporary) > 0:
# This was a temporary failure, so just ignore it.
return False
else:
# See if this was a probe message.
addresses = ProbeVERP().get_verp(mlist, msg)
if len(addresses) > 0:
context = BounceContext.probe
else:
# That didn't give us anything useful, so try the old fashion
# bounce matching modules. This returns only the permanently
# failing addresses. Since Mailman currently doesn't score
# temporary failures, if we get no permanent failures, we're
# done.s
addresses = scan_message(msg)
# If that still didn't return us any useful addresses, then send it on
# or discard it. The addresses will come back from flufl.bounce as
# bytes/8-bit strings, but we must store them as unicodes in the
# database. Assume utf-8 encoding, but be cautious.
if len(addresses) > 0:
for address in addresses:
if isinstance(address, bytes):
try:
address = address.decode('utf-8')
except UnicodeError:
log.exception('Ignoring non-UTF-8 encoded '
'address: {}'.format(address))
continue
self._processor.register(mlist, address, msg, context)
else:
log.info('Bounce message w/no discernable addresses: %s',
msg.get('message-id', 'n/a'))
maybe_forward(mlist, msg)
# Dequeue this message.
return False
def handle(self, lines):
answer = self.answer_class()
msgtxt = ''.join(lines)
msg = email.message_from_string(msgtxt)
temporary, permanent = all_failures(msg)
if temporary or permanent:
answer.is_bounced = True
answer.email_from = scan_message(msg).pop()
else:
answer.email_from = msg["From"]
the_recipient = msg["To"]
answer.subject = msg["Subject"]
answer.when = msg["Date"]
the_recipient = re.sub(r"\n", "", the_recipient)
regex = re.compile(r".*[\+\-](.*)@.*")
answer.outbound_message_identifier = regex.match(the_recipient).groups()[0]
charset = msg.get_charset()
logging.info("Reading the parts")
for part in msg.walk():
logging.info("Part of type "+part.get_content_type())
if part.get_content_type() == 'text/plain':
charset = part.get_content_charset()
if not charset:
charset = "ISO-8859-1"
data = part.get_payload(decode=True).decode(charset)
text = EmailReplyParser.parse_reply(data)
text.strip()
answer.content_text = text
#logging stuff
log = 'New incoming email from %(from)s sent on %(date)s with subject %(subject)s and content %(content)s'
log = log % {
'from':answer.email_from,
'date':answer.when,
'subject':answer.subject,
'content':answer.content_text
}
logging.info(log)
return answer
def handle_message(self, body):
'''
handles soft and hard bounced emails
'''
msg = email.message_from_bytes(bytes(body))
logger.info("------------- INCOMING MESSAGE -------------")
for key, value in msg.items():
if any(key.startswith(h) for h in ['From', 'To', 'Subject']):
logger.info("%s:\t%s", key, value)
for domain in self._get_origin_to_domains(msg):
if domain in self.handler_config['domains'].keys():
break
else:
raise BouncedEmailException("Domain '%s' not found" % domain)
t, p = all_failures(msg)
def validate_addresses(bounced_addresses):
address_list = []
for address in bounced_addresses:
address = address.decode('utf-8')
if validate_email(address):
address_list.append(address)
return address_list
temporary = validate_addresses(t)
permanent = validate_addresses(p)
if not (temporary or permanent):
return self._handle_out_of_office_message(msg)
logger.info("Domain: %s", domain)
for bounced_address in temporary:
# sometimes a temporary failure is a permanent failure as well (strange, but yes)
if bounced_address in permanent:
continue
logger.info("Temporary: %s", bounced_address)
self._handle_temporary_bounced_address(bounced_address, domain,
body)
for bounced_address in permanent:
logger.info("Permanent: %s", bounced_address)
self._handle_permanent_bounced_address(bounced_address, domain,
body)
def parse_bounces(filename, mintime=None, maxtime=None):
mb = mbox(filename)
errors = set()
count = 0
for key, msg in mb.iteritems():
message = mboxMessage(msg)
msgtime = parsedate(message['Date'])
count += 1
print >> sys.stderr, '\r', "%6d" % count, \
strftime("%Y/%m/%d %H:%M:%S", msgtime),
sys.stdout.flush()
t = timegm(msgtime)
if mintime and t < mintime:
continue
if maxtime and t > maxtime:
continue
temporary, permanent = all_failures(msg)
errors.update([p for p in permanent if '@' in p])
print >> sys.stderr, ""
print >> sys.stderr, "%s issues" % len(errors)
return errors
def parse_bounces(filename, mintime=None, maxtime=None):
mb = mbox(filename)
errors = set()
count = 0
for key, msg in mb.iteritems():
message = mboxMessage(msg)
msgtime = parsedate(message['Date'])
count += 1
print >> sys.stderr, '\r', "%6d" % count, \
strftime("%Y/%m/%d %H:%M:%S", msgtime),
sys.stdout.flush()
t = timegm(msgtime)
if mintime and t < mintime:
continue
if maxtime and t > maxtime:
continue
temporary, permanent = all_failures(msg)
errors.update([p for p in permanent if '@' in p])
print >> sys.stderr, ""
print >> sys.stderr, "%s issues" % len(errors)
return errors
def handle(self, *args, **options):
user = raw_input('Email login: '******'@')[1]
passwd = getpass.getpass()
mail_server = poplib.POP3_SSL(host)
mail_server.user(user)
mail_server.pass_(passwd)
emails_to_delete = set()
num_messages = len(mail_server.list()[1])
for i in range(num_messages):
msg = '\n'.join(mail_server.retr(i+1)[1])
parsed_email = message_from_string(msg)
try:
temporary, permanent = all_failures(parsed_email)
except Exception:
self.stderr.write('Skipping message {} ({})\n'.format(parsed_email['Subject'], parsed_email['From']))
continue
if temporary:
emails_to_delete |= temporary
for email in temporary:
self.stdout.write("temporary bounce: {}\n".format(email))
if permanent:
emails_to_delete |= permanent
for email in permanent:
self.stdout.write("permanent bounce: {}\n".format(email))
self.stdout.write('Total {} emails\n'.format(len(emails_to_delete)))
if options['delete']:
queryset = Email.objects.filter(address__in=emails_to_delete)
num_deleted_emails = queryset.count()
queryset.delete()
self.stdout.write('Deleted {} emails\n'.format(num_deleted_emails))
mail_server.quit()
