def process_maildir(d, dosa=1, pats=None):
parser = Parser()
for fn in glob.glob(os.path.join(d, "cur", "*")):
print ("reading from %s..." % fn),
file = open(fn)
msg = parser.parse(file)
process_message(msg, dosa, pats)
tmpfn = os.path.join(d, "tmp", os.path.basename(fn))
tmpfile = open(tmpfn, "w")
print "writing to %s" % tmpfn
gen = email.Generator.Generator(tmpfile, maxheaderlen=0)
gen.flatten(msg, unixfrom=0)
os.rename(tmpfn, fn)
def create_message(corpus, data, msg0=None):
from email import Charset, Parser, MIMEMessage
if not isinstance(data, unicode):
raise TypeError('data must be a unicode.')
p = Parser.FeedParser()
p.feed(data)
msg1 = p.close()
csmap = Charset.Charset(config.MESSAGE_CHARSET)
attach_msgs = []
# Re-encode the headers.
headers = []
labels = []
for (k, v) in msg1.items():
v = rmsp(v)
if not v: continue
kl = k.lower()
if kl == 'x-forward-msg':
attach_msgs.extend(get_numbers(v))
continue
if kl == 'label':
labels = v.split(',')
continue
headers.append((k.encode('ascii', 'strict'), encode_header(v, csmap)))
# Remove all the existing headers.
for k in msg1.keys():
del msg1[k]
# Reattach the headers.
for (k, v) in headers:
msg1[k] = v
# Change the body.
data = msg1.get_payload(decode=False)
try:
# First try to encode with us-ascii.
data.encode('ascii', 'strict')
# Succeed.
msg1.set_charset('ascii')
except UnicodeError:
# Re-encode the body.
if not csmap.output_charset:
csmap = Charset.Charset('utf-8')
msg1.set_charset(str(csmap.output_charset))
data = data.encode(str(csmap.output_codec), 'replace')
msg1.set_payload(data)
# Attach other messages (for forwarding).
if attach_msgs:
for loc in attach_msgs:
p = Parser.FeedParser()
p.feed(corpus.get_message(loc))
msg1 = mime_add(msg1, MIMEMessage.MIMEMessage(p.close()))
# Integrate other mime objects.
if msg0 and msg0.is_multipart():
for obj in msg0.get_payload()[1:]:
msg1 = mime_add(msg1, obj)
validate_message_structure(msg1)
return (msg1, labels)
def process_maildir(d, dosa=1, pats=None):
parser = Parser()
for fn in glob.glob(os.path.join(d, "cur", "*")):
print("reading from %s..." % fn),
file = open(fn)
msg = parser.parse(file)
process_message(msg, dosa, pats)
tmpfn = os.path.join(d, "tmp", os.path.basename(fn))
tmpfile = open(tmpfn, "w")
print "writing to %s" % tmpfn
gen = email.Generator.Generator(tmpfile, maxheaderlen=0)
gen.flatten(msg, unixfrom=0)
os.rename(tmpfn, fn)
def _createBounceMessage(self, log, toAddress, msg):
"""
Create a multipart MIME message that will be sent to the user to indicate
that their message has bounced.
@param log: ???
@param toAddress: The email address that bounced
@param msg: The message that bounced
@return: L{MP.MIMEMultipart}
"""
bounceText = (
'Your message to %(recipient)s, subject "%(subject)s", '
'could not be delivered.')
bounceText %= {
'recipient': toAddress,
'subject': msg.impl.getHeader(u'subject')}
original = P.Parser().parse(msg.impl.source.open())
m = MMP.MIMEMultipart(
'mixed',
None,
[MT.MIMEText(bounceText, 'plain'),
MT.MIMEText(log, 'plain'),
MM.MIMEMessage(original)])
m['Subject'] = 'Unable to deliver message to ' + toAddress
m['From'] = '<>'
m['To'] = ''
return m
def test_createMessageWrapsLines(self):
"""
Ensure that the text of an outgoing message is wrapped to 78
characters and that its MIME type is 'text/plain; format=flowed'.
"""
self.cf._sendOrSave(fromAddress=self.defaultFromAddr,
toAddresses=[
mimeutil.EmailAddress(u'[email protected]',
mimeEncoded=False)
],
subject=u'The subject of the message.',
messageBody=u' '.join([u'some words'] * 1000),
cc=[],
bcc=[],
files=[],
draft=False)
msg = self.userStore.findUnique(
smtpout.DeliveryToAddress)._getMessageSource()
textPart = Parser.Parser().parse(msg)
self.assertEqual(textPart.get_content_type(), "text/plain")
self.assertEqual(textPart.get_param("format"), "flowed")
body = textPart.get_payload().decode("quoted-printable")
maxLineLength = max([len(line) for line in body.split("\n")])
self.failIf(maxLineLength > 78)
def createRedirectedMessage(self, fromAddress, toAddresses, message):
"""
Create a L{Message} item based on C{message}, with the C{Resent-From}
and C{Resent-To} headers set
@type fromAddress: L{smtpout.FromAddress}
@type toAddresses: sequence of L{mimeutil.EmailAddress}
@type message: L{Message}
@rtype: L{Message}
"""
m = P.Parser().parse(message.impl.source.open())
def insertResentHeaders(i):
m._headers.insert(i, ('resent-from', MH.Header(
fromAddress.address).encode()))
m._headers.insert(i, ('resent-to', MH.Header(
mimeutil.flattenEmailAddresses(toAddresses)).encode()))
m._headers.insert(i, ('resent-date', EU.formatdate()))
m._headers.insert(i, ('resent-message-id',
smtp.messageid('divmod.xquotient')))
for (i, (name, _)) in enumerate(m._headers):
#insert Resent-* headers after all Received headers but
#before the rest
if name.lower() != "received":
insertResentHeaders(i)
break
else:
insertResentHeaders(0)
s = S.StringIO()
G.Generator(s).flatten(m)
s.seek(0)
return self.createMessageAndQueueIt(fromAddress.address, s, True)
def test_redirectHeaderOrdering(self):
"""
Test that Resent headers get added after Received headers but
before the rest.
"""
msgtext = """\
Received: from bob by example.com with smtp (SuparMTA 9.99)
id 1BfraZ-0001D0-QA
for [email protected]; Thu, 01 Jul 2004 02:46:15 +0000
received: from bob by example.com with smtp (SuparMTA 9.99)
id 1BfraZ-0001D0-QA
for [email protected]; Thu, 01 Jul 2004 02:46:17 +0000
From: <*****@*****.**>
To: <*****@*****.**>
Subject: Hi
Hi
""".replace('\n', '\r\n')
class StubMsgFile:
def open(self):
return StringIO(msgtext)
message = StubStoredMessageAndImplAndSource(store=self.userStore)
message.__dict__['_source'] = StubMsgFile()
msg = self.composer.createRedirectedMessage(
self.defaultFromAddr,
[mimeutil.EmailAddress(u'testuser@localhost', mimeEncoded=False)],
message)
m = Parser.Parser().parse(msg.impl.source.open())
self.assertEqual(len(m._headers), 9)
self.assertEqual(m._headers[0][0].lower(), "received")
self.assertEqual(m._headers[6][0].lower(), "from")
def __init__(self, fromlines=None, fromstring=None, fromfile=None):
#self.log = Logger()
self.recipient = None
self.received_by = None
self.received_from = None
self.received_with = None
self.__raw = None
try:
parser = Parser.BytesParser()
parsestr = parser.parsebytes
except:
parser = Parser.Parser()
parsestr = parser.parsestr
# Message is instantiated with fromlines for POP3, fromstring for
# IMAP (both of which can be badly-corrupted or invalid, i.e. spam,
# MS worms, etc). It's instantiated with fromfile for the output
# of filters, etc, which should be saner.
if fromlines:
try:
self.__msg = parsestr(_NL.join(fromlines))
except Errors.MessageError as o:
self.__msg = corrupt_message(o, fromlines=fromlines)
self.__raw = _NL.join(fromlines)
elif fromstring:
try:
self.__msg = parsestr(fromstring)
except Errors.MessageError as o:
self.__msg = corrupt_message(o, fromstring=fromstring)
self.__raw = fromstring
elif fromfile:
try:
self.__msg = parser.parse(fromfile)
except Errors.MessageError as o:
# Shouldn't happen
self.__msg = corrupt_message(o, fromstring=fromfile.read())
# fromfile is only used by getmail_maildir, getmail_mbox, and
# from reading the output of a filter. Ignore __raw here.
else:
# Can't happen?
raise SystemExit('Message() called with wrong arguments')
self.sender = address_no_brackets(self.__msg['Return-Path']
or self.__msg['Sender']
or 'unknown')
def test_createMessageHonorsSmarthostFromAddress(self):
"""
Sending a message through the Compose UI should honor the from
address we give to it
"""
self.defaultFromAddr.address = u'*****@*****.**'
msg = createMessage(
self.composer, self.cabinet, None, self.defaultFromAddr,
[mimeutil.EmailAddress('*****@*****.**', mimeEncoded=False)],
u'Sup dood', u'A body', (), (), u'')
file = msg.impl.source.open()
msg = Parser.Parser().parse(file)
self.assertEquals(msg["from"], '*****@*****.**')
def _fileItemToEmailPart(fileItem):
"""
Convert a L{File} item into an appropriate MIME part object
understandable by the stdlib's C{email} package
"""
(majorType, minorType) = fileItem.type.split('/')
if majorType == 'multipart':
part = P.Parser().parse(fileItem.body.open())
else:
part = MB.MIMEBase(majorType, minorType)
if majorType == 'message':
part.set_payload([P.Parser().parse(fileItem.body.open())])
else:
part.set_payload(fileItem.body.getContent())
if majorType == 'text':
EE.encode_quopri(part)
else:
EE.encode_base64(part)
part.add_header('content-disposition',
'attachment',
filename=fileItem.name)
return part
def test_createRedirectedMessage(self):
"""
Test that L{compose.Composer.createRedirectedMessage} sets the right
headers
"""
message = StubStoredMessageAndImplAndSource(store=self.userStore)
msg = self.composer.createRedirectedMessage(
self.defaultFromAddr,
[mimeutil.EmailAddress(u'testuser@localhost', mimeEncoded=False)],
message)
m = Parser.Parser().parse(msg.impl.source.open())
self.assertEquals(m['Resent-To'], 'testuser@localhost')
self.assertEquals(m['Resent-From'], self.defaultFromAddr.address)
self.failIfEqual(m['Resent-Date'], None)
self.failIfEqual(m['Resent-Message-ID'], None)
def read_messages(self):
from email import Parser
for fname in os.listdir(self.dirname):
if fname.startswith('.'): continue
fname = os.path.join(self.dirname, fname)
fp = file(fname, 'rb')
p = Parser.FeedParser()
for line in fp:
p.feed(line)
if not line.strip(): break
msg = p.close()
if self.ruleset:
labels = self.ruleset.apply_msg(msg)
else:
labels = []
self.msgs.append((fname, labels, get_message_date(msg)))
return
def finish(self):
import poplib
from email import Parser
for i in xrange(self.msgcount):
try:
data = self.server.retr(i+1)[1]
except poplib.error_proto, e:
raise MessagePOP3Error(str(e))
data = '\r\n'.join(data)
p = Parser.FeedParser()
p.feed(data)
msg = p.close()
if self.ruleset:
labels = self.ruleset.apply_msg(msg)
else:
labels = []
yield (data, labels, get_message_date(msg))
def finish(self):
from mailbox import PortableUnixMailbox
from email import Parser
try:
fp = file(self.fname, 'rb')
except IOError:
return
for data in PortableUnixMailbox(fp, lambda msgfp: msgfp.read()):
p = Parser.FeedParser()
p.feed(data)
msg = p.close()
if self.ruleset:
labels = self.ruleset.apply_msg(msg)
else:
labels = []
yield (data, labels, get_message_date(msg))
fp.close()
return
def test_redirect(self):
"""
Test L{compose.Composer.redirect}
"""
message = StubStoredMessageAndImplAndSource(store=self.userStore)
msg = self.composer.redirect(
self.defaultFromAddr,
[mimeutil.EmailAddress(u'testuser@localhost', mimeEncoded=False)],
message)
self.assertEquals(str(self.reactor.factory.fromEmail),
self.defaultFromAddr.address)
self.assertEquals(list(self.reactor.factory.toEmail),
['testuser@localhost'])
m = Parser.Parser().parse(
self.userStore.findUnique(exmess.Message).impl.source.open())
self.assertEquals(m['Resent-From'], self.defaultFromAddr.address)
self.assertEquals(m['Resent-To'], 'testuser@localhost')
self.assertEquals(message.statuses, [exmess.REDIRECTED_STATUS])
def process_message(self, peer, mailfrom, rcpttos, data):
ipaddr = peer[0]
print "Message sent from %s" % ipaddr
if ipaddr[0:3] == localNetwork:
#Parse the message into an email.Message instance
p = Parser.Parser()
m = p.parsestr(data)
print "Received Message"
body = m.get_payload(decode=1)
toAddr = m['To']
if string.find(toAddr, '<') > 0:
toAddr = re.search("<(?P<email>.+)>", toAddr).group('email')
status, encMessage = gnupg.encryptMessage(body, toAddr)
if status == 1:
m.set_payload(encMessage)
self.sendEncMessage(m)
status = ''
else:
print "Error in encrypting message"
else:
print "Not authorised smtp user"
status = "Not authorised smtp user"
return status
def process_mailbox(f, dosa=1, pats=None):
gen = email.Generator.Generator(sys.stdout, maxheaderlen=0)
for msg in mailbox.PortableUnixMailbox(f, Parser().parse):
process_message(msg, dosa, pats)
gen.flatten(msg, unixfrom=1)
def _post(self):
if self.request.body:
message = self.request.body
else:
message = self.get_argument('message')
parser = Parser.Parser()
msg = parser.parsestr(message)
## Check that it's sent to the right address
email_tos = parse_email_line(msg['To'])
email_reminder = None
# need to turn then reply to address into a regex
reply_to_parts = EMAIL_REMINDER_SENDER % dict(id='XXX')
reply_to_parts = reply_to_parts.split('XXX')
assert len(reply_to_parts) == 2
regex = re.compile(re.escape(reply_to_parts[0]) + '(\w+)' \
+ re.escape(reply_to_parts[1]), re.I)
for email_to in email_tos:
if regex.findall(email_to):
_id = regex.findall(email_to)[0]
try:
email_reminder = self.db.EmailReminder\
.one({'_id': ObjectId(_id)})
except InvalidId:
pass
## Check that it was sent from someone we know
from_user = None
for from_email in parse_email_line(msg['From']):
from_user = self.find_user(from_email)
message_body = None
character_set = 'utf-8'
if msg.is_multipart():
for part in msg.walk():
if part.is_multipart():
continue
name = part.get_param('name')
if not part.get_param('name') and part.get_content_type(
).lower() == 'text/plain':
message_body = part.get_payload(decode=True)
if isinstance(message_body, str):
for charset in part.get_charsets():
if not charset:
charset = 'utf8'
message_body = unicode(message_body, charset)
character_set = charset
break
#print "BODY"
#print repr(message_body)
#print message_body
# If it's a multipart email, only use the first part
break
else:
# not a multipart message then it's easy
message_body = msg.get_payload(decode=True)
for each in msg.get_charsets():
if each:
character_set = each
break
if character_set:
message_body = unicode(message_body, character_set)
else:
message_body = unicode(message_body, 'utf-8')
if not from_user:
# only bother to reply if the email appears to be sent from
# us originally
if 'INSTRUCTIONS' in message_body and 'DoneCal' in msg['Subject']:
self.error_reply("Not a registered account: %s" % msg['From'],
msg, message_body)
self.write("Not recognized from user (%r)" % msg['From'])
return
assert from_user
if not email_reminder:
# At this point, we know it was not a proper reply,
# but it was sent from one of our users because otherwise it would have been
# dealt with just above this
err_msg = u"This is not a reply to an email reminder from your account.\n"\
u"To set some up, go to http://%s/emailreminders/" % \
self.request.host
self.error_reply(err_msg, msg, message_body)
self.write("Not a reply to an email reminder")
return
assert email_reminder
if email_reminder.user._id != from_user._id:
if 'INSTRUCTIONS' in message_body \
and 'DoneCal' in msg['Subject']:
owner_email = email_reminder.user.email
p = owner_email.find('@')
owner_email_brief = owner_email[:p -
2] + '...' + owner_email[p +
2:]
err_msg = u"This is a reply to someone else's email reminder "\
u"that belonged to: %s" % owner_email_brief
self.error_reply(err_msg,
msg,
message_body,
email_reminder=email_reminder)
self.write("No email reminders set up")
return
CRLF = '\r\n'
# formatflowed expects the line breakers to be \r\n
if not message_body.count(CRLF):
message_body = message_body.replace('\n', CRLF)
original_message_lines = [
x for x in message_body.splitlines() if x.startswith('> ')
]
if not original_message_lines:
# then desperately look for something like
# ------Mensaje original------
new_text = []
for line in message_body.splitlines():
if line.startswith('------') and line.endswith('------'):
break
new_text.append(line)
else:
err_msg = u"Your email is not a reply to an email reminder. "\
u"When sending your reply try to use inline style."
self.error_reply(err_msg, msg, message_body)
self.write("Not a reply email")
return
remove_last_paragraph = False
else:
remove_last_paragraph = True
message_body_ascii = message_body.encode(character_set)
from formatflowed import decode
try:
if character_set:
textflow = decode(message_body_ascii,
character_set=character_set)
else:
textflow = decode(message_body_ascii)
except LookupError:
if not character_set:
raise
# _character_set is quite likly 'iso-8859-1;format=flowed'
_character_set = _character_set.split(';')[0].strip()
textflow = decode(message_body_ascii,
character_set=character_set)
new_text = []
for segment in textflow:
if not segment[0]['quotedepth']:
# level zero
new_text.append(segment[1])
else:
break
new_text = u"\n".join(new_text)
if email_reminder:
if email_reminder.time[0] > 12:
about_today = True
else:
about_today = False
else:
if 'today' in msg['Subject']:
about_today = True
else:
about_today = False
tz_offset = email_reminder.tz_offset
count_new_events = 0
paragraphs = re.split('\n\n+', new_text.strip())
if remove_last_paragraph:
# Because we expect the last line to be something like
# On 25 December 2010 09:00, DoneCal
# <*****@*****.**> wrote:
#
# So we have to remove it.
paragraphs = paragraphs[:-1]
previous_duration = None
for text in paragraphs:
if not text.strip():
# it was blank
continue
try:
event = self.parse_and_create_event(
from_user,
text,
about_today,
tz_offset,
previous_duration=previous_duration)
if event:
if not event.all_day:
previous_duration = (event.end -
event.start).seconds / 60
self.write("Created %r\n" % event.title)
count_new_events += 1
except ParseEventError, exception_message:
self.write("Parse event error (%s)\n" % exception_message)
if 'INSTRUCTIONS' in message_body and \
'DoneCal' in msg['Subject']:
err_msg = "Failed to create an event from this line:\n\t%s\n" \
% text
err_msg += "(Error message: %s)\n" % exception_message
self.error_reply(err_msg,
msg,
message_body,
email_reminder=email_reminder)
else:
logging.error("Parse event error on email not replied to",
exc_info=True)
sys.exit(69) # Hard Bounce - don't retry bad arguments
logging.debug("From : %s, to : %r" % (cli_from, cli_to))
try:
conn = sqlite3.connect('/etc/postfix/db/undel.db')
logging.debug("Opening database")
except Exception, e:
logging.error("Error opening database: %s" % (e))
sys.exit(75) # tempfail
if "uatbounce." not in cli_from:
logging.debug("Starting Outbound Mail Process")
verp = "uatbounce." + str(time.time()) + "." + cli_from
sender = re.sub(r'\+.+@', '@', cli_from)
content = ''.join(sys.stdin.readlines())
p = Parser.Parser()
parsed = p.parsestr(content, True)
logging.debug("email source : %s" % parsed.as_string())
subject = parsed.get('Subject')
try:
sql = ("INSERT INTO mails"
"(verp, sender, recipient, subject, bouncetime)"
" VALUES ('%s', '%s', '%s', '%s', 0)" %
(verp, sender, cli_to, subject))
logging.debug("Trying SQL: %s" % (sql))
conn.execute(sql)
except Exception, e:
logging.error("Error inserting record: %s" % (e))
conn.commit()
conn.close()
"""
items = decode_header(h)
parts = []
for item in items:
if item[1] == None:
parts.append(item[0])
else:
parts.append(item[0].decode(item[1]).encode('utf-8'))
return ' '.join(parts)
for partner in session.query(Partner).all():
log(3, 'partner', partner=partner)
log(3, 'begin mail processing from stdin')
mail_parser = Parser.Parser()
parsed_mail = mail_parser.parsestr(sys.stdin.read())
log(3, 'mail parsed')
from email.header import decode_header
log(3, 'parsing mail From: header')
mail_from = parseaddr(parsed_mail.get('From'))
mail_from_decoded = decode_header_ng(mail_from[0])
mail_domain = mail_from[1].split('@')[1]
log(3,
'mail From: header parsed',
raw_name=mail_from[0],
decoded_name=mail_from_decoded,
mail_address=mail_from[1],
mail_domain=mail_domain)