def get_mails(self):
try:
self.__connect()
except ImapBoxConnectionError:
raise ImapBoxConnectionError()
except ImapBoxAuthError:
raise ImapBoxAuthError()
mails = []
try:
if self.use_default_mbox:
result, message = self.mbox.select(readonly=1)
else:
result, message = self.mbox.select(self.mbox_dir, readonly=1)
if result != 'OK':
raise Exception, message
# retrieve only unseen messages
typ, data = self.mbox.search(None, 'UNSEEN')
for num in data[0].split():
# fetch only needed fields
f = self.mbox.fetch(
num,
'(BODY.PEEK[HEADER.FIELDS (SUBJECT FROM MESSAGE-ID)])')
hp = HeaderParser()
m = hp.parsestr(f[1][0][1])
sub = utils.mime_decode(m['subject'])
fr = utils.mime_decode(m['from'])
mails.append([m['Message-ID'], sub, fr])
except Exception, e:
print str(e)
def get_mails(self):
try:
self.__connect()
except ImapBoxConnectionError:
raise ImapBoxConnectionError()
except ImapBoxAuthError:
raise ImapBoxAuthError()
mails = []
try:
if self.use_default_mbox:
result, message = self.mbox.select(readonly=1)
else:
result, message = self.mbox.select(self.mbox_dir, readonly=1)
if result != 'OK':
raise Exception, message
# retrieve only unseen messages
typ, data = self.mbox.search(None, 'UNSEEN')
for num in data[0].split():
# fetch only needed fields
f = self.mbox.fetch(num, '(BODY.PEEK[HEADER.FIELDS (SUBJECT FROM MESSAGE-ID)])')
hp = HeaderParser()
m = hp.parsestr(f[1][0][1])
sub = utils.mime_decode(m['subject'])
fr = utils.mime_decode(m['from'])
mails.append([m['Message-ID'], sub, fr])
except Exception, e:
print str(e)
def getmail(user, password, gmailfolder):
conn = imaplib.IMAP4_SSL("imap.gmail.com", 993)
conn.login(user, password)
conn.select(gmailfolder) # mailbox/gmail label
# print conn.status(gmailfolder,"(MESSAGES)")#print message count
since = datetime.datetime.today() - datetime.timedelta(days=90)
since = since.strftime("%d-%b-%Y")
dates = []
typ, mailnumlist = conn.search(None, "SINCE", since) # or ALL
typ, mailheaders = conn.fetch(mailnumlist[0].replace(" ", ","), "(BODY.PEEK[HEADER])")
for response_part in mailheaders:
if isinstance(response_part, tuple):
header_data = response_part[1]
headerparser = HeaderParser()
msg = headerparser.parsestr(header_data)
# Convert long form date into UTC timestamp while considering tz
msgdatetimestamp = parsedate_tz(msg["Date"])
# Discard hours/minutes/seconds as we are aggregating by whole dates
msgdate = msgdatetimestamp[0:3] + (0, 0, 0, 0, 0, 0, 0, -18000)
# Format cleanly for output as Javascript timestamp
dates.append(str(int(mktime_tz(msgdate))))
conn.close()
datesdict = {}
output = ""
alldates = [key for key, _ in groupby(dates)]
for i in alldates:
datesdict[i] = dates.count(i)
output += '{ "label": "Email", \n'
output += ' "data": [\n'
for k, v in sorted(datesdict.items()):
output += " [%s, %s],\n" % (k, v)
output = output[:-2] + "\n]}"
return output
def close(self):
if self.ran:
return
# Restore stdout so we can pipe our data out
sys.stdout = self.real_stdout
body = self.buffer.getvalue()
headers = ""
self.buffer.close()
# Look for any headers already written
if re.search("location:", body, re.I):
self.redirecting = True
self.sendEntityHeaders = False
msg = HeaderParser().parsestr(body, True)
# Add written headers to our header list
for name, value in msg.items():
self.headers.add_header(name, value)
elif re.search("content-type:", body, re.I):
try:
endOfHeaders = body.index("\n\n")
headers = body[:endOfHeaders]
body = body[endOfHeaders + 2 :]
msg = HeaderParser().parsestr(headers, True)
# Add written headers to our header list
for name, value in msg.items():
self.headers.add_header(name, value)
except ValueError:
raise error, "Headers detected, sort of"
# Get the response body, possibly compressed
if self.sendEntityHeaders:
body = self.getResponseBody(body)
del self.headers["Content-Length"]
self.headers["Content-Length"] = str(len(body))
if not self.redirecting and (not "content-type" in self.headers):
self.headers["Content-Type"] = self.contenttype
etag = base64.encodestring(md5.new(body).digest())[:-1]
self.headers["ETag"] = '"' + etag + '"'
if "HTTP_IF_NONE_MATCH" in os.environ:
if etag in ETags(os.environ["HTTP_IF_NONE_MATCH"]):
print "Status: 304 Not Modified"
self.sendEntityHeaders = False
self._send_headers()
if self.sendEntityHeaders:
sys.stdout.write(body)
self.ran = True
def getHeadersDic(self,file):
ret={}
f=open(file)
p=HeaderParser().parse(f)
ret["Message-ID"] = self.id
ret["subject"] = p.__getitem__("Subject")
ret["retmailfrom"] = p.__getitem__("From")
ret["sentDate"] = p.__getitem__("Date")
ret["xMailer"] = p.__getitem__("User-Agent")
ret["allTo"] = p.__getitem__("To")
return ret
def test_header_parser(self):
eq = self.assertEqual
# Parse only the headers of a complex multipart MIME document
p = HeaderParser()
fp = openfile('msg_02.txt')
msg = p.parse(fp)
eq(msg['from'], '*****@*****.**')
eq(msg['to'], '*****@*****.**')
eq(msg.get_type(), 'multipart/mixed')
eq(msg.is_multipart(), 0)
self.failUnless(isinstance(msg.get_payload(), StringType))
def test_header_parser(self):
eq = self.assertEqual
# Parse only the headers of a complex multipart MIME document
p = HeaderParser()
fp = openfile('msg_02.txt')
msg = p.parse(fp)
eq(msg['from'], '*****@*****.**')
eq(msg['to'], '*****@*****.**')
eq(msg.get_type(), 'multipart/mixed')
eq(msg.is_multipart(), 0)
self.failUnless(isinstance(msg.get_payload(), StringType))
def processRedirect(self, code, match, end): # TODO: detect redirect loops # TODO: permanently change URL on 301? if code not in ['301', '302', '303', '307']: return False parser = HeaderParser() message = parser.parsestr(self.readHeaders[match.end():end], True) if 'Location' not in message: return False location = message['Location'] self.socket.close() self.__init__(location,self.proxy,self.rawServer,self.callback,self.args,self.kwargs) self.get() return True
def processResponse(self, end):
match = STATUS_RE.match(self.readHeaders)
if not match:
self.reportFailure('Invalid response')
return
code, reason = match.groups()
if code != '200':
if not self.processRedirect(code, match, end):
self.reportFailure('Bad response: %s %s' % (code, reason))
return
parser = HeaderParser()
message = parser.parsestr(self.readHeaders[match.end():end], True)
if not self.callback(CONNECT, message, *self.args, **self.kwargs):
self.socket.close()
return
data = self.readHeaders[end:]
self.readHeaders = None
# TODO: Error on unknown transfer-encoding
if 'Transfer-Encoding' in message and \
message['Transfer-Encoding'] == 'chunked':
self.chunkSize = 0
self.sendData(data)
def GetEmailPackageObject(self, strip_mime_headers=True):
header_text, body, html = self._GetMessageTextParts()
try: # catch all exceptions!
attachments = self._GetAttachmentsToInclude()
new_content_type = None
if attachments:
_class = MIMEMultipart
payload = []
if body:
payload.append(MIMEText(body))
if html:
payload.append(MIMEText(html, 'html'))
payload += attachments
new_content_type = "multipart/mixed"
else:
_class = Message
payload = body + '\n' + html
try:
root_msg = HeaderParser(_class=_class).parsestr(header_text)
except email.Errors.HeaderParseError:
raise # sob
if strip_mime_headers:
for h, new_val in (('content-type', new_content_type),
('content-transfer-encoding', None)):
try:
root_msg['X-SpamBayes-Original-' + h] = root_msg[h]
del root_msg[h]
except KeyError:
pass
if new_val is not None:
root_msg[h] = new_val
root_msg.set_payload(payload)
except:
text = '\r\n'.join([header_text, body, html])
print "FAILED to create email.message from: ", `text`
raise
return root_msg
def parse_docstring(docstring):
"""
Parse out the parts of a docstring. Returns (title, body, metadata).
"""
docstring = trim_docstring(docstring)
parts = re.split(r'\n{2,}', docstring)
title = parts[0]
if len(parts) == 1:
body = ''
metadata = {}
else:
parser = HeaderParser()
try:
metadata = parser.parsestr(parts[-1])
except HeaderParseError:
metadata = {}
body = "\n\n".join(parts[1:])
else:
metadata = dict(list(metadata.items()))
if metadata:
body = "\n\n".join(parts[1:-1])
else:
body = "\n\n".join(parts[1:])
return title, body, metadata
def parse_docstring(docstring):
"""
Parse out the parts of a docstring. Returns (title, body, metadata).
"""
docstring = trim_docstring(docstring)
parts = re.split(r'\n{2,}', docstring)
title = parts[0]
if len(parts) == 1:
body = ''
metadata = {}
else:
parser = HeaderParser()
try:
metadata = parser.parsestr(parts[-1])
except HeaderParseError:
metadata = {}
body = "\n\n".join(parts[1:])
else:
metadata = dict(metadata.items())
if metadata:
body = "\n\n".join(parts[1:-1])
else:
body = "\n\n".join(parts[1:])
return title, body, metadata
def index():
if request.method == 'POST':
mail_data = request.form['headers'].strip().encode('ascii', 'ignore')
r = {}
n = HeaderParser().parsestr(mail_data)
graph = []
received = n.get_all('Received')
if received:
received = [i for i in received if ('from' in i or 'by' in i)]
else:
received = re.findall('Received:\s*(.*?)\n\S+:\s+', mail_data,
re.X | re.DOTALL | re.I)
c = len(received)
for i in range(len(received)):
if ';' in received[i]:
line = received[i].split(';')
else:
line = received[i].split('\r\n')
line = map(str.strip, line)
line = map(lambda x: x.replace('\r\n', ' '), line)
try:
if ';' in received[i + 1]:
next_line = received[i + 1].split(';')
else:
next_line = received[i + 1].split('\r\n')
next_line = map(str.strip, next_line)
next_line = map(lambda x: x.replace('\r\n', ''), next_line)
except IndexError:
next_line = None
org_time = dateParser(line[-1])
if not next_line:
next_time = org_time
else:
next_time = dateParser(next_line[-1])
if line[0].startswith('from'):
data = re.findall(
"""
from\s+
(.*?)\s+
by(.*?)
(?:
(?:with|via)
(.*?)
(?:\sid\s|$)
|\sid\s|$
)""", line[0], re.DOTALL | re.X)
else:
data = re.findall(
"""
()by
(.*?)
(?:
(?:with|via)
(.*?)
(?:\sid\s|$)
|\sid\s
)""", line[0], re.DOTALL | re.X)
delay = (org_time - next_time).seconds
if delay < 0:
delay = 0
try:
ftime = org_time.utctimetuple()
ftime = time.strftime('%m/%d/%Y %I:%M:%S %p', ftime)
r[c] = {
'Timestmp':
org_time,
'Time':
ftime,
'Delay':
delay,
'Direction':
map(lambda x: x.replace('\n', ' '),
map(str.strip, data[0]))
}
c -= 1
except IndexError:
pass
for i in r.values():
if i['Direction'][0]:
graph.append(["From: %s" % i['Direction'][0], i['Delay']])
else:
graph.append(["By: %s" % i['Direction'][1], i['Delay']])
totalDelay = sum(map(lambda x: x['Delay'], r.values()))
fTotalDelay = utility_processor()['duration'](totalDelay)
delayed = True if totalDelay else False
custom_style = Style(
background='transparent',
plot_background='transparent',
font_family='googlefont:Open Sans',
# title_font_size=12,
)
line_chart = pygal.HorizontalBar(style=custom_style,
height=250,
legend_at_bottom=True,
tooltip_border_radius=10)
line_chart.tooltip_fancy_mode = False
line_chart.title = 'Total Delay is: %s' % fTotalDelay
line_chart.x_title = 'Delay in seconds.'
for i in graph:
line_chart.add(i[0], i[1])
chart = line_chart.render(is_unicode=True)
summary = {
'From':
n.get('From') or getHeaderVal('from', mail_data),
'To':
n.get('to') or getHeaderVal('to', mail_data),
'Cc':
n.get('cc') or getHeaderVal('cc', mail_data),
'Subject':
n.get('Subject') or getHeaderVal('Subject', mail_data),
'MessageID':
n.get('Message-ID') or getHeaderVal('Message-ID', mail_data),
'Date':
n.get('Date') or getHeaderVal('Date', mail_data),
}
security_headers = [
'Received-SPF', 'Authentication-Results', 'DKIM-Signature',
'ARC-Authentication-Results'
]
return render_template('index.html',
data=r,
delayed=delayed,
summary=summary,
n=n,
chart=chart,
security_headers=security_headers)
else:
return render_template('index.html')
# Copyright (C) 2001 Python Software Foundation
def verpdeliver(mlist, msg, msgdata, envsender, failures, conn):
for recip in msgdata['recips']:
# We now need to stitch together the message with its header and
# footer. If we're VERPIng, we have to calculate the envelope sender
# for each recipient. Note that the list of recipients must be of
# length 1.
#
# BAW: ezmlm includes the message number in the envelope, used when
# sending a notification to the user telling her how many messages
# they missed due to bouncing. Neat idea.
msgdata['recips'] = [recip]
# Make a copy of the message and decorate + delivery that
msgcopy = copy.deepcopy(msg)
Decorate.process(mlist, msgcopy, msgdata)
# Calculate the envelope sender, which we may be VERPing
if msgdata.get('verp'):
bmailbox, bdomain = Utils.ParseEmail(envsender)
rmailbox, rdomain = Utils.ParseEmail(recip)
if rdomain is None:
# The recipient address is not fully-qualified. We can't
# deliver it to this person, nor can we craft a valid verp
# header. I don't think there's much we can do except ignore
# this recipient.
syslog('smtp', 'Skipping VERP delivery to unqual recip: %s',
recip)
continue
d = {'bounces': bmailbox,
'mailbox': rmailbox,
'host' : DOT.join(rdomain),
}
envsender = '%s@%s' % ((mm_cfg.VERP_FORMAT % d), DOT.join(bdomain))
if mlist.personalize == 2:
# When fully personalizing, we want the To address to point to the
# recipient, not to the mailing list
del msgcopy['to']
name = None
if mlist.isMember(recip):
name = mlist.getMemberName(recip)
if name:
# Convert the name to an email-safe representation. If the
# name is a byte string, convert it first to Unicode, given
# the character set of the member's language, replacing bad
# characters for which we can do nothing about. Once we have
# the name as Unicode, we can create a Header instance for it
# so that it's properly encoded for email transport.
charset = Utils.GetCharSet(mlist.getMemberLanguage(recip))
if charset == 'us-ascii':
# Since Header already tries both us-ascii and utf-8,
# let's add something a bit more useful.
charset = 'iso-8859-1'
charset = Charset(charset)
codec = charset.input_codec or 'ascii'
if not isinstance(name, UnicodeType):
name = unicode(name, codec, 'replace')
name = Header(name, charset).encode()
msgcopy['To'] = formataddr((name, recip))
else:
msgcopy['To'] = recip
# We can flag the mail as a duplicate for each member, if they've
# already received this message, as calculated by Message-ID. See
# AvoidDuplicates.py for details.
del msgcopy['x-mailman-copy']
if msgdata.get('add-dup-header', {}).has_key(recip):
msgcopy['X-Mailman-Copy'] = 'yes'
# GPG encryption
if 'encrypted_gpg' in msgdata and msgdata['encrypted_gpg'] and mlist.encrypt_policy!=0:
# Encryption is not forbidden in config
try:
keyids=mlist.getGPGKeyIDs(recip)
except:
keyids=None
if enforceEncryptPolicy(mlist,msg,msgdata) and keyids==None:
syslog('gpg','Encryption mandatory, but no keys found for %s: '\
'Discarding message',recip)
failures[recip]=(550,'Encryption mandatory, but no keys found')
return
gh = GPGUtils.GPGHelper(mlist)
# Extract / generate plaintext
gpg_use_inlineformat = False # TODO: Create config setting
if not msgcopy.is_multipart() and gpg_use_inlineformat:
plaintext=msgcopy.get_payload()
else:
if not msgcopy.is_multipart():
plaintext = 'Content-Type: %s\n' \
'Content-Disposition: inline\n' \
% msgcopy.get('Content-Type')
if not msgcopy.get('Content-Transfer-Encoding') is None:
plaintext += 'Content-Transfer-Encoding: %s\n' \
% msgcopy.get('Content-Transfer-Encoding')
plaintext += '\n%s' % msgcopy.get_payload()
else:
hp = HeaderParser()
tmp = msgcopy.as_string()
tmpmsg = hp.parsestr(tmp)
plaintext = 'Content-Type: %s\n' \
'Content-Disposition: inline\n\n%s' \
% (msgcopy.get('Content-Type'),tmpmsg.get_payload())
# Do encryption, report errors
ciphertext = None
if not keyids is None:
# Can encrypt.
# No signing policy, or voluntary and original wasn't signed: just encrypt
if mlist.sign_policy == 0 or \
(mlist.sign_policy==1 and not msgdata['signed_gpg']):
ciphertext = gh.encryptMessage(plaintext,keyids)
else:
ciphertext = gh.encryptSignMessage(plaintext,keyids)
if ciphertext==None:
# Must always encrypt, since if we arrived here encrypt_policy
# is either Mantatory or (Voluntary and incoming msg was encrypted).
syslog('gpg',"Can't encrypt message to %s: " \
"Discarding message",keyids)
failures[recip]=(550,'Unable to encrypt message')
return
# Compile encrypted message
if not ciphertext is None:
if msgcopy.has_key('Content-Transfer-Encoding'):
msgcopy.replace_header('Content-Transfer-Encoding','7bit')
else:
msgcopy.add_header('Content-Transfer-Encoding','7bit')
if not msgcopy.is_multipart() and gpg_use_inlineformat:
msgcopy.set_payload(ciphertext)
msgcopy.set_param('x-action','pgp-encrypted')
else:
msgcopy.replace_header('Content-Type','multipart/encrypted')
msgcopy.set_param('protocol','application/pgp-encrypted')
msgcopy.set_payload(None)
submsg = Message()
submsg.add_header('Content-Type','application/pgp-encrypted')
submsg.set_payload('Version: 1\n')
msgcopy.attach(submsg)
submsg = Message()
submsg.add_header('Content-Type','application/octet-stream; name="encrypted.asc"')
submsg.add_header('Content-Disposition','inline; filename="encrypted.asc"')
submsg.set_payload(ciphertext)
msgcopy.attach(submsg)
syslog('gpg','Sending encrypted message to %s',recip)
else:
syslog('gpg','Sending unencrypted message to %s',recip)
if 'encrypted_smime' in msgdata and msgdata['encrypted_smime'] and mlist.encrypt_policy != 0:
# FIXME: this is as crude as can be
sm = SMIMEUtils.SMIMEHelper(mlist)
recipfile = sm.getSMIMEMemberCertFile(recip)
if not recipfile:
failures[recip]=(550,'No S/MIME key found')
return
else:
plaintext=msgcopy.get_payload()
if not msgcopy.is_multipart():
plaintext = msgcopy.get_payload()
syslog('gpg', "About to S/MIME encrypt plaintext from singlepart")
else:
# message contains e.g. signature?
# FIXME we fetch only the first attachment. We search for
# attachments only 2 levels deep. That's suboptimal...
# perhaps the PGP way (invoking
# hp = HeaderParser()
# ) is better.
submsgs = msgcopy.get_payload()
submsg = submsgs[0]
if not submsg.is_multipart():
plaintext = submsg.get_payload()
else:
subsubmsgs = submsg.get_payload()
subsubmsg = subsubmsgs[0]
plaintext = subsubmsg.get_payload()
syslog('gpg', "About to S/MIME encrypt plaintext from multipart")
if mlist.sign_policy == 0 or \
(mlist.sign_policy==1 and not msgdata['signed_smime']):
ciphertext = sm.encryptMessage(plaintext,recipfile)
else:
ciphertext = sm.encryptSignMessage(plaintext,recipfile)
# deal with both header and body-part of ciphertext
(header, body) = ciphertext.split("\n\n", 1)
for l in header.split("\n"):
(k, v) = l.split(": ", 1)
# behave sane with borken openssl like 0.9.7e (e.g. Debian's 0.9.7e-3sarge1)
# openssl 0.9.8a-4a0.sarge.1 is known to work OK.
# A borken openssl (and therefore sm.encryptMessage) returns
# Content-Type: application/x-pkcs7-mime; name="smime.p7m"
# while we need a
# Content-Type: application/x-pkcs7-mime; smime-type=enveloped-data; name="smime.p7m"
if v == 'application/x-pkcs7-mime; name="smime.p7m"':
v = 'application/x-pkcs7-mime; smime-type=enveloped-data; name="smime.p7m"'
try:
msgcopy.replace_header(k, v)
except KeyError:
msgcopy.add_header(k, v)
msgcopy.set_payload(body)
# For the final delivery stage, we can just bulk deliver to a party of
# one. ;)
bulkdeliver(mlist, msgcopy, msgdata, envsender, failures, conn)
def index():
if request.method == 'POST':
data = request.form['headers'].strip()
r = {}
n = HeaderParser().parsestr(data.encode('ascii', 'ignore'))
graph = []
c = len(n.get_all('Received'))
for i in range(len(n.get_all('Received'))):
line = n.get_all('Received')[i].split(';')
try:
next_line = n.get_all('Received')[i + 1].split(';')
except IndexError:
next_line = None
org_time = email.utils.mktime_tz(email.utils.parsedate_tz(line[1]))
if not next_line:
next_time = org_time
else:
next_time = email.utils.mktime_tz(
email.utils.parsedate_tz(next_line[1]))
if line[0].startswith('from'):
data = re.findall(
"""
from\s+
(.*?)\s+
by(.*?)
(?:
(?:with|via)
(.*?)
(?:id|$)
|id|$
)""", line[0], re.DOTALL | re.X)
else:
data = re.findall(
"""
()by
(.*?)
(?:
(?:with|via)
(.*?)
(?:id|$)
|id
)""", line[0], re.DOTALL | re.X)
delay = org_time - next_time
if delay < 0:
delay = 0
try:
time = datetime.fromtimestamp(org_time)
ftime = time.strftime('%m/%d/%Y %I:%M:%S %p')
r[c] = {
'Timestmp':
org_time,
'Time':
ftime,
'Delay':
delay,
'Direction':
map(lambda x: x.replace('\n', ' '),
map(str.strip, data[0]))
}
c -= 1
except IndexError:
pass
for i in r.values():
if i['Direction'][0]:
graph.append(["From: %s" % i['Direction'][0], i['Delay']])
else:
graph.append(["By: %s" % i['Direction'][1], i['Delay']])
totalDelay = sum(map(lambda x: x['Delay'], r.values()))
fTotalDelay = utility_processor()['duration'](totalDelay)
delayed = True if totalDelay else False
custom_style = Style(
background='transparent',
plot_background='transparent',
font_family='googlefont:Open Sans',
title_font_size=12,
)
line_chart = pygal.HorizontalBar(style=custom_style, height=200)
line_chart.tooltip_fancy_mode = False
line_chart.js = ['%s/js/pygal-tooltips.min.js' % app.static_url_path]
line_chart.title = 'Total Delay is: %s' % fTotalDelay
line_chart.x_title = 'Delay in seconds.'
for i in graph:
line_chart.add(i[0], i[1])
chart = line_chart.render(is_unicode=True)
summary = {
'From': n.get('from'),
'To': n.get('to'),
'Cc': n.get('cc'),
'Subject': n.get('Subject'),
'MessageID': n.get('Message-ID'),
'Date': n.get('Date'),
}
return render_template('index.html',
data=r,
delayed=delayed,
summary=summary,
n=n,
chart=chart)
else:
return render_template('index.html')
