def __init__(self, message, message_id=None, gpg=None):
"""
:arg str message: raw text of the email
:arg str message_id: IMAP message ID or maildir message path
:arg gpg: :class:`GnuPG` instance or None (which will create one)
"""
Message.__init__(self, message)
self._message_id = message_id
if not gpg:
self._gpg = GnuPG()
else:
self._gpg = gpg
self._content_types = ["text/plain", "text/html",
"application/pgp-signature", "application/pgp-keys",
"application/octet-stream"]
self._parts = []
for part in self.walk():
content_type = part.get_content_type()
if content_type in self._content_types:
payload = quopri.decodestring(part.get_payload().strip())
self._parts.append( (content_type, payload) )
self._parse_for_openpgp()
def append_messages(self, messages, last_uid):
self.select()
for msg in messages:
del msg['X-Norless-Id']
msg['X-Norless-Id'] = msg.msgkey
del msg['Message-ID']
msg['Message-ID'] = msg.msgkey
self.box.client.append(self.name, '(\\Seen)', time.time(), msg.as_string())
result = self.box.client.uid('search', '(UID {}:*)'.format(last_uid + 1))
uids = [r for r in result[1][0].split() if int(r) > last_uid]
stored_messages = []
if uids:
result = self.box.client.uid('fetch', ','.join(uids),
'(UID BODY.PEEK[HEADER])')
it = iter(result[1])
for info, msg in it:
uid = get_field(info, 'UID')
msg = Message(msg.replace('\r\n', '\n'))
if 'X-Norless-Id' in msg:
stored_messages.append((int(uid), msg['X-Norless-Id'].strip()))
next(it)
return stored_messages
def __init__(self, message, message_id=None, gpg=None):
"""
:arg str message: raw text of the email
:arg str message_id: IMAP message ID or maildir message path
:arg gpg: :class:`GnuPG` instance or None (which will create one)
"""
Message.__init__(self, message)
self._message_id = message_id
if not gpg:
self._gpg = GnuPG()
else:
self._gpg = gpg
self._content_types = [
"text/plain", "text/html", "application/pgp-signature",
"application/pgp-keys", "application/octet-stream"
]
self._parts = []
for part in self.walk():
content_type = part.get_content_type()
if content_type in self._content_types:
payload = quopri.decodestring(part.get_payload().strip())
self._parts.append((content_type, payload))
self._parse_for_openpgp()
def send_async_email(email_data):
"""Background task to send an email with Flask-Mail."""
msg = Message(email_data['subject'],
sender=app.config['MAIL_DEFAULT_SENDER'],
recipients=[email_data['to']])
msg.body = email_data['body']
with app.app_context():
mail.send(msg)
def filename_from_content_disposition(content_disposition):
""" Extract and validate filename from a Content-Disposition header. """
msg = Message('Content-Disposition: %s' % content_disposition)
filename = msg.get_filename()
if filename:
# Basic sanitation.
filename = os.path.basename(filename).lstrip('.').strip()
if filename:
return filename
def __init__(self, raw_data):
"""Parse an RFC2822,5322 mail message."""
self.raw = raw_data
try:
self.mail = Message(raw_data)
except Exception as exc:
log.error('Parse message failed %s' % exc)
raise exc
if self.mail.defects:
# XXX what to do ?
log.warn('Defects on parsed mail %r' % self.mail.defects)
self.warning = self.mail.defects
self.get_bodies()
def test_happy_path():
p = DowHourHeatmap()
msg = Message()
msg.set_payload('Hello world!', 'utf-8')
msg["Date"] = 'Fri, 9 Jun 2006 12:34:56 -0200'
p.process(msg)
generated = p.report_snippet()
assert '"type": "heatmap"' in generated
assert '"Mon", "Tue", "Wed"' in generated
hrs = [0] * 24
assert generated.count(str(hrs)) == 6
# element 12 + 2 is 1, because it is UTC
hrs[14] = 1
assert str(hrs) in generated
def message_factory(message):
m = Message(message)
logger.debug('Handling mail from %s with subject "%s"', m['From'],
m['Subject'])
logger.debug(m['Date'])
try:
d = dateparse(m['Date'])
except ValueError:
logger.warning(
'Mail from %s with subject "%s" Does not have a '
'"Date" header! Some features will fail because of '
'this!', m['From'], m['Subject'])
d = datetime.now()
m.set_date(d.timestamp())
return m
def forward(self, command, forward_to, m_from=None):
if type(command) is not str:
logger.error(
"Trying to forward message %s without setting the "
"command to use!", command)
m = Message()
m['From'] = m_from or self['to']
m['To'] = forward_to
m['Subject'] = "Fwd: " + self['subject']
m.set_payload(self.get_payload())
run(command + " " + m['To'],
stdout=PIPE,
input=m.as_string().encode('utf-8'),
shell=True,
check=True)
return self
def filename_from_content_disposition(content_disposition):
"""
Extract and validate filename from a Content-Disposition header.
:param content_disposition: Content-Disposition value
:return: the filename if present and valid, otherwise `None`
"""
# attachment; filename=jakubroztocil-httpie-0.4.1-20-g40bd8f6.tar.gz
msg = Message('Content-Disposition: %s' % content_disposition)
filename = msg.get_filename()
if filename:
# Basic sanitation.
filename = os.path.basename(filename).lstrip('.').strip()
if filename:
return filename
def filename_from_content_disposition(content_disposition):
"""
Extract and validate filename from a Content-Disposition header.
:param content_disposition: Content-Disposition value
:return: the filename if present and valid, otherwise `None`
"""
# attachment; filename=jakubroztocil-httpie-0.4.1-20-g40bd8f6.tar.gz
msg = Message('Content-Disposition: %s' % content_disposition)
filename = msg.get_filename()
if filename:
# Basic sanitation.
filename = os.path.basename(filename).lstrip('.').strip()
if filename:
return filename
def process_message(self, page):
id_ = fid, tid, mid = page.id
headers = {
'GGE-Url': page.url,
'GGE-Id': '{}/{}/{}'.format(*id_),
'GGE-Forum-Id': fid,
'GGE-Topic-Id': tid,
'GGE-Message-Id': mid,
}
msg = Message(str(page))
for k, v in headers.items():
msg.add_header(k, v)
self.mbox.add(msg)
self.summary['messages'] += 1
def process_message(self, page):
id_ = fid, tid, mid = page.id
headers = {
'GGE-Url': page.url,
'GGE-Id': '{}/{}/{}'.format(*id_),
'GGE-Forum-Id': fid,
'GGE-Topic-Id': tid,
'GGE-Message-Id': mid,
}
msg = Message(str(page))
for k, v in headers.items():
msg.add_header(k, v)
self.mbox.add(msg)
self.summary['messages'] += 1
def load_mail(filename):
"""Read email from fixtures of an user."""
# XXX tofix: set fixtures in a more convenient way to not
# have dirty hacking on relative path
dir_path = os.path.dirname(os.path.realpath(__file__))
path = '{}/fixtures'.format(dir_path)
with open('{}/{}'.format(path, filename)) as f:
data = f.read()
return Message(data)
def __init__(self, raw):
"""Initialize structure from a raw mail."""
try:
self.mail = Message(raw)
except Exception as exc:
log.error('Parse message failed %s' % exc)
raise
if self.mail.defects:
# XXX what to do ?
log.warn('Defects on parsed mail %r' % self.mail.defects)
self.recipients = self._extract_recipients()
self.parts = self._extract_parts()
self.headers = self._extract_headers()
self.subject = self.mail.get('Subject')
tmp_date = parsedate_tz(self.mail['Date'])
self.date = datetime.fromtimestamp(mktime_tz(tmp_date))
self.external_message_id = self.mail.get('Message-Id')
self.external_parent_id = self.mail.get('In-Reply-To')
self.size = len(raw)
def __init__(self, raw_data):
"""Parse an RFC2822,5322 mail message."""
self.raw = raw_data
try:
self.mail = Message(raw_data)
except Exception as exc:
log.error('Parse message failed %s' % exc)
raise exc
if self.mail.defects:
# XXX what to do ?
log.warn('Defects on parsed mail %r' % self.mail.defects)
self.warning = self.mail.defects
self.get_bodies()
def get_fields(self, uid, fields):
""" Return an mailbox.Message object containing only the requested
header fields of the message with UID.
The fields parameter is a string ofheader fields seperated by
spaces, e.g. 'From SUBJECT date'
Raise KeyError if there if there is no message with that UID.
"""
(code, data) = self._server.uid('fetch', uid,
"(BODY.PEEK[HEADER.FIELDS (%s)])"
% fields)
if code != 'OK':
raise ImapNotOkError("%s in fetch_header(%s)" % (code, uid))
try:
rfc822string = data[0][1]
except TypeError:
raise KeyError("No UID %s in get_fields" % uid)
result = Message(rfc822string)
return result
def __init__(self, raw):
"""Initialize structure from a raw mail."""
try:
self.mail = Message(raw)
except Exception as exc:
log.error('Parse message failed %s' % exc)
raise
if self.mail.defects:
# XXX what to do ?
log.warn('Defects on parsed mail %r' % self.mail.defects)
self.recipients = self._extract_recipients()
self.parts = self._extract_parts()
self.headers = self._extract_headers()
self.subject = self.mail.get('Subject')
tmp_date = parsedate_tz(self.mail['Date'])
self.date = datetime.fromtimestamp(mktime_tz(tmp_date))
self.external_message_id = self.mail.get('Message-Id')
self.external_parent_id = self.mail.get('In-Reply-To')
self.size = len(raw)
def test_happy_path():
p = ActivityOverTime()
msg = Message()
msg.set_payload('Hello world!', 'utf-8')
msg["Date"] = 'Fri, 9 Jun 2006 12:34:56 -0200'
p.process(msg)
generated = p.report_snippet()
assert '"x": ["2006-06-09"], "y": [1]' in generated
msg = Message()
# no payload this time
msg["Date"] = 'Sun, 13 Jun 2006 12:34:56 -0200'
p.process(msg)
generated = p.report_snippet()
assert '"x": ["2006-06-09", "2006-06-13"], "y": [1, 1]' in generated
def get_message(self, key):
info, payload = self.get(key)
return Message(payload)
def get_message(self, key):
"""Return a Message representation or raise a KeyError."""
return Message(self._get(key))
def msg_has_attachment(msg: Message) -> bool:
return (msg.get_content_type() != "multipart"
and msg.get("Content-Disposition") and msg.get_filename())
def get_attachment_msgs(msg: Message, mime_type: str) -> Generator:
return (msg for msg in msg.walk()
if msg_has_attachment(msg) and msg.get_content_type() == mime_type)
class MailMessage(object):
"""
Mail message structure.
Got a mail in raw rfc2822 format, parse it to
resolve all recipients emails, parts and group headers
"""
zope.interface.implements(IMessageParser)
recipient_headers = ['From', 'To', 'Cc', 'Bcc']
message_protocol = 'email'
warnings = []
body_html = ""
body_plain = ""
def __init__(self, raw_data):
"""Parse an RFC2822,5322 mail message."""
self.raw = raw_data
try:
self.mail = Message(raw_data)
except Exception as exc:
log.error('Parse message failed %s' % exc)
raise exc
if self.mail.defects:
# XXX what to do ?
log.warn('Defects on parsed mail %r' % self.mail.defects)
self.warning = self.mail.defects
self.get_bodies()
def get_bodies(self):
"""Extract body alternatives, if any."""
body_html = ""
body_plain = ""
if self.mail.get("Content-Type", None):
if self.mail.is_multipart():
for top_level_part in self.mail.get_payload():
if top_level_part.get_content_maintype() == "multipart":
for alternative in top_level_part.get_payload():
charset = alternative.get_param("charset")
if isinstance(charset, tuple):
charset = unicode(charset[2], charset[0]
or "us-ascii")
if alternative.get_content_type() == "text/plain":
body_plain = alternative.get_payload(
decode=True)
self.body_plain = to_utf8(body_plain, charset)
elif alternative.get_content_type() == "text/html":
body_html = alternative. \
get_payload(decode=True)
self.body_html = to_utf8(body_html, charset)
break
else:
charset = top_level_part.get_param("charset")
if isinstance(charset, tuple):
charset = unicode(charset[2], charset[0]
or "us-ascii")
if top_level_part.get_content_type() == "text/plain":
body_plain = top_level_part. \
get_payload(decode=True)
self.body_plain = to_utf8(body_plain, charset)
elif top_level_part.get_content_type() == "text/html":
body_html = top_level_part.get_payload(decode=True)
self.body_html = to_utf8(body_html, charset)
else:
charset = self.mail.get_param("charset")
if isinstance(charset, tuple):
charset = unicode(charset[2], charset[0] or "us-ascii")
if self.mail.get_content_type() == "text/html":
body_html = self.mail.get_payload(decode=True)
self.body_html = to_utf8(body_html, charset)
else:
body_plain = self.mail.get_payload(decode=True)
self.body_plain = to_utf8(body_plain, charset)
else:
self.body_plain = self.mail.get_payload(decode=True)
@property
def subject(self):
"""Mail subject."""
s = decode_header(self.mail.get('Subject'))
charset = s[0][1]
if charset is not None:
return s[0][0].decode(charset, "replace"). \
encode("utf-8", "replace")
else:
return s[0][0]
@property
def size(self):
"""Get mail size in bytes."""
return len(self.mail.as_string())
@property
def external_references(self):
"""Return mail references to be used as external references.
making use of RFC5322 headers :
message-id
in-reply-to
references
headers' strings are pruned to extract email addresses only.
"""
ext_id = self.mail.get('Message-Id')
parent_id = self.mail.get('In-Reply-To')
ref = self.mail.get_all("References")
ref_addr = getaddresses(ref) if ref else None
ref_ids = [address[1] for address in ref_addr] if ref_addr else []
mid = clean_email_address(ext_id)[1] if ext_id else None
pid = clean_email_address(parent_id)[1] if parent_id else None
return {'message_id': mid, 'parent_id': pid, 'ancestors_ids': ref_ids}
@property
def date(self):
"""Get UTC date from a mail message."""
mail_date = self.mail.get('Date')
if mail_date:
tmp_date = parsedate_tz(mail_date)
return datetime.datetime.fromtimestamp(mktime_tz(tmp_date))
log.debug('No date on mail using now (UTC)')
return datetime.datetime.now(tz=pytz.utc)
@property
def participants(self):
"""Mail participants."""
participants = []
for header in self.recipient_headers:
addrs = []
participant_type = header.capitalize()
if self.mail.get(header):
if ',' in self.mail.get(header):
parts = self.mail.get(header).split(',')
filtered = [x for x in parts if '@' in x]
addrs.extend(filtered)
else:
addrs.append(self.mail.get(header))
for addr in addrs:
participant = MailParticipant(participant_type, addr)
participants.append(participant)
return participants
@property
def hash_participants(self):
"""Create an hash from participants addresses for global lookup."""
addresses = [x.address for x in self.participants]
addresses = list(set(addresses))
addresses.sort()
return hashlib.sha256(''.join(addresses)).hexdigest()
@property
def attachments(self):
"""Extract parts which we consider as attachments."""
if not self.mail.is_multipart():
return []
attchs = []
for p in walk_with_boundary(self.mail, ""):
if not p.is_multipart():
if MailAttachment.is_attachment(p):
attchs.append(MailAttachment(p))
return attchs
@property
def extra_parameters(self):
"""Mail message extra parameters."""
lists = self.mail.get_all("List-ID")
lists_addr = getaddresses(lists) if lists else None
lists_ids = [address[1] for address in lists_addr] \
if lists_addr else []
return {'lists': lists_ids}
def lookup_discussion_sequence(self, *args, **kwargs):
"""Return list of lookup type, value from a mail message."""
seq = []
# list lookup first
for list_id in self.extra_parameters.get('lists', []):
seq.append(('list', list_id))
seq.append(('global', self.hash_participants))
# try to link message to external thread's root message-id
if len(self.external_references["ancestors_ids"]) > 0:
seq.append(
("thread", self.external_references["ancestors_ids"][0]))
elif self.external_references["parent_id"]:
seq.append(("thread", self.external_references["parent_id"]))
elif self.external_references["message_id"]:
seq.append(("thread", self.external_references["message_id"]))
return seq
# Others parameters specific for mail message
@property
def headers(self):
"""Extract all headers into list.
Duplicate on headers exists, group them by name
with a related list of values
"""
def keyfunc(item):
return item[0]
# Group multiple value for same headers into a dict of list
headers = {}
data = sorted(self.mail.items(), key=keyfunc)
for k, g in groupby(data, key=keyfunc):
headers[k] = [x[1] for x in g]
return headers
def get_attachments(self, email: mailbox.Message):
if email is None:
return
week_str = re.search(r'(\d+/\d+/\d+)', email['subject']).group(1)
week = datetime.datetime.strptime(week_str, '%m/%d/%Y').date()
attachment_url = re.search(r'Name: completed_failed_avr_time.csv[^<]*URL: <([^>]+)>', email.get_payload(), re.DOTALL).group(1)
filename = os.path.join(self._tmpdir, 'buildbot_stats_{}.csv'.format(week.isoformat()))
self.download(attachment_url, filename)
class MailMessage(object):
"""
Mail message structure.
Got a mail in raw rfc2822 format, parse it to
resolve all recipients emails, parts and group headers
"""
recipient_headers = ['From', 'To', 'Cc', 'Bcc']
message_type = 'mail'
def __init__(self, raw):
"""Initialize structure from a raw mail."""
try:
self.mail = Message(raw)
except Exception as exc:
log.error('Parse message failed %s' % exc)
raise
if self.mail.defects:
# XXX what to do ?
log.warn('Defects on parsed mail %r' % self.mail.defects)
self.recipients = self._extract_recipients()
self.parts = self._extract_parts()
self.headers = self._extract_headers()
self.subject = self.mail.get('Subject')
tmp_date = parsedate_tz(self.mail['Date'])
self.date = datetime.fromtimestamp(mktime_tz(tmp_date))
self.external_message_id = self.mail.get('Message-Id')
self.external_parent_id = self.mail.get('In-Reply-To')
self.size = len(raw)
@property
def text(self):
"""Message all text."""
# XXX : more complexity ?
return "\n".join([x.data for x in self.parts if x.can_index])
def _extract_recipients(self):
recip = {}
for header in self.recipient_headers:
addrs = []
recipient_type = header.lower()
if self.mail.get(header):
if ',' in self.mail.get(header):
addrs.extend(self.mail.get(header).split(','))
else:
addrs.append(self.mail.get(header))
addrs = [clean_email_address(x) for x in addrs]
recip[recipient_type] = addrs
return recip
def _extract_headers(self):
"""
Extract all headers into list.
Duplicate on headers exists, group them by name
with a related list of values
"""
def keyfunc(item):
return item[0]
# Group multiple value for same headers into a dict of list
headers = {}
data = sorted(self.mail.items(), key=keyfunc)
for k, g in groupby(data, key=keyfunc):
headers[k] = [x[1] for x in g]
return headers
def _extract_parts(self):
"""Multipart message, extract parts."""
parts = []
for p in self.mail.walk():
if not p.is_multipart():
parts.append(self._process_part(p))
return parts
def _process_part(self, part):
return MailPart(part)
@property
def transport_privacy_index(self):
"""Evaluate transport privacy index."""
# XXX : TODO
return random.randint(0, 50)
@property
def content_privacy_index(self):
"""Evaluate content privacy index."""
# XXX: real evaluation needed ;)
if 'PGP' in [x.content_type for x in self.parts]:
return random.randint(50, 100)
else:
return 0.0
@property
def spam_level(self):
"""Report spam level."""
try:
score = self.headers.get('X-Spam-Score')
score = float(score[0])
except:
score = 0.0
if score < 5.0:
return 0.0
if score >= 5.0 and score < 15.0:
return min(score * 10, 100.0)
return 100.0
@property
def importance_level(self):
"""Return percent estimated importance level of this message."""
# XXX. real compute needed
return 0 if self.spam_level else random.randint(50, 100)
@property
def lists(self):
"""List related to message."""
lists = []
for list_name in self.headers.get('List-ID', []):
lists.append(list_name)
return lists
@property
def from_(self):
"""Get from recipient."""
from_ = self.recipients.get('from')
if from_:
# XXX should do better
return from_[0][1]
return None
def lookup_sequence(self):
"""Build parameter sequence for lookups."""
seq = []
# first from parent
if self.external_parent_id:
seq.append(('parent', self.external_parent_id))
# then list lookup
for listname in self.lists:
seq.append(('list', listname))
# last try to lookup from sender address
if self.from_:
seq.append(('from', self.from_))
return seq
def to_parameter(self):
"""Transform mail to a NewMessage parameter."""
msg = NewMessage()
msg.type = 'email'
msg.subject = self.subject
msg.from_ = self.from_
# XXX need transform to part parameter
for part in self.parts:
param = Part()
param.content_type = part.content_type
param.data = part.data
param.size = part.size
param.filename = part.filename
param.can_index = part.can_index
msg.parts.append(param)
msg.headers = self.headers
msg.date = self.date
msg.size = self.size
msg.text = self.text
msg.external_parent_id = self.external_parent_id
msg.external_message_id = self.external_message_id
# XXX well ....
msg.privacy_index = (self.transport_privacy_index +
self.content_privacy_index) / 2
msg.importance_level = self.importance_level
return msg
class MailMessage(object):
"""
Mail message structure.
Got a mail in raw rfc2822 format, parse it to
resolve all recipients emails, parts and group headers
"""
zope.interface.implements(IMessageParser)
recipient_headers = ['From', 'To', 'Cc', 'Bcc']
message_protocol = 'email'
warnings = []
body_html = ""
body_plain = ""
def __init__(self, raw_data):
"""Parse an RFC2822,5322 mail message."""
self.raw = raw_data
self._extra_parameters = {}
try:
self.mail = Message(raw_data)
except Exception as exc:
log.error('Parse message failed %s' % exc)
raise exc
if self.mail.defects:
# XXX what to do ?
log.warn('Defects on parsed mail %r' % self.mail.defects)
self.warning = self.mail.defects
self.get_bodies()
def get_bodies(self):
"""Extract body alternatives, if any."""
body_html = ""
body_plain = ""
if self.mail.get("Content-Type", None):
if self.mail.is_multipart():
if self.mail.get_content_subtype() == 'encrypted':
parts = self.mail.get_payload()
if len(parts) == 2:
self.body_plain = parts[1].get_payload()
return
else:
log.warn('Encrypted message with invalid parts count')
for top_level_part in self.mail.get_payload():
if top_level_part.get_content_maintype() == "multipart":
for alternative in top_level_part.get_payload():
charset = alternative.get_param("charset")
if isinstance(charset, tuple):
charset = unicode(charset[2], charset[0]
or "us-ascii")
if alternative.get_content_type() == "text/plain":
body_plain = alternative.get_payload(
decode=True)
self.body_plain = to_utf8(body_plain, charset)
elif alternative.get_content_type() == "text/html":
body_html = alternative. \
get_payload(decode=True)
self.body_html = to_utf8(body_html, charset)
break
else:
charset = top_level_part.get_param("charset")
if isinstance(charset, tuple):
charset = unicode(charset[2], charset[0]
or "us-ascii")
if top_level_part.get_content_type() == "text/plain":
body_plain = top_level_part. \
get_payload(decode=True)
self.body_plain = to_utf8(body_plain, charset)
elif top_level_part.get_content_type() == "text/html":
body_html = top_level_part.get_payload(decode=True)
self.body_html = to_utf8(body_html, charset)
else:
charset = self.mail.get_param("charset")
if isinstance(charset, tuple):
charset = unicode(charset[2], charset[0] or "us-ascii")
if self.mail.get_content_type() == "text/html":
body_html = self.mail.get_payload(decode=True)
self.body_html = to_utf8(body_html, charset)
else:
body_plain = self.mail.get_payload(decode=True)
self.body_plain = to_utf8(body_plain, charset)
else:
self.body_plain = self.mail.get_payload(decode=True)
@property
def subject(self):
"""Mail subject."""
s = decode_header(self.mail.get('Subject'))
charset = s[0][1]
if charset is not None:
return s[0][0].decode(charset, "replace"). \
encode("utf-8", "replace")
else:
try:
return s[0][0].decode('utf-8', errors='ignore')
except UnicodeError:
log.warn('Invalid subject encoding')
return s[0][0]
@property
def size(self):
"""Get mail size in bytes."""
return len(self.mail.as_string())
@property
def external_references(self):
"""Return mail references to be used as external references.
making use of RFC5322 headers :
message-id
in-reply-to
references
headers' strings are pruned to extract email addresses only.
"""
ext_id = self.mail.get('Message-Id')
parent_id = self.mail.get('In-Reply-To')
ref = self.mail.get_all("References")
ref_addr = getaddresses(ref) if ref else None
ref_ids = [address[1] for address in ref_addr] if ref_addr else []
mid = clean_email_address(ext_id)[1] if ext_id else None
if not mid:
log.error('Unable to find correct message_id {}'.format(ext_id))
mid = ext_id
pid = clean_email_address(parent_id)[1] if parent_id else None
if not pid:
pid = parent_id
return {'message_id': mid, 'parent_id': pid, 'ancestors_ids': ref_ids}
@property
def date(self):
"""Get UTC date from a mail message."""
mail_date = self.mail.get('Date')
if mail_date:
try:
tmp_date = parsedate_tz(mail_date)
return datetime.datetime.fromtimestamp(mktime_tz(tmp_date))
except TypeError:
log.error('Invalid date in mail {}'.format(mail_date))
log.debug('No date on mail using now (UTC)')
return datetime.datetime.now(tz=pytz.utc)
@property
def participants(self):
"""Mail participants."""
participants = []
for header in self.recipient_headers:
addrs = []
participant_type = header.capitalize()
if self.mail.get(header):
parts = self.mail.get(header).split('>,')
if not parts:
pass
if parts and parts[0] == 'undisclosed-recipients:;':
pass
filtered = [x for x in parts if '@' in x]
addrs.extend(filtered)
for addr in addrs:
participant = MailParticipant(participant_type, addr.lower())
if participant.address == '' and participant.label == '':
log.warn('Invalid email address {}'.format(addr))
else:
participants.append(participant)
return participants
@property
def attachments(self):
"""Extract parts which we consider as attachments."""
if not self.mail.is_multipart():
return []
attchs = []
for p in walk_with_boundary(self.mail, ""):
if not p.is_multipart():
if p.get_content_subtype() == 'pgp-encrypted':
# Special consideration. Do not present it as an attachment
# but set _extra_parameters accordingly
self._extra_parameters.update({'encrypted': 'pgp'})
continue
if MailAttachment.is_attachment(p):
attchs.append(MailAttachment(p))
return attchs
@property
def extra_parameters(self):
"""Mail message extra parameters."""
lists = self.mail.get_all("List-ID")
lists_addr = getaddresses(lists) if lists else None
lists_ids = [address[1] for address in lists_addr] \
if lists_addr else []
self._extra_parameters.update({'lists': lists_ids})
return self._extra_parameters
# Others parameters specific for mail message
@property
def headers(self):
"""Extract all headers into list.
Duplicate on headers exists, group them by name
with a related list of values
"""
def keyfunc(item):
return item[0]
# Group multiple value for same headers into a dict of list
headers = {}
data = sorted(self.mail.items(), key=keyfunc)
for k, g in groupby(data, key=keyfunc):
headers[k] = [x[1] for x in g]
return headers
@property
def external_flags(self):
"""
Get headers added by our fetcher that represent flags or labels
set by external provider,
returned as list of tags
"""
tags = []
for h in ['X-Fetched-Imap-Flags', 'X-Fetched-X-GM-LABELS']:
enc_flags = self.mail.get(h)
if enc_flags:
flags_str = base64.decodestring(enc_flags)
for flag in string.split(flags_str, '\r\n'):
if flag not in EXCLUDED_EXT_FLAGS:
tag = Tag()
tag.name = flag
tag.label = flag
tag.type = 'imported'
tags.append(tag)
return tags
class MailMessage(object):
"""
Mail message structure.
Got a mail in raw rfc2822 format, parse it to
resolve all recipients emails, parts and group headers
"""
zope.interface.implements(IMessageParser)
recipient_headers = ['From', 'To', 'Cc', 'Bcc']
message_protocol = 'email'
warnings = []
body_html = ""
body_plain = ""
def __init__(self, raw_data):
"""Parse an RFC2822,5322 mail message."""
self.raw = raw_data
try:
self.mail = Message(raw_data)
except Exception as exc:
log.error('Parse message failed %s' % exc)
raise exc
if self.mail.defects:
# XXX what to do ?
log.warn('Defects on parsed mail %r' % self.mail.defects)
self.warning = self.mail.defects
self.get_bodies()
def get_bodies(self):
"""Extract body alternatives, if any."""
body_html = ""
body_plain = ""
if self.mail.get("Content-Type", None):
if self.mail.is_multipart():
if self.mail.get_content_subtype() == 'encrypted':
parts = self.mail.get_payload()
if len(parts) == 2:
self.body_plain = parts[1].get_payload()
return
else:
log.warn('Encrypted message with invalid parts count')
for top_level_part in self.mail.get_payload():
if top_level_part.get_content_maintype() == "multipart":
for alternative in top_level_part.get_payload():
charset = alternative.get_param("charset")
if isinstance(charset, tuple):
charset = unicode(charset[2],
charset[0] or "us-ascii")
if alternative.get_content_type() == "text/plain":
body_plain = alternative.get_payload(
decode=True)
self.body_plain = to_utf8(body_plain, charset)
elif alternative.get_content_type() == "text/html":
body_html = alternative. \
get_payload(decode=True)
self.body_html = to_utf8(body_html, charset)
break
else:
charset = top_level_part.get_param("charset")
if isinstance(charset, tuple):
charset = unicode(charset[2],
charset[0] or "us-ascii")
if top_level_part.get_content_type() == "text/plain":
body_plain = top_level_part. \
get_payload(decode=True)
self.body_plain = to_utf8(body_plain, charset)
elif top_level_part.get_content_type() == "text/html":
body_html = top_level_part.get_payload(decode=True)
self.body_html = to_utf8(body_html, charset)
else:
charset = self.mail.get_param("charset")
if isinstance(charset, tuple):
charset = unicode(charset[2], charset[0] or "us-ascii")
if self.mail.get_content_type() == "text/html":
body_html = self.mail.get_payload(decode=True)
self.body_html = to_utf8(body_html, charset)
else:
body_plain = self.mail.get_payload(decode=True)
self.body_plain = to_utf8(body_plain, charset)
else:
self.body_plain = self.mail.get_payload(decode=True)
@property
def subject(self):
"""Mail subject."""
s = decode_header(self.mail.get('Subject'))
charset = s[0][1]
if charset is not None:
return s[0][0].decode(charset, "replace"). \
encode("utf-8", "replace")
else:
return s[0][0]
@property
def size(self):
"""Get mail size in bytes."""
return len(self.mail.as_string())
@property
def external_references(self):
"""Return mail references to be used as external references.
making use of RFC5322 headers :
message-id
in-reply-to
references
headers' strings are pruned to extract email addresses only.
"""
ext_id = self.mail.get('Message-Id')
parent_id = self.mail.get('In-Reply-To')
ref = self.mail.get_all("References")
ref_addr = getaddresses(ref) if ref else None
ref_ids = [address[1] for address in ref_addr] if ref_addr else []
mid = clean_email_address(ext_id)[1] if ext_id else None
pid = clean_email_address(parent_id)[1] if parent_id else None
return {
'message_id': mid,
'parent_id': pid,
'ancestors_ids': ref_ids}
@property
def date(self):
"""Get UTC date from a mail message."""
mail_date = self.mail.get('Date')
if mail_date:
tmp_date = parsedate_tz(mail_date)
return datetime.datetime.fromtimestamp(mktime_tz(tmp_date))
log.debug('No date on mail using now (UTC)')
return datetime.datetime.now(tz=pytz.utc)
@property
def participants(self):
"""Mail participants."""
participants = []
for header in self.recipient_headers:
addrs = []
participant_type = header.capitalize()
if self.mail.get(header):
if ',' in self.mail.get(header):
parts = self.mail.get(header).split(',')
filtered = [x for x in parts if '@' in x]
addrs.extend(filtered)
else:
addrs.append(self.mail.get(header))
for addr in addrs:
participant = MailParticipant(participant_type, addr)
participants.append(participant)
return participants
@property
def attachments(self):
"""Extract parts which we consider as attachments."""
if not self.mail.is_multipart():
return []
attchs = []
for p in walk_with_boundary(self.mail, ""):
if not p.is_multipart():
if MailAttachment.is_attachment(p):
attchs.append(MailAttachment(p))
return attchs
@property
def extra_parameters(self):
"""Mail message extra parameters."""
lists = self.mail.get_all("List-ID")
lists_addr = getaddresses(lists) if lists else None
lists_ids = [address[1] for address in lists_addr] \
if lists_addr else []
return {'lists': lists_ids}
# Others parameters specific for mail message
@property
def headers(self):
"""Extract all headers into list.
Duplicate on headers exists, group them by name
with a related list of values
"""
def keyfunc(item):
return item[0]
# Group multiple value for same headers into a dict of list
headers = {}
data = sorted(self.mail.items(), key=keyfunc)
for k, g in groupby(data, key=keyfunc):
headers[k] = [x[1] for x in g]
return headers
class MailMessage(object):
"""
Mail message structure.
Got a mail in raw rfc2822 format, parse it to
resolve all recipients emails, parts and group headers
"""
recipient_headers = ['From', 'To', 'Cc', 'Bcc']
message_type = 'mail'
def __init__(self, raw):
"""Initialize structure from a raw mail."""
try:
self.mail = Message(raw)
except Exception as exc:
log.error('Parse message failed %s' % exc)
raise
if self.mail.defects:
# XXX what to do ?
log.warn('Defects on parsed mail %r' % self.mail.defects)
self.recipients = self._extract_recipients()
self.parts = self._extract_parts()
self.headers = self._extract_headers()
self.subject = self.mail.get('Subject')
tmp_date = parsedate_tz(self.mail['Date'])
self.date = datetime.fromtimestamp(mktime_tz(tmp_date))
self.external_message_id = self.mail.get('Message-Id')
self.external_parent_id = self.mail.get('In-Reply-To')
self.size = len(raw)
@property
def text(self):
"""Message all text."""
# XXX : more complexity ?
return "\n".join([x.data for x in self.parts if x.can_index])
def _extract_recipients(self):
recip = {}
for header in self.recipient_headers:
addrs = []
recipient_type = header.lower()
if self.mail.get(header):
if ',' in self.mail.get(header):
addrs.extend(self.mail.get(header).split(','))
else:
addrs.append(self.mail.get(header))
addrs = [clean_email_address(x) for x in addrs]
recip[recipient_type] = addrs
return recip
def _extract_headers(self):
"""
Extract all headers into list.
Duplicate on headers exists, group them by name
with a related list of values
"""
def keyfunc(item):
return item[0]
# Group multiple value for same headers into a dict of list
headers = {}
data = sorted(self.mail.items(), key=keyfunc)
for k, g in groupby(data, key=keyfunc):
headers[k] = [x[1] for x in g]
return headers
def _extract_parts(self):
"""Multipart message, extract parts."""
parts = []
for p in self.mail.walk():
if not p.is_multipart():
parts.append(self._process_part(p))
return parts
def _process_part(self, part):
return MailPart(part)
@property
def transport_privacy_index(self):
"""Evaluate transport privacy index."""
# XXX : TODO
return random.randint(0, 50)
@property
def content_privacy_index(self):
"""Evaluate content privacy index."""
# XXX: real evaluation needed ;)
if 'PGP' in [x.content_type for x in self.parts]:
return random.randint(50, 100)
else:
return 0.0
@property
def spam_level(self):
"""Report spam level."""
try:
score = self.headers.get('X-Spam-Score')
score = float(score[0])
except:
score = 0.0
if score < 5.0:
return 0.0
if score >= 5.0 and score < 15.0:
return min(score * 10, 100.0)
return 100.0
@property
def importance_level(self):
"""Return percent estimated importance level of this message."""
# XXX. real compute needed
return 0 if self.spam_level else random.randint(50, 100)
@property
def lists(self):
"""List related to message."""
lists = []
for list_name in self.headers.get('List-ID', []):
lists.append(list_name)
return lists
@property
def from_(self):
"""Get from recipient."""
from_ = self.recipients.get('from')
if from_:
# XXX should do better
return from_[0][1]
return None
def lookup_sequence(self):
"""Build parameter sequence for lookups."""
seq = []
# first from parent
if self.external_parent_id:
seq.append(('parent', self.external_parent_id))
# then list lookup
for listname in self.lists:
seq.append(('list', listname))
# last try to lookup from sender address
if self.from_:
seq.append(('from', self.from_))
return seq
def to_parameter(self):
"""Transform mail to a NewMessage parameter."""
msg = NewMessage()
msg.type = 'email'
msg.subject = self.subject
msg.from_ = self.from_
# XXX need transform to part parameter
for part in self.parts:
param = Part()
param.content_type = part.content_type
param.data = part.data
param.size = part.size
param.filename = part.filename
param.can_index = part.can_index
msg.parts.append(param)
msg.headers = self.headers
msg.date = self.date
msg.size = self.size
msg.text = self.text
msg.external_parent_id = self.external_parent_id
msg.external_message_id = self.external_message_id
# XXX well ....
msg.privacy_index = (self.transport_privacy_index +
self.content_privacy_index) / 2
msg.importance_level = self.importance_level
return msg
def as_message(self):
"""Returns the mail as an instance of ``mailbox.Message``."""
return Message(self.render())
def __init__(self, raw):
try:
self.mail = Message(raw)
except Exception, exc:
log.error('Parse message failed %s' % exc)
raise
