def can_sort_by_pubdate( self):
for episode in self:
try:
mktime_tz(parsedate_tz( episode.pubDate))
except:
log('Episode %s has non-parseable pubDate. Sorting disabled.', episode.title)
return False
return True
def compare_pubdate( self, pubdate):
try:
timestamp_self=int(mktime_tz( parsedate_tz( self.pubDate)))
except:
return -1
try:
timestamp_other=int(mktime_tz( parsedate_tz( pubdate)))
except:
return 1
return timestamp_self - timestamp_other
def __cmp__( self, other):
if self.pubDate == other.pubDate:
log('pubDate equal, comparing titles (buggy feed?)', sender=self)
return cmp(self.title, other.title)
try:
timestamp_self=int(mktime_tz( parsedate_tz( self.pubDate)))
timestamp_other=int(mktime_tz( parsedate_tz( other.pubDate)))
except:
# by default, do as if this is not the same
# this is here so that comparisons with None
# can be allowed (item is not None -> True)
return -1
return timestamp_self - timestamp_other
def renderHTTP(self, ctx):
"""
Checks the file modification time from the client against the
modification time on the server. Also checks the file cache to
see if it needs to be refreshed.
"""
request = inevow.IRequest(ctx)
self._check_cache()
temp_date = request.getHeader("If-Modified-Since")
if temp_date:
parsed_date = parsedate_tz(temp_date)
if parsed_date:
client_time = mktime_tz(parsed_date)
else:
client_time = 0
else:
client_time = 0
##
## Check to see if the file has been modified since the client's mtime
##
request.setHeader('content-type', "text/javascript; charset=UTF-8")
if self.mtime > client_time:
## New version available.
request.setLastModified(self.mtime)
stamp = datetime.now() + timedelta(365)
request.setHeader('expires', datetimeToString(mktime(stamp.timetuple())))
request.setHeader('content-length', str(len(self.data)))
request.setHeader('content-type', "text/javascript; charset=UTF-8")
return self.data
else:
request.setResponseCode(http.NOT_MODIFIED)
return "OK"
def was_modified_since(header=None, mtime=0, size=0):
"""
Was something modified since the user last downloaded it?
header
This is the value of the If-Modified-Since header. If this is None,
I'll just return True.
mtime
This is the modification time of the item we're talking about.
size
This is the size of the item we're talking about.
"""
try:
if header is None:
raise ValueError
matches = re.match('^([^;]+)(; length=([0-9]+))?$', header,
re.IGNORECASE)
header_mtime = mktime_tz(parsedate_tz(matches.group(1)))
header_len = matches.group(3)
if header_len and int(header_len) != size:
raise ValueError
if mtime > header_mtime:
raise ValueError
except (AttributeError, ValueError):
return True
return False
def get_time(fpath):
fh = file(fpath, 'rb')
lines = iter(fh)
for line in lines:
if line.lower().startswith("received:"):
break
else:
print "\nNo Received header found."
fh.close()
return None
received = line
for line in lines:
if line[0] in ' \t':
received += line
else:
break
fh.close()
i = received.rfind(';')
if i < 0:
print "\n" + received
print "No semicolon found in Received header."
return None
datestring = received[i+1:]
datestring = ' '.join(datestring.split())
as_tuple = parsedate_tz(datestring)
if as_tuple is None:
print "\n" + received
print "Couldn't parse the date: %r" % datestring
return None
return mktime_tz(as_tuple)
def purgeGroups(self):
purge_read_limit = int(self._configs["purge_read"])
purge_read_limit_secs = purge_read_limit * 24 * 60 * 60
purge_unread_limit = int(self._configs["purge_unread"])
purge_unread_limit_secs = purge_unread_limit * 24 * 60 * 60
time_now = mktime_tz(parsedate_tz(time.ctime()))
subscribed = self.getSubscribed()
for group in subscribed:
c = self._getCursor(group[0])
conn = self._getConnection(group[0])
if purge_read_limit_secs > 0:
c.execute(
"""delete from articles where secs < ? and read='1' and kept='0'""",
(time_now - purge_read_limit_secs, ))
conn.commit()
if purge_unread_limit_secs > 0:
c.execute(
"""delete from articles where secs < ? and read='0' and kept='0'""",
(time_now - purge_unread_limit_secs, ))
conn.commit()
c.execute("""vacuum""")
conn.commit()
yield group[0]
def renderHTTP(self, ctx):
"""
Checks the file modification time from the client against the
modification time on the server. Also checks the file cache to
see if it needs to be refreshed.
"""
request = inevow.IRequest(ctx)
self._check_cache()
temp_date = request.getHeader("If-Modified-Since")
if temp_date:
parsed_date = parsedate_tz(temp_date)
if parsed_date:
client_time = mktime_tz(parsed_date)
else:
client_time = 0
else:
client_time = 0
##
## Check to see if the file has been modified since the client's mtime
##
request.setHeader('content-type', "text/javascript")
if self.mtime > client_time:
## New version available.
request.setLastModified(self.mtime)
stamp = datetime.now() + timedelta(365)
request.setHeader('expires',
datetimeToString(mktime(stamp.timetuple())))
request.setHeader('content-length', str(len(self.data)))
request.setHeader('content-type', "text/javascript")
return self.data
else:
request.setResponseCode(http.NOT_MODIFIED)
return "OK"
def get_time(fpath):
fh = file(fpath, 'rb')
lines = iter(fh)
# Find first Received header.
for line in lines:
if line.lower().startswith("received:"):
break
else:
print "\nNo Received header found."
fh.close()
return None
# Paste on continuation lines, if any.
received = line
for line in lines:
if line[0] in ' \t':
received += line
else:
break
fh.close()
# RFC 2822 says the date-time field must follow a semicolon at the end.
i = received.rfind(';')
if i < 0:
print "\n" + received
print "No semicolon found in Received header."
return None
# We only want the part after the semicolon.
datestring = received[i+1:]
# It may still be split across lines (like "Wed, \r\n\t22 Oct ...").
datestring = ' '.join(datestring.split())
as_tuple = parsedate_tz(datestring)
if as_tuple is None:
print "\n" + received
print "Couldn't parse the date: %r" % datestring
return None
return mktime_tz(as_tuple)
def _check_cache_date(self, media_info):
"""
Checks the If-Modified-Since header, if supplied by the client,
and determines whether the client's cached copy is recent enough
to be used.
@param media_info: Information about the media being served.
@type media_info: Dictionary
"""
if not media_info:
return False
##
## See if this is a cache-freshness query
##
time_stamp = self.request.getHeader("If-Modified-Since")
if time_stamp:
parsed_date = parsedate_tz(time_stamp)
if parsed_date:
header_time = mktime_tz(parsed_date)
else:
header_time = 0
file_time = time.mktime(media_info['updated'].timetuple())
if file_time == header_time:
## Client's cache is fresh
self.request.setResponseCode(http.NOT_MODIFIED)
self.request.setHeader('content-type', 'image/jpeg')
self.request.setHeader('content-length', 0)
self.request.setLastModified(file_time)
return True
# Catchall
return False
def getmail(user, password, gmailfolder):
conn = imaplib.IMAP4_SSL("imap.gmail.com", 993)
conn.login(user, password)
conn.select(gmailfolder) # mailbox/gmail label
# print conn.status(gmailfolder,"(MESSAGES)")#print message count
since = datetime.datetime.today() - datetime.timedelta(days=90)
since = since.strftime("%d-%b-%Y")
dates = []
typ, mailnumlist = conn.search(None, "SINCE", since) # or ALL
typ, mailheaders = conn.fetch(mailnumlist[0].replace(" ", ","), "(BODY.PEEK[HEADER])")
for response_part in mailheaders:
if isinstance(response_part, tuple):
header_data = response_part[1]
headerparser = HeaderParser()
msg = headerparser.parsestr(header_data)
# Convert long form date into UTC timestamp while considering tz
msgdatetimestamp = parsedate_tz(msg["Date"])
# Discard hours/minutes/seconds as we are aggregating by whole dates
msgdate = msgdatetimestamp[0:3] + (0, 0, 0, 0, 0, 0, 0, -18000)
# Format cleanly for output as Javascript timestamp
dates.append(str(int(mktime_tz(msgdate))))
conn.close()
datesdict = {}
output = ""
alldates = [key for key, _ in groupby(dates)]
for i in alldates:
datesdict[i] = dates.count(i)
output += '{ "label": "Email", \n'
output += ' "data": [\n'
for k, v in sorted(datesdict.items()):
output += " [%s, %s],\n" % (k, v)
output = output[:-2] + "\n]}"
return output
def was_modified_since(header=None, mtime=0, size=0):
"""
Was something modified since the user last downloaded it?
header
This is the value of the If-Modified-Since header. If this is None,
I'll just return True.
mtime
This is the modification time of the item we're talking about.
size
This is the size of the item we're talking about.
"""
try:
if header is None:
raise ValueError
matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header,
re.IGNORECASE)
header_mtime = mktime_tz(parsedate_tz(matches.group(1)))
header_len = matches.group(3)
if header_len and int(header_len) != size:
raise ValueError
if mtime > header_mtime:
raise ValueError
except (AttributeError, ValueError):
return True
return False
def get_time(fpath):
fh = file(fpath, 'rb')
lines = iter(fh)
# Find first Received header.
for line in lines:
if line.lower().startswith("received:"):
break
else:
print "\nNo Received header found."
fh.close()
return None
# Paste on continuation lines, if any.
received = line
for line in lines:
if line[0] in ' \t':
received += line
else:
break
fh.close()
# RFC 2822 says the date-time field must follow a semicolon at the end.
i = received.rfind(';')
if i < 0:
print "\n" + received
print "No semicolon found in Received header."
return None
# We only want the part after the semicolon.
datestring = received[i+1:]
# It may still be split across lines (like "Wed, \r\n\t22 Oct ...").
datestring = ' '.join(datestring.split())
as_tuple = parsedate_tz(datestring)
if as_tuple is None:
print "\n" + received
print "Couldn't parse the date: %r" % datestring
return None
return mktime_tz(as_tuple)
def getStartDate(self, default=_MARKER):
"""
Returns the date from the mail date
"""
if not self.hasFile():
# Return the standard start date if no file was provided
if default is _MARKER:
return self._baseGetStartDate()
else:
return self._baseGetStartDate(default)
date_string = self.getContentInformation().get('Date', None)
if date_string:
parsed_date_string = parsedate_tz(date_string)
if parsed_date_string is not None:
try:
try:
time = mktime_tz(parsed_date_string)
except OverflowError: # No way to get a date - parsing failed (XXX)
time = None
if time:
return DateTime(time)
except ValueError:
# If the parsed date is not conformant, nothing to do
pass
return self.getCreationDate()
def apply_action_rules(self,xpn_article,group):
score=xpn_article.get_score()
i=0
time_now=mktime_tz(parsedate_tz(time.ctime()))
age=(time_now-xpn_article.secs)/(24.0*60*60)
#compatibility check
#there is no need here, it's has been already performed in apply_score_rules
xpost=xpn_article.xref.count(":")
fields={"subject":xpn_article.subj,"from":xpn_article.from_name,"date":xpn_article.date,
"message-id":xpn_article.msgid,"references":xpn_article.ref,"age":age,"xref":xpn_article.xref,
"bytes":xpn_article.bytes,"lines":xpn_article.lines,"xpost":xpost,"score":score}
actions=[]
for group_scope in self.group_scope_list:
if group_scope=="*" or (group in group_scope):
#current group is in this scope
for rule in self.group_rules_list[i]:
#let's apply rule
match_rule=self.build_match_rule(rule["match"])
if match_rule(fields[rule["header"]],rule["match"]):
action=rule["score_mod"]
if action[0]=="!":
#this is an action
if action[1:]=="kill":
#xpn_article=None
pass #need this to let reapply_rules work
elif action[1:]=="markread":
xpn_article.is_read=True
elif action[1:]=="markunread":
xpn_article.is_read=False
elif action[1:]=="mark":
xpn_article.marked_for_download=True
elif action[1:]=="unmark":
xpn_article.marked_for_download=False
elif action[1:]=="keep":
xpn_article.keep=True
elif action[1:]=="unkeep":
xpn_article.keep=False
elif action[1:]=="watch":
xpn_article.watch=True
xpn_article.ignore=False
elif action[1:]=="ignore":
xpn_article.watch=False
xpn_article.ignore=True
xpn_article.is_read=True
elif action[1:]=="unsetwatchignore":
xpn_article.watch=False
xpn_article.ignore=False
elif action[1:].startswith("setcolor"):
regex=re.compile("(^setcolor)\((([a-z]+)|(#\w+));(([a-z]+)|(#\w+))\)")
matches=regex.match(action[1:])
xpn_article.fg_color=str(matches.groups()[1])
xpn_article.bg_color=str(matches.groups()[4])
actions.append(action[1:])
i=i+1
return xpn_article,actions
def getLatestUploadtime(self):
result, mailCount = self.mail.select(mailBoxLabel)
if result == 'OK':
pass
elif result == 'NO' and mailCount[0][0:13] == '[NONEXISTENT]':
result, status = self.mail.create(mailBoxLabel)
if result != 'OK':
print "mailbox can not be created."
return None
else:
print "mailbox created"
self.mail.select(mailBoxLabel)
mailCount = ['0']
else:
print mailCount
return None
if mailCount == ['0']: return 0
msgUIDs = ['']
""" initial search time within 1 day """
expandTime = 86400
while (msgUIDs == ['']):
""" check for the maximum time """
if expandTime > 88473600:
return (time.time() - expandTime)
sinceTime = imaplib.Time2Internaldate(time.time() - expandTime)
result, msgUIDs = self.mail.search(None, 'SINCE', sinceTime)
expandTime = expandTime * 2
idString = ','.join(msgUIDs[0].split())
result, msgIDs = self.mail.fetch(idString,
'(BODY[HEADER.FIELDS (MESSAGE-ID)])')
latest = 0
for msgID in msgIDs:
if len(msgID) < 2: continue
try:
epoch = msgID[1].split('.')[1][0:13]
except:
latest = -1
break
if epoch > latest:
latest = epoch
if latest == -1:
print "unrecognized message id"
result, msgDate = self.mail.fetch(idString, '(INTERNALDATE)')
latest = 0
for date in msgDate:
_date = date.split('"')[1]
seconds = mktime_tz(parsedate_tz(_date))
if seconds > latest:
latest = seconds
latest = latest * 1000
return latest
def floatdate(header):
missing = []
datestr = message.get(header, missing)
if datestr is missing:
return None
date = parsedate_tz(datestr)
try:
return mktime_tz(date)
except (TypeError, ValueError, OverflowError):
return None
def floatdate(header):
missing = []
datestr = message.get(header, missing)
if datestr is missing:
return None
date = parsedate_tz(datestr)
try:
return mktime_tz(date)
except (TypeError, ValueError, OverflowError):
return None
def __mktime__(self, date):
if isinstance(date, float) or isinstance(date, int):
return date
if date is None or '' == date:
return None
try:
return mktime_tz(parsedate_tz(date))
except TypeError:
log('Could not convert "%s" to a unix timestamp.', date)
return None
def getLatestUploadtime(self):
result, mailCount = self.mail.select(mailBoxLabel)
if result == 'OK':
pass
elif result == 'NO' and mailCount[0][0:13] == '[NONEXISTENT]':
result, status = self.mail.create(mailBoxLabel)
if result != 'OK':
print "mailbox can not be created."
return None
else:
print "mailbox created"
self.mail.select(mailBoxLabel)
mailCount = ['0']
else:
print mailCount
return None
if mailCount == ['0']: return 0
msgUIDs = ['']
""" initial search time within 1 day """
expandTime = 86400
while (msgUIDs == ['']):
""" check for the maximum time """
if expandTime > 88473600:
return (time.time() - expandTime)
sinceTime = imaplib.Time2Internaldate(time.time() - expandTime)
result, msgUIDs = self.mail.search(None, 'SINCE', sinceTime)
expandTime = expandTime * 2
idString = ','.join(msgUIDs[0].split())
result, msgIDs = self.mail.fetch(idString, '(BODY[HEADER.FIELDS (MESSAGE-ID)])')
latest = 0
for msgID in msgIDs:
if len(msgID) < 2: continue
try:
epoch = msgID[1].split('.')[1][0:13]
except:
latest = -1
break
if epoch > latest:
latest = epoch
if latest == -1:
print "unrecognized message id"
result, msgDate = self.mail.fetch(idString, '(INTERNALDATE)')
latest = 0
for date in msgDate:
_date = date.split('"')[1]
seconds = mktime_tz(parsedate_tz(_date))
if seconds > latest:
latest = seconds
latest = latest * 1000
return latest
def cute_pubdate( self):
try:
timestamp=int(mktime_tz(parsedate_tz(self.pubDate)))
except:
timestamp=None
result=util.format_date(timestamp)
if result is None:
return '(%s)' % _('unknown')
else:
return result
def __init__(self, message, hash_value, uid):
self.hash_value = hash_value
self.uid = uid
message = email.message_from_string(message)
def _get_header_data(key):
value = message.get(key, '<unknown>')
value = decode_mail_headers(decodeUnknown(message.get_charset(), value))
return value
self.subject = _get_header_data('subject').strip()
self.sender = parseaddr(message.get('from'))
received = None
date_str= message.get('date')
if date_str:
date_tuple= parsedate_tz(date_str)
if date_tuple:
received=datetime.datetime.fromtimestamp(mktime_tz(date_tuple))
self.received = received
self.recipients = getaddresses(message.get_all('to', []) + message.get_all('cc', []) + message.get_all('resent-to', []) + message.get_all('resent-cc', []))
counter = 0
files = []
body_plain, body_html = '', ''
for part in message.walk():
if part.get_content_maintype() == 'multipart':
continue
name = part.get_param("name")
if name:
name = collapse_rfc2231_value(name)
if part.get_content_maintype() == 'text' and name == None:
if part.get_content_subtype() == 'plain':
body_plain = decodeUnknown(part.get_content_charset(), part.get_payload(decode=True))
else:
body_html = part.get_payload(decode=True)
else:
if not name:
ext = mimetypes.guess_extension(part.get_content_type())
name = "part-%i%s" % (counter, ext)
files.append({
'filename': name,
'content': part.get_payload(decode=True),
'type': part.get_content_type()},
)
counter += 1
self.body_plain = body_plain
self.body_html = mark_safe(strip_empty_tags(strip_tags(body_html, ['html', 'head', 'body', 'meta'])))
self.files = files
def get_date_header(msg, name):
"""Returns an UTC timestamp from a header field containing a date.
Compensates for the timezone difference if the header contains
timezone information.
"""
value = get_header(msg, name)
ts = None
try:
ts = mktime_tz(parsedate_tz(value))
except TypeError:
pass
return ts
def _modified_since(request, file_modified_time):
modified_since_str = request.env.get('HTTP_IF_MODIFIED_SINCE')
if modified_since_str:
try:
modified_since = mktime_tz(parsedate_tz(modified_since_str))
except:
# couln't parse
return True
if math.floor(file_modified_time) <= modified_since:
return False
return True
def process_message(message):
""" Processes the read message and decodes attachments. """
attachments = []
html_data = []
text_data = []
from_addr = get_addrs(message, 'From')[0]
to_addrs = get_addrs(message, 'To')
cc_addrs = get_addrs(message, 'Cc')
bcc_addrs = get_addrs(message, 'Bcc') # depending on sending MTA, this can be present or not
envelope_to_addrs = get_addrs(message, 'X-Original-To') + get_addrs(message, 'X-Envelope-To') # Postfix / Sendmail does this
target_addrs = to_addrs + cc_addrs + bcc_addrs + envelope_to_addrs
subject = decode_2044(message['Subject'])
date = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(mktime_tz(parsedate_tz(message['Date']))))
log("Processing mail:\n To: %r\n From: %r\n Subject: %r" % (to_addrs[0], from_addr, subject))
for part in message.walk():
log(" Part " + repr((part.get_charsets(), part.get_content_charset(), part.get_content_type(), part.is_multipart(), )))
ct = part.get_content_type()
cs = part.get_content_charset() or "latin1"
payload = part.get_payload(None, True)
fn = part.get_filename()
if fn is not None and fn.startswith("=?"): # heuristics ...
fn = decode_2044(fn)
if fn is None and part["Content-Disposition"] is not None and "attachment" in part["Content-Disposition"]:
# this doesn't catch the case where there is no content-disposition but there is a file to offer to the user
# i hope that this can be only found in mails that are older than 10 years,
# so I won't care about it here
fn = part["Content-Description"] or "NoName"
if fn:
a = attachment(fn, ct, payload)
attachments.append(a)
else:
if ct == 'text/plain':
text_data.append(payload.decode(cs))
log(repr(payload.decode(cs)))
elif ct == 'text/html':
html_data.append(payload.decode(cs))
elif not part.is_multipart():
log("Unknown mail part " + repr((part.get_charsets(), part.get_content_charset(), part.get_content_type(), part.is_multipart(), )))
return {'text': u"".join(text_data), 'html': u"".join(html_data),
'attachments': attachments,
'target_addrs': target_addrs,
'to_addrs': to_addrs, 'cc_addrs': cc_addrs, 'bcc_addrs': bcc_addrs, 'envelope_to_addrs': envelope_to_addrs,
'from_addr': from_addr,
'subject': subject, 'date': date}
def floatdate(datestr):
if not datestr:
return None
date = parsedate_tz(datestr)
try:
date = mktime_tz(date)
if (date < 0 or
date - time.time() >
mm_cfg.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW
):
return None
return date
except (TypeError, ValueError, OverflowError):
return None
def floatdate(datestr):
if not datestr:
return None
date = parsedate_tz(datestr)
try:
date = mktime_tz(date)
if (date < 0 or
date - time.time() >
mm_cfg.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW
):
return None
return date
except (TypeError, ValueError, OverflowError):
return None
def _dispose(self, mlist, msg, msgdata):
# Support clobber_date, i.e. setting the date in the archive to the
# received date, not the (potentially bogus) Date: header of the
# original message.
clobber = 0
originaldate = msg.get('date')
receivedtime = formatdate(msgdata['received_time'])
if not originaldate:
clobber = 1
elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 1:
clobber = 1
elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 2:
# what's the timestamp on the original message?
tup = parsedate_tz(originaldate)
now = time.time()
try:
if not tup:
clobber = 1
elif abs(now - mktime_tz(tup)) > \
mm_cfg.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW:
clobber = 1
except (ValueError, OverflowError):
# The likely cause of this is that the year in the Date: field
# is horribly incorrect, e.g. (from SF bug # 571634):
# Date: Tue, 18 Jun 0102 05:12:09 +0500
# Obviously clobber such dates.
clobber = 1
if clobber:
del msg['date']
del msg['x-original-date']
msg['Date'] = receivedtime
if originaldate:
msg['X-Original-Date'] = originaldate
# Always put an indication of when we received the message.
msg['X-List-Received-Date'] = receivedtime
# Now try to get the list lock
try:
mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
except LockFile.TimeOutError:
# oh well, try again later
return 1
try:
# Archiving should be done in the list's preferred language, not
# the sender's language.
i18n.set_language(mlist.preferred_language)
mlist.ArchiveMail(msg)
mlist.Save()
finally:
mlist.Unlock()
def _was_modified_since(header=None, mtime=0, size=0):
try:
if header is None:
raise ValueError
matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header,
re.IGNORECASE)
header_mtime = mktime_tz(parsedate_tz(matches.group(1)))
header_len = matches.group(3)
if header_len and int(header_len) != size:
raise ValueError
if mtime > header_mtime:
raise ValueError
except (AttributeError, ValueError):
return True
return False
def fromRFC2822(klass, rfc822string):
"""Return a new Time instance from a string formated as described in RFC 2822.
RFC 2822 specifies the format of email messages.
Some of the obsoleted elements of the specification are not parsed
correctly, and will raise ValueError.
"""
date = parsedate_tz(rfc822string)
if date is None:
raise ValueError, 'could not parse RFC 2822 date %r' % (
rfc822string, )
self = klass.fromStructTime(time.gmtime(mktime_tz(date)))
self.resolution = datetime.timedelta(seconds=1)
return self
def _was_modified_since(header=None, mtime=0, size=0):
try:
if header is None:
raise ValueError
matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header,
re.IGNORECASE)
header_mtime = mktime_tz(parsedate_tz(matches.group(1)))
header_len = matches.group(3)
if header_len and int(header_len) != size:
raise ValueError
if mtime > header_mtime:
raise ValueError
except (AttributeError, ValueError):
return True
return False
def utcdatetime_from_field(field_value):
"""Turn an RFC 2822 Date: header value into a Python datetime (UTC).
:param field_value: The value of the Date: header
:type field_value: string
:return: The corresponding datetime (UTC)
:rtype: `datetime.datetime`
:raise `InvalidEmailMessage`: when the date string cannot be converted.
"""
try:
date_tuple = parsedate_tz(field_value)
timestamp = mktime_tz(date_tuple)
return datetime.fromtimestamp(timestamp, tz=pytz.timezone('UTC'))
except (TypeError, ValueError, OverflowError):
raise InvalidEmailMessage('Invalid date %s' % field_value)
def utcdatetime_from_field(field_value):
"""Turn an RFC 2822 Date: header value into a Python datetime (UTC).
:param field_value: The value of the Date: header
:type field_value: string
:return: The corresponding datetime (UTC)
:rtype: `datetime.datetime`
:raise `InvalidEmailMessage`: when the date string cannot be converted.
"""
try:
date_tuple = parsedate_tz(field_value)
timestamp = mktime_tz(date_tuple)
return datetime.fromtimestamp(timestamp, tz=pytz.timezone('UTC'))
except (TypeError, ValueError, OverflowError):
raise InvalidEmailMessage('Invalid date %s' % field_value)
def parse_date(self,date):
#data=parsedate(date)
try: #trying to prevent the rfc822.parsedate bug with Tue,26 instead of Tue, 26
secs=mktime_tz(parsedate_tz(date))
except:
secs=time()
data=parsedate(ctime(secs))
if data[3]<10:
ora="0"+repr(data[3])
else:
ora=repr(data[3])
if data[4]<10:
minuti="0"+repr(data[4])
else:
minuti=repr(data[4])
return repr(data[2])+"/"+repr(data[1])+"/"+repr(data[0])+" "+ora+":"+minuti,secs
def was_modified_since(header=None, mtime=0, size=0):
if header is None:
return True
matches = re.match(r'^([^;]+)(; length=([0-9]+))?$', header, re.IGNORECASE)
header_mtime = mktime_tz(parsedate_tz(matches.group(1)))
header_len = matches.group(3)
if header_len and int(header_len) != size:
return True
if mtime > header_mtime:
return True
return False
def _dispose(self, mlist, msg, msgdata):
# Support clobber_date, i.e. setting the date in the archive to the
# received date, not the (potentially bogus) Date: header of the
# original message.
clobber = 0
originaldate = msg.get('date')
receivedtime = formatdate(msgdata['received_time'])
if not originaldate:
clobber = 1
elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 1:
clobber = 1
elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 2:
# what's the timestamp on the original message?
tup = parsedate_tz(originaldate)
now = time.time()
try:
if not tup:
clobber = 1
elif abs(now - mktime_tz(tup)) > \
mm_cfg.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW:
clobber = 1
except (ValueError, OverflowError):
# The likely cause of this is that the year in the Date: field
# is horribly incorrect, e.g. (from SF bug # 571634):
# Date: Tue, 18 Jun 0102 05:12:09 +0500
# Obviously clobber such dates.
clobber = 1
if clobber:
del msg['date']
del msg['x-original-date']
msg['Date'] = receivedtime
if originaldate:
msg['X-Original-Date'] = originaldate
# Always put an indication of when we received the message.
msg['X-List-Received-Date'] = receivedtime
# Now try to get the list lock
try:
mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
except LockFile.TimeOutError:
# oh well, try again later
return 1
try:
mlist.ArchiveMail(msg)
mlist.Save()
finally:
mlist.Unlock()
def parse_date(self, date):
#data=parsedate(date)
try: #trying to prevent the rfc822.parsedate bug with Tue,26 instead of Tue, 26
secs = mktime_tz(parsedate_tz(date))
except:
secs = time()
data = parsedate(ctime(secs))
if data[3] < 10:
ora = "0" + repr(data[3])
else:
ora = repr(data[3])
if data[4] < 10:
minuti = "0" + repr(data[4])
else:
minuti = repr(data[4])
return repr(data[2]) + "/" + repr(data[1]) + "/" + repr(
data[0]) + " " + ora + ":" + minuti, secs
def was_modified_since(self, mtime=0, size=0):
header = self.request.headers.get('HTTP_IF_MODIFIED_SINCE')
if header is None:
return True
matches = re.match(r'^([^;]+)(; length=([0-9]+))?$', header,
re.IGNORECASE)
header_mtime = mktime_tz(parsedate_tz(matches.group(1)))
header_len = matches.group(3)
if header_len and int(header_len) != size:
return True
if mtime > header_mtime:
return True
return False
def apply_score_rules(self,xpn_article,group):
score=xpn_article.get_score()
i=0
time_now=mktime_tz(parsedate_tz(time.ctime()))
age=(time_now-xpn_article.secs)/(24.0*60*60)
#compatibility check
try: xpn_article.xref
except AttributeError: xpn_article.xref=""
try: xpn_article.bytes
except AttributeError: xpn_article.bytes=0
try: xpn_article.lines
except AttributeError: xpn_article.lines=0
xpost=xpn_article.xref.count(":")
fields={"subject":xpn_article.subj,"from":xpn_article.from_name,"date":xpn_article.date,
"message-id":xpn_article.msgid,"references":xpn_article.ref,"age":age,"xref":xpn_article.xref,
"bytes":xpn_article.bytes,"lines":xpn_article.lines,"xpost":xpost}
for group_scope in self.group_scope_list:
if group_scope=="*" or (group in group_scope):
#current group is in this scope
for rule in self.group_rules_list[i]:
#let's apply rule
score_mod=rule["score_mod"]
if score_mod[0]=="!":
#this is an action
continue
match_rule=self.build_match_rule(rule["match"])
if match_rule(fields[rule["header"]],rule["match"]):
score_mod=rule["score_mod"]
if score_mod[0]=="=":
score=int(score_mod[1:])
if score>9999: score=9999
if score<-9999: score=-9999
return score
else:
score=score+int(score_mod)
i=i+1
if score>9999: score=9999
if score<-9999: score=-9999
return score
def _getDateForComment(self, parsed_comment):
"""Return the correct date for a comment.
:param parsed_comment: An `email.Message.Message` instance
containing a parsed DebBugs comment.
:return: The correct date to use for the comment contained in
`parsed_comment`. If a date is specified in a Received
header on `parsed_comment` that we can use, return that.
Otherwise, return the Date field of `parsed_comment`.
"""
# Check for a Received: header on the comment and use
# that to get the date, if possible. We only use the
# date received by this host (nominally bugs.debian.org)
# since that's the one that's likely to be correct.
received_headers = parsed_comment.get_all('received')
if received_headers is not None:
host_name = urlsplit(self.baseurl)[1]
received_headers = [
header for header in received_headers
if host_name in header]
# If there are too many - or too few - received headers then
# something's gone wrong and we default back to using
# the Date field.
if received_headers is not None and len(received_headers) == 1:
received_string = received_headers[0]
received_by, date_string = received_string.split(';', 2)
else:
date_string = parsed_comment['date']
# We parse the date_string if we can, otherwise we just return
# None.
if date_string is not None:
date_with_tz = parsedate_tz(date_string)
timestamp = mktime_tz(date_with_tz)
msg_date = datetime.fromtimestamp(timestamp,
tz=pytz.timezone('UTC'))
else:
msg_date = None
return msg_date
def _getDateForComment(self, parsed_comment):
"""Return the correct date for a comment.
:param parsed_comment: An `email.Message.Message` instance
containing a parsed DebBugs comment.
:return: The correct date to use for the comment contained in
`parsed_comment`. If a date is specified in a Received
header on `parsed_comment` that we can use, return that.
Otherwise, return the Date field of `parsed_comment`.
"""
# Check for a Received: header on the comment and use
# that to get the date, if possible. We only use the
# date received by this host (nominally bugs.debian.org)
# since that's the one that's likely to be correct.
received_headers = parsed_comment.get_all('received')
if received_headers is not None:
host_name = urlsplit(self.baseurl)[1]
received_headers = [
header for header in received_headers if host_name in header
]
# If there are too many - or too few - received headers then
# something's gone wrong and we default back to using
# the Date field.
if received_headers is not None and len(received_headers) == 1:
received_string = received_headers[0]
received_by, date_string = received_string.split(';', 2)
else:
date_string = parsed_comment['date']
# We parse the date_string if we can, otherwise we just return
# None.
if date_string is not None:
date_with_tz = parsedate_tz(date_string)
timestamp = mktime_tz(date_with_tz)
msg_date = datetime.fromtimestamp(timestamp,
tz=pytz.timezone('UTC'))
else:
msg_date = None
return msg_date
def test_CVSMaildirSource_create_change_from_cvs1_11msg(self):
m = message_from_string(cvs1_11_msg)
src = CVSMaildirSource('/dev/null')
try:
src, chdict = src.parse( m )
except:
self.fail('Failed to get change from email message.')
self.assertNotEqual(chdict, None)
self.assertEqual(chdict['author'], 'andy')
self.assertEqual(len(chdict['files']), 1)
self.assertEqual(chdict['files'][0], 'base/module/src/make/GNUmakefile')
self.assertEqual(chdict['comments'], 'Commented out some stuff.\n')
self.assertFalse(chdict['isdir'])
self.assertEqual(chdict['revision'], '2010-08-07 11:11:49')
dateTuple = parsedate_tz('Sat, 07 Aug 2010 11:11:49 +0000')
self.assertEqual(chdict['when'], mktime_tz(dateTuple))
self.assertEqual(chdict['branch'], None)
self.assertEqual(chdict['repository'], ':ext:cvshost.example.com:/cvsroot')
self.assertEqual(chdict['project'], 'MyModuleName')
self.assertEqual(len(chdict['properties']), 0)
self.assertEqual(src, 'cvs')
def test_CVSMaildirSource_create_change_from_cvs1_11msg(self):
m = message_from_string(cvs1_11_msg)
src = CVSMaildirSource('/dev/null', urlmaker=fileToUrl)
try:
chdict = src.parse( m )
except:
self.fail('Failed to get change from email message.')
self.assert_(chdict != None)
self.assert_(chdict['who'] == 'andy')
self.assert_(len(chdict['files']) == 1)
self.assert_(chdict['files'][0] == 'base/module/src/make/GNUmakefile')
self.assert_(chdict['comments'] == 'Commented out some stuff.\n')
self.assert_(chdict['isdir'] == False)
self.assert_(len(chdict['links']) == 1)
self.assert_(chdict['revision'] == '2010-08-07 11:11:49')
dateTuple = parsedate_tz('Sat, 07 Aug 2010 11:11:49 +0000')
self.assert_(chdict['when'] == mktime_tz(dateTuple))
self.assert_(chdict['branch'] == None)
self.assert_(chdict['repository'] == ':ext:cvshost.example.com:/cvsroot')
self.assert_(chdict['project'] == 'MyModuleName')
self.assert_(len(chdict['properties']) == 0)
def test_CVSMaildirSource_create_change_from_cvs1_11msg(self):
m = message_from_string(cvs1_11_msg)
src = CVSMaildirSource('/dev/null', urlmaker=fileToUrl)
try:
chdict = src.parse(m)
except:
self.fail('Failed to get change from email message.')
self.assert_(chdict != None)
self.assert_(chdict['who'] == 'andy')
self.assert_(len(chdict['files']) == 1)
self.assert_(chdict['files'][0] == 'base/module/src/make/GNUmakefile')
self.assert_(chdict['comments'] == 'Commented out some stuff.\n')
self.assert_(chdict['isdir'] == False)
self.assert_(len(chdict['links']) == 1)
self.assert_(chdict['revision'] == '2010-08-07 11:11:49')
dateTuple = parsedate_tz('Sat, 07 Aug 2010 11:11:49 +0000')
self.assert_(chdict['when'] == mktime_tz(dateTuple))
self.assert_(chdict['branch'] == None)
self.assert_(
chdict['repository'] == ':ext:cvshost.example.com:/cvsroot')
self.assert_(chdict['project'] == 'MyModuleName')
self.assert_(len(chdict['properties']) == 0)
def test_CVSMaildirSource_create_change_from_cvs1_11msg(self):
m = message_from_string(cvs1_11_msg)
src = CVSMaildirSource('/dev/null', urlmaker=fileToUrl)
try:
change = src.parse(m)
except:
self.fail('Failed to get change from email message.')
self.assert_(change != None)
self.assert_(change.who == 'andy')
self.assert_(len(change.files) == 1)
self.assert_(change.files[0] == 'base/module/src/make/GNUmakefile')
self.assert_(change.comments == 'Commented out some stuff.\n')
self.assert_(change.isdir == False)
self.assert_(len(change.links) == 1)
self.assert_(change.revision == '2010-08-07 11:11:49')
dateTuple = parsedate_tz('Sat, 07 Aug 2010 11:11:49 +0000')
self.assert_(change.when == mktime_tz(dateTuple))
self.assert_(change.branch == None)
self.assert_(change.revlink == '')
self.assert_(change.repository == ':ext:cvshost.example.com:/cvsroot')
self.assert_(change.project == 'MyModuleName')
propList = change.properties.asList()
self.assert_(len(propList) == 0)
def test_CVSMaildirSource_create_change_from_cvs1_12msg(self):
m = message_from_string(cvs1_12_msg)
src = CVSMaildirSource('/dev/null', urlmaker=fileToUrl)
try:
chdict = src.parse(m)
except:
self.fail('Failed to get change from email message.')
self.assert_(chdict != None)
self.assert_(chdict['who'] == 'andy')
self.assert_(len(chdict['files']) == 2)
self.assert_(chdict['files'][0] == 'base/module/src/file1.cpp')
self.assert_(chdict['files'][1] == 'base/module/src/file2.cpp')
self.assert_(chdict['comments'] == 'Changes for changes sake\n')
self.assert_(chdict['isdir'] == False)
self.assert_(len(chdict['links']) == 2)
self.assert_(chdict['revision'] == '2010-08-11 04:56:44')
dateTuple = parsedate_tz('Wed, 11 Aug 2010 04:56:44 +0000')
self.assert_(chdict['when'] == mktime_tz(dateTuple))
self.assert_(chdict['branch'] == None)
self.assert_(
chdict['repository'] == ':ext:cvshost.example.com:/cvsroot')
self.assert_(chdict['project'] == 'MyModuleName')
self.assert_(len(chdict['properties']) == 0)
def test_CVSMaildirSource_create_change_from_cvs1_12msg(self):
m = message_from_string(cvs1_12_msg)
src = CVSMaildirSource('/dev/null')
try:
src, chdict = src.parse(m)
except:
self.fail('Failed to get change from email message.')
self.assertNotEqual(chdict, None)
self.assertEqual(chdict['author'], 'andy')
self.assertEqual(len(chdict['files']), 2)
self.assertEqual(chdict['files'][0], 'base/module/src/file1.cpp')
self.assertEqual(chdict['files'][1], 'base/module/src/file2.cpp')
self.assertEqual(chdict['comments'], 'Changes for changes sake\n')
self.assertFalse(chdict['isdir'])
self.assertEqual(chdict['revision'], '2010-08-11 04:56:44')
dateTuple = parsedate_tz('Wed, 11 Aug 2010 04:56:44 +0000')
self.assertEqual(chdict['when'], mktime_tz(dateTuple))
self.assertEqual(chdict['branch'], None)
self.assertEqual(chdict['repository'],
':ext:cvshost.example.com:/cvsroot')
self.assertEqual(chdict['project'], 'MyModuleName')
self.assertEqual(len(chdict['properties']), 0)
self.assertEqual(src, 'cvs')
def test_CVSMaildirSource_create_change_from_cvs1_11msg(self):
m = message_from_string(cvs1_11_msg)
src = CVSMaildirSource('/dev/null')
try:
src, chdict = src.parse(m)
except:
self.fail('Failed to get change from email message.')
self.assertNotEqual(chdict, None)
self.assertEqual(chdict['author'], 'andy')
self.assertEqual(len(chdict['files']), 1)
self.assertEqual(chdict['files'][0],
'base/module/src/make/GNUmakefile')
self.assertEqual(chdict['comments'], 'Commented out some stuff.\n')
self.assertFalse(chdict['isdir'])
self.assertEqual(chdict['revision'], '2010-08-07 11:11:49')
dateTuple = parsedate_tz('Sat, 07 Aug 2010 11:11:49 +0000')
self.assertEqual(chdict['when'], mktime_tz(dateTuple))
self.assertEqual(chdict['branch'], None)
self.assertEqual(chdict['repository'],
':ext:cvshost.example.com:/cvsroot')
self.assertEqual(chdict['project'], 'MyModuleName')
self.assertEqual(len(chdict['properties']), 0)
self.assertEqual(src, 'cvs')
def test_CVSMaildirSource_create_change_from_cvs1_12msg(self):
m = message_from_string(cvs1_12_msg)
src = CVSMaildirSource('/dev/null', urlmaker=fileToUrl)
try:
change = src.parse(m)
except:
self.fail('Failed to get change from email message.')
self.assert_(change != None)
self.assert_(change.who == 'andy')
self.assert_(len(change.files) == 2)
self.assert_(change.files[0] == 'base/module/src/file1.cpp')
self.assert_(change.files[1] == 'base/module/src/file2.cpp')
self.assert_(change.comments == 'Changes for changes sake\n')
self.assert_(change.isdir == False)
self.assert_(len(change.links) == 2)
self.assert_(change.revision == '2010-08-11 04:56:44')
dateTuple = parsedate_tz('Wed, 11 Aug 2010 04:56:44 +0000')
self.assert_(change.when == mktime_tz(dateTuple))
self.assert_(change.branch == None)
self.assert_(change.revlink == '')
self.assert_(change.repository == ':ext:cvshost.example.com:/cvsroot')
self.assert_(change.project == 'MyModuleName')
propList = change.properties.asList()
self.assert_(len(propList) == 0)
def cookieDate2Ts(self, cd):
#Sat, 02-Nov-2013 11:33:51 GMT
if cd.strip() != "" and cd != None:
return mktime_tz(parsedate_tz(cd.strip()))
return long(0)
def parse(self, m, prefix=None):
"""Parse messages sent by the 'buildbot-cvs-mail' program.
"""
# The mail is sent from the person doing the checkin. Assume that the
# local username is enough to identify them (this assumes a one-server
# cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
# model)
name, addr = parseaddr(m["from"])
if not addr:
return None # no From means this message isn't from buildbot-cvs-mail
at = addr.find("@")
if at == -1:
author = addr # might still be useful
else:
author = addr[:at]
# CVS accecpts RFC822 dates. buildbot-cvs-mail adds the date as
# part of the mail header, so use that.
# This assumes cvs is being access via ssh or pserver, so the time
# will be the CVS server's time.
# calculate a "revision" based on that timestamp, or the current time
# if we're unable to parse the date.
log.msg('Processing CVS mail')
dateTuple = parsedate_tz(m["date"])
if dateTuple == None:
when = util.now()
else:
when = mktime_tz(dateTuple)
theTime = datetime.datetime.utcfromtimestamp(float(when))
rev = theTime.strftime('%Y-%m-%d %H:%M:%S')
catRE = re.compile('^Category:\s*(\S.*)')
cvsRE = re.compile('^CVSROOT:\s*(\S.*)')
cvsmodeRE = re.compile('^Cvsmode:\s*(\S.*)')
filesRE = re.compile('^Files:\s*(\S.*)')
modRE = re.compile('^Module:\s*(\S.*)')
pathRE = re.compile('^Path:\s*(\S.*)')
projRE = re.compile('^Project:\s*(\S.*)')
singleFileRE = re.compile('(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)')
tagRE = re.compile('^\s+Tag:\s*(\S.*)')
updateRE = re.compile('^Update of:\s*(\S.*)')
comments = ""
branch = None
cvsroot = None
fileList = None
files = []
isdir = 0
path = None
project = None
lines = list(body_line_iterator(m))
while lines:
line = lines.pop(0)
m = catRE.match(line)
if m:
category = m.group(1)
continue
m = cvsRE.match(line)
if m:
cvsroot = m.group(1)
continue
m = cvsmodeRE.match(line)
if m:
cvsmode = m.group(1)
continue
m = filesRE.match(line)
if m:
fileList = m.group(1)
continue
m = modRE.match(line)
if m:
# We don't actually use this
#module = m.group(1)
continue
m = pathRE.match(line)
if m:
path = m.group(1)
continue
m = projRE.match(line)
if m:
project = m.group(1)
continue
m = tagRE.match(line)
if m:
branch = m.group(1)
continue
m = updateRE.match(line)
if m:
# We don't actually use this
#updateof = m.group(1)
continue
if line == "Log Message:\n":
break
# CVS 1.11 lists files as:
# repo/path file,old-version,new-version file2,old-version,new-version
# Version 1.12 lists files as:
# file1 old-version new-version file2 old-version new-version
#
# files consists of tuples of 'file-name old-version new-version'
# The versions are either dotted-decimal version numbers, ie 1.1
# or NONE. New files are of the form 'NONE NUMBER', while removed
# files are 'NUMBER NONE'. 'NONE' is a literal string
# Parsing this instead of files list in 'Added File:' etc
# makes it possible to handle files with embedded spaces, though
# it could fail if the filename was 'bad 1.1 1.2'
# For cvs version 1.11, we expect
# my_module new_file.c,NONE,1.1
# my_module removed.txt,1.2,NONE
# my_module modified_file.c,1.1,1.2
# While cvs version 1.12 gives us
# new_file.c NONE 1.1
# removed.txt 1.2 NONE
# modified_file.c 1.1,1.2
if fileList is None:
log.msg('CVSMaildirSource Mail with no files. Ignoring')
return None # We don't have any files. Email not from CVS
if cvsmode == '1.11':
# Please, no repo paths with spaces!
m = re.search('([^ ]*) ', fileList)
if m:
path = m.group(1)
else:
log.msg(
'CVSMaildirSource can\'t get path from file list. Ignoring mail'
)
return
fileList = fileList[len(path):].strip()
singleFileRE = re.compile(
'(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)'
)
elif cvsmode == '1.12':
singleFileRE = re.compile(
'(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)'
)
if path is None:
raise ValueError(
'CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config'
)
else:
raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' %
cvsmode)
log.msg("CVSMaildirSource processing filelist: %s" % fileList)
while (fileList):
m = singleFileRE.match(fileList)
if m:
curFile = path + '/' + m.group(1)
files.append(curFile)
fileList = fileList[m.end():]
else:
log.msg('CVSMaildirSource no files matched regex. Ignoring')
return None # bail - we couldn't parse the files that changed
# Now get comments
while lines:
line = lines.pop(0)
comments += line
comments = comments.rstrip() + "\n"
if comments == '\n':
comments = None
return ('cvs',
dict(author=author,
files=files,
comments=comments,
isdir=isdir,
when=when,
branch=branch,
revision=rev,
category=category,
repository=cvsroot,
project=project,
properties=self.properties))
def process_message(message):
""" Processes the read message and decodes attachments. """
attachments = []
html_data = []
text_data = []
from_addr = get_addrs(message, 'From')[0]
to_addrs = get_addrs(message, 'To')
cc_addrs = get_addrs(message, 'Cc')
bcc_addrs = get_addrs(
message, 'Bcc') # depending on sending MTA, this can be present or not
envelope_to_addrs = get_addrs(message, 'X-Original-To') + get_addrs(
message, 'X-Envelope-To') # Postfix / Sendmail does this
target_addrs = to_addrs + cc_addrs + bcc_addrs + envelope_to_addrs
subject = decode_2044(message['Subject'])
date = time.strftime("%Y-%m-%dT%H:%M:%S",
time.gmtime(mktime_tz(parsedate_tz(message['Date']))))
log("Processing mail:\n To: %r\n From: %r\n Subject: %r" %
(to_addrs[0], from_addr, subject))
for part in message.walk():
log(" Part " + repr((
part.get_charsets(),
part.get_content_charset(),
part.get_content_type(),
part.is_multipart(),
)))
ct = part.get_content_type()
cs = part.get_content_charset() or "latin1"
payload = part.get_payload(None, True)
fn = part.get_filename()
if fn is not None and fn.startswith("=?"): # heuristics ...
fn = decode_2044(fn)
if fn is None and part[
"Content-Disposition"] is not None and "attachment" in part[
"Content-Disposition"]:
# this doesn't catch the case where there is no content-disposition but there is a file to offer to the user
# i hope that this can be only found in mails that are older than 10 years,
# so I won't care about it here
fn = part["Content-Description"] or "NoName"
if fn:
a = attachment(fn, ct, payload)
attachments.append(a)
else:
if ct == 'text/plain':
text_data.append(payload.decode(cs))
log(repr(payload.decode(cs)))
elif ct == 'text/html':
html_data.append(payload.decode(cs))
elif not part.is_multipart():
log("Unknown mail part " + repr((
part.get_charsets(),
part.get_content_charset(),
part.get_content_type(),
part.is_multipart(),
)))
return {
'text': u"".join(text_data),
'html': u"".join(html_data),
'attachments': attachments,
'target_addrs': target_addrs,
'to_addrs': to_addrs,
'cc_addrs': cc_addrs,
'bcc_addrs': bcc_addrs,
'envelope_to_addrs': envelope_to_addrs,
'from_addr': from_addr,
'subject': subject,
'date': date
}
