def can_sort_by_pubdate( self): for episode in self: try: mktime_tz(parsedate_tz( episode.pubDate)) except: log('Episode %s has non-parseable pubDate. Sorting disabled.', episode.title) return False return True
def compare_pubdate( self, pubdate): try: timestamp_self=int(mktime_tz( parsedate_tz( self.pubDate))) except: return -1 try: timestamp_other=int(mktime_tz( parsedate_tz( pubdate))) except: return 1 return timestamp_self - timestamp_other
def __cmp__( self, other): if self.pubDate == other.pubDate: log('pubDate equal, comparing titles (buggy feed?)', sender=self) return cmp(self.title, other.title) try: timestamp_self=int(mktime_tz( parsedate_tz( self.pubDate))) timestamp_other=int(mktime_tz( parsedate_tz( other.pubDate))) except: # by default, do as if this is not the same # this is here so that comparisons with None # can be allowed (item is not None -> True) return -1 return timestamp_self - timestamp_other
def renderHTTP(self, ctx): """ Checks the file modification time from the client against the modification time on the server. Also checks the file cache to see if it needs to be refreshed. """ request = inevow.IRequest(ctx) self._check_cache() temp_date = request.getHeader("If-Modified-Since") if temp_date: parsed_date = parsedate_tz(temp_date) if parsed_date: client_time = mktime_tz(parsed_date) else: client_time = 0 else: client_time = 0 ## ## Check to see if the file has been modified since the client's mtime ## request.setHeader('content-type', "text/javascript; charset=UTF-8") if self.mtime > client_time: ## New version available. request.setLastModified(self.mtime) stamp = datetime.now() + timedelta(365) request.setHeader('expires', datetimeToString(mktime(stamp.timetuple()))) request.setHeader('content-length', str(len(self.data))) request.setHeader('content-type', "text/javascript; charset=UTF-8") return self.data else: request.setResponseCode(http.NOT_MODIFIED) return "OK"
def was_modified_since(header=None, mtime=0, size=0): """ Was something modified since the user last downloaded it? header This is the value of the If-Modified-Since header. If this is None, I'll just return True. mtime This is the modification time of the item we're talking about. size This is the size of the item we're talking about. """ try: if header is None: raise ValueError matches = re.match('^([^;]+)(; length=([0-9]+))?$', header, re.IGNORECASE) header_mtime = mktime_tz(parsedate_tz(matches.group(1))) header_len = matches.group(3) if header_len and int(header_len) != size: raise ValueError if mtime > header_mtime: raise ValueError except (AttributeError, ValueError): return True return False
def get_time(fpath): fh = file(fpath, 'rb') lines = iter(fh) for line in lines: if line.lower().startswith("received:"): break else: print "\nNo Received header found." fh.close() return None received = line for line in lines: if line[0] in ' \t': received += line else: break fh.close() i = received.rfind(';') if i < 0: print "\n" + received print "No semicolon found in Received header." return None datestring = received[i+1:] datestring = ' '.join(datestring.split()) as_tuple = parsedate_tz(datestring) if as_tuple is None: print "\n" + received print "Couldn't parse the date: %r" % datestring return None return mktime_tz(as_tuple)
def purgeGroups(self): purge_read_limit = int(self._configs["purge_read"]) purge_read_limit_secs = purge_read_limit * 24 * 60 * 60 purge_unread_limit = int(self._configs["purge_unread"]) purge_unread_limit_secs = purge_unread_limit * 24 * 60 * 60 time_now = mktime_tz(parsedate_tz(time.ctime())) subscribed = self.getSubscribed() for group in subscribed: c = self._getCursor(group[0]) conn = self._getConnection(group[0]) if purge_read_limit_secs > 0: c.execute( """delete from articles where secs < ? and read='1' and kept='0'""", (time_now - purge_read_limit_secs, )) conn.commit() if purge_unread_limit_secs > 0: c.execute( """delete from articles where secs < ? and read='0' and kept='0'""", (time_now - purge_unread_limit_secs, )) conn.commit() c.execute("""vacuum""") conn.commit() yield group[0]
def renderHTTP(self, ctx): """ Checks the file modification time from the client against the modification time on the server. Also checks the file cache to see if it needs to be refreshed. """ request = inevow.IRequest(ctx) self._check_cache() temp_date = request.getHeader("If-Modified-Since") if temp_date: parsed_date = parsedate_tz(temp_date) if parsed_date: client_time = mktime_tz(parsed_date) else: client_time = 0 else: client_time = 0 ## ## Check to see if the file has been modified since the client's mtime ## request.setHeader('content-type', "text/javascript") if self.mtime > client_time: ## New version available. request.setLastModified(self.mtime) stamp = datetime.now() + timedelta(365) request.setHeader('expires', datetimeToString(mktime(stamp.timetuple()))) request.setHeader('content-length', str(len(self.data))) request.setHeader('content-type', "text/javascript") return self.data else: request.setResponseCode(http.NOT_MODIFIED) return "OK"
def get_time(fpath): fh = file(fpath, 'rb') lines = iter(fh) # Find first Received header. for line in lines: if line.lower().startswith("received:"): break else: print "\nNo Received header found." fh.close() return None # Paste on continuation lines, if any. received = line for line in lines: if line[0] in ' \t': received += line else: break fh.close() # RFC 2822 says the date-time field must follow a semicolon at the end. i = received.rfind(';') if i < 0: print "\n" + received print "No semicolon found in Received header." return None # We only want the part after the semicolon. datestring = received[i+1:] # It may still be split across lines (like "Wed, \r\n\t22 Oct ..."). datestring = ' '.join(datestring.split()) as_tuple = parsedate_tz(datestring) if as_tuple is None: print "\n" + received print "Couldn't parse the date: %r" % datestring return None return mktime_tz(as_tuple)
def _check_cache_date(self, media_info): """ Checks the If-Modified-Since header, if supplied by the client, and determines whether the client's cached copy is recent enough to be used. @param media_info: Information about the media being served. @type media_info: Dictionary """ if not media_info: return False ## ## See if this is a cache-freshness query ## time_stamp = self.request.getHeader("If-Modified-Since") if time_stamp: parsed_date = parsedate_tz(time_stamp) if parsed_date: header_time = mktime_tz(parsed_date) else: header_time = 0 file_time = time.mktime(media_info['updated'].timetuple()) if file_time == header_time: ## Client's cache is fresh self.request.setResponseCode(http.NOT_MODIFIED) self.request.setHeader('content-type', 'image/jpeg') self.request.setHeader('content-length', 0) self.request.setLastModified(file_time) return True # Catchall return False
def getmail(user, password, gmailfolder): conn = imaplib.IMAP4_SSL("imap.gmail.com", 993) conn.login(user, password) conn.select(gmailfolder) # mailbox/gmail label # print conn.status(gmailfolder,"(MESSAGES)")#print message count since = datetime.datetime.today() - datetime.timedelta(days=90) since = since.strftime("%d-%b-%Y") dates = [] typ, mailnumlist = conn.search(None, "SINCE", since) # or ALL typ, mailheaders = conn.fetch(mailnumlist[0].replace(" ", ","), "(BODY.PEEK[HEADER])") for response_part in mailheaders: if isinstance(response_part, tuple): header_data = response_part[1] headerparser = HeaderParser() msg = headerparser.parsestr(header_data) # Convert long form date into UTC timestamp while considering tz msgdatetimestamp = parsedate_tz(msg["Date"]) # Discard hours/minutes/seconds as we are aggregating by whole dates msgdate = msgdatetimestamp[0:3] + (0, 0, 0, 0, 0, 0, 0, -18000) # Format cleanly for output as Javascript timestamp dates.append(str(int(mktime_tz(msgdate)))) conn.close() datesdict = {} output = "" alldates = [key for key, _ in groupby(dates)] for i in alldates: datesdict[i] = dates.count(i) output += '{ "label": "Email", \n' output += ' "data": [\n' for k, v in sorted(datesdict.items()): output += " [%s, %s],\n" % (k, v) output = output[:-2] + "\n]}" return output
def was_modified_since(header=None, mtime=0, size=0): """ Was something modified since the user last downloaded it? header This is the value of the If-Modified-Since header. If this is None, I'll just return True. mtime This is the modification time of the item we're talking about. size This is the size of the item we're talking about. """ try: if header is None: raise ValueError matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header, re.IGNORECASE) header_mtime = mktime_tz(parsedate_tz(matches.group(1))) header_len = matches.group(3) if header_len and int(header_len) != size: raise ValueError if mtime > header_mtime: raise ValueError except (AttributeError, ValueError): return True return False
def getStartDate(self, default=_MARKER): """ Returns the date from the mail date """ if not self.hasFile(): # Return the standard start date if no file was provided if default is _MARKER: return self._baseGetStartDate() else: return self._baseGetStartDate(default) date_string = self.getContentInformation().get('Date', None) if date_string: parsed_date_string = parsedate_tz(date_string) if parsed_date_string is not None: try: try: time = mktime_tz(parsed_date_string) except OverflowError: # No way to get a date - parsing failed (XXX) time = None if time: return DateTime(time) except ValueError: # If the parsed date is not conformant, nothing to do pass return self.getCreationDate()
def apply_action_rules(self,xpn_article,group): score=xpn_article.get_score() i=0 time_now=mktime_tz(parsedate_tz(time.ctime())) age=(time_now-xpn_article.secs)/(24.0*60*60) #compatibility check #there is no need here, it's has been already performed in apply_score_rules xpost=xpn_article.xref.count(":") fields={"subject":xpn_article.subj,"from":xpn_article.from_name,"date":xpn_article.date, "message-id":xpn_article.msgid,"references":xpn_article.ref,"age":age,"xref":xpn_article.xref, "bytes":xpn_article.bytes,"lines":xpn_article.lines,"xpost":xpost,"score":score} actions=[] for group_scope in self.group_scope_list: if group_scope=="*" or (group in group_scope): #current group is in this scope for rule in self.group_rules_list[i]: #let's apply rule match_rule=self.build_match_rule(rule["match"]) if match_rule(fields[rule["header"]],rule["match"]): action=rule["score_mod"] if action[0]=="!": #this is an action if action[1:]=="kill": #xpn_article=None pass #need this to let reapply_rules work elif action[1:]=="markread": xpn_article.is_read=True elif action[1:]=="markunread": xpn_article.is_read=False elif action[1:]=="mark": xpn_article.marked_for_download=True elif action[1:]=="unmark": xpn_article.marked_for_download=False elif action[1:]=="keep": xpn_article.keep=True elif action[1:]=="unkeep": xpn_article.keep=False elif action[1:]=="watch": xpn_article.watch=True xpn_article.ignore=False elif action[1:]=="ignore": xpn_article.watch=False xpn_article.ignore=True xpn_article.is_read=True elif action[1:]=="unsetwatchignore": xpn_article.watch=False xpn_article.ignore=False elif action[1:].startswith("setcolor"): regex=re.compile("(^setcolor)\((([a-z]+)|(#\w+));(([a-z]+)|(#\w+))\)") matches=regex.match(action[1:]) xpn_article.fg_color=str(matches.groups()[1]) xpn_article.bg_color=str(matches.groups()[4]) actions.append(action[1:]) i=i+1 return xpn_article,actions
def getLatestUploadtime(self): result, mailCount = self.mail.select(mailBoxLabel) if result == 'OK': pass elif result == 'NO' and mailCount[0][0:13] == '[NONEXISTENT]': result, status = self.mail.create(mailBoxLabel) if result != 'OK': print "mailbox can not be created." return None else: print "mailbox created" self.mail.select(mailBoxLabel) mailCount = ['0'] else: print mailCount return None if mailCount == ['0']: return 0 msgUIDs = [''] """ initial search time within 1 day """ expandTime = 86400 while (msgUIDs == ['']): """ check for the maximum time """ if expandTime > 88473600: return (time.time() - expandTime) sinceTime = imaplib.Time2Internaldate(time.time() - expandTime) result, msgUIDs = self.mail.search(None, 'SINCE', sinceTime) expandTime = expandTime * 2 idString = ','.join(msgUIDs[0].split()) result, msgIDs = self.mail.fetch(idString, '(BODY[HEADER.FIELDS (MESSAGE-ID)])') latest = 0 for msgID in msgIDs: if len(msgID) < 2: continue try: epoch = msgID[1].split('.')[1][0:13] except: latest = -1 break if epoch > latest: latest = epoch if latest == -1: print "unrecognized message id" result, msgDate = self.mail.fetch(idString, '(INTERNALDATE)') latest = 0 for date in msgDate: _date = date.split('"')[1] seconds = mktime_tz(parsedate_tz(_date)) if seconds > latest: latest = seconds latest = latest * 1000 return latest
def floatdate(header): missing = [] datestr = message.get(header, missing) if datestr is missing: return None date = parsedate_tz(datestr) try: return mktime_tz(date) except (TypeError, ValueError, OverflowError): return None
def __mktime__(self, date): if isinstance(date, float) or isinstance(date, int): return date if date is None or '' == date: return None try: return mktime_tz(parsedate_tz(date)) except TypeError: log('Could not convert "%s" to a unix timestamp.', date) return None
def cute_pubdate( self): try: timestamp=int(mktime_tz(parsedate_tz(self.pubDate))) except: timestamp=None result=util.format_date(timestamp) if result is None: return '(%s)' % _('unknown') else: return result
def __init__(self, message, hash_value, uid): self.hash_value = hash_value self.uid = uid message = email.message_from_string(message) def _get_header_data(key): value = message.get(key, '<unknown>') value = decode_mail_headers(decodeUnknown(message.get_charset(), value)) return value self.subject = _get_header_data('subject').strip() self.sender = parseaddr(message.get('from')) received = None date_str= message.get('date') if date_str: date_tuple= parsedate_tz(date_str) if date_tuple: received=datetime.datetime.fromtimestamp(mktime_tz(date_tuple)) self.received = received self.recipients = getaddresses(message.get_all('to', []) + message.get_all('cc', []) + message.get_all('resent-to', []) + message.get_all('resent-cc', [])) counter = 0 files = [] body_plain, body_html = '', '' for part in message.walk(): if part.get_content_maintype() == 'multipart': continue name = part.get_param("name") if name: name = collapse_rfc2231_value(name) if part.get_content_maintype() == 'text' and name == None: if part.get_content_subtype() == 'plain': body_plain = decodeUnknown(part.get_content_charset(), part.get_payload(decode=True)) else: body_html = part.get_payload(decode=True) else: if not name: ext = mimetypes.guess_extension(part.get_content_type()) name = "part-%i%s" % (counter, ext) files.append({ 'filename': name, 'content': part.get_payload(decode=True), 'type': part.get_content_type()}, ) counter += 1 self.body_plain = body_plain self.body_html = mark_safe(strip_empty_tags(strip_tags(body_html, ['html', 'head', 'body', 'meta']))) self.files = files
def get_date_header(msg, name): """Returns an UTC timestamp from a header field containing a date. Compensates for the timezone difference if the header contains timezone information. """ value = get_header(msg, name) ts = None try: ts = mktime_tz(parsedate_tz(value)) except TypeError: pass return ts
def _modified_since(request, file_modified_time): modified_since_str = request.env.get('HTTP_IF_MODIFIED_SINCE') if modified_since_str: try: modified_since = mktime_tz(parsedate_tz(modified_since_str)) except: # couln't parse return True if math.floor(file_modified_time) <= modified_since: return False return True
def process_message(message): """ Processes the read message and decodes attachments. """ attachments = [] html_data = [] text_data = [] from_addr = get_addrs(message, 'From')[0] to_addrs = get_addrs(message, 'To') cc_addrs = get_addrs(message, 'Cc') bcc_addrs = get_addrs(message, 'Bcc') # depending on sending MTA, this can be present or not envelope_to_addrs = get_addrs(message, 'X-Original-To') + get_addrs(message, 'X-Envelope-To') # Postfix / Sendmail does this target_addrs = to_addrs + cc_addrs + bcc_addrs + envelope_to_addrs subject = decode_2044(message['Subject']) date = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(mktime_tz(parsedate_tz(message['Date'])))) log("Processing mail:\n To: %r\n From: %r\n Subject: %r" % (to_addrs[0], from_addr, subject)) for part in message.walk(): log(" Part " + repr((part.get_charsets(), part.get_content_charset(), part.get_content_type(), part.is_multipart(), ))) ct = part.get_content_type() cs = part.get_content_charset() or "latin1" payload = part.get_payload(None, True) fn = part.get_filename() if fn is not None and fn.startswith("=?"): # heuristics ... fn = decode_2044(fn) if fn is None and part["Content-Disposition"] is not None and "attachment" in part["Content-Disposition"]: # this doesn't catch the case where there is no content-disposition but there is a file to offer to the user # i hope that this can be only found in mails that are older than 10 years, # so I won't care about it here fn = part["Content-Description"] or "NoName" if fn: a = attachment(fn, ct, payload) attachments.append(a) else: if ct == 'text/plain': text_data.append(payload.decode(cs)) log(repr(payload.decode(cs))) elif ct == 'text/html': html_data.append(payload.decode(cs)) elif not part.is_multipart(): log("Unknown mail part " + repr((part.get_charsets(), part.get_content_charset(), part.get_content_type(), part.is_multipart(), ))) return {'text': u"".join(text_data), 'html': u"".join(html_data), 'attachments': attachments, 'target_addrs': target_addrs, 'to_addrs': to_addrs, 'cc_addrs': cc_addrs, 'bcc_addrs': bcc_addrs, 'envelope_to_addrs': envelope_to_addrs, 'from_addr': from_addr, 'subject': subject, 'date': date}
def floatdate(datestr): if not datestr: return None date = parsedate_tz(datestr) try: date = mktime_tz(date) if (date < 0 or date - time.time() > mm_cfg.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW ): return None return date except (TypeError, ValueError, OverflowError): return None
def _dispose(self, mlist, msg, msgdata): # Support clobber_date, i.e. setting the date in the archive to the # received date, not the (potentially bogus) Date: header of the # original message. clobber = 0 originaldate = msg.get('date') receivedtime = formatdate(msgdata['received_time']) if not originaldate: clobber = 1 elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 1: clobber = 1 elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 2: # what's the timestamp on the original message? tup = parsedate_tz(originaldate) now = time.time() try: if not tup: clobber = 1 elif abs(now - mktime_tz(tup)) > \ mm_cfg.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW: clobber = 1 except (ValueError, OverflowError): # The likely cause of this is that the year in the Date: field # is horribly incorrect, e.g. (from SF bug # 571634): # Date: Tue, 18 Jun 0102 05:12:09 +0500 # Obviously clobber such dates. clobber = 1 if clobber: del msg['date'] del msg['x-original-date'] msg['Date'] = receivedtime if originaldate: msg['X-Original-Date'] = originaldate # Always put an indication of when we received the message. msg['X-List-Received-Date'] = receivedtime # Now try to get the list lock try: mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT) except LockFile.TimeOutError: # oh well, try again later return 1 try: # Archiving should be done in the list's preferred language, not # the sender's language. i18n.set_language(mlist.preferred_language) mlist.ArchiveMail(msg) mlist.Save() finally: mlist.Unlock()
def _was_modified_since(header=None, mtime=0, size=0): try: if header is None: raise ValueError matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header, re.IGNORECASE) header_mtime = mktime_tz(parsedate_tz(matches.group(1))) header_len = matches.group(3) if header_len and int(header_len) != size: raise ValueError if mtime > header_mtime: raise ValueError except (AttributeError, ValueError): return True return False
def fromRFC2822(klass, rfc822string): """Return a new Time instance from a string formated as described in RFC 2822. RFC 2822 specifies the format of email messages. Some of the obsoleted elements of the specification are not parsed correctly, and will raise ValueError. """ date = parsedate_tz(rfc822string) if date is None: raise ValueError, 'could not parse RFC 2822 date %r' % ( rfc822string, ) self = klass.fromStructTime(time.gmtime(mktime_tz(date))) self.resolution = datetime.timedelta(seconds=1) return self
def utcdatetime_from_field(field_value): """Turn an RFC 2822 Date: header value into a Python datetime (UTC). :param field_value: The value of the Date: header :type field_value: string :return: The corresponding datetime (UTC) :rtype: `datetime.datetime` :raise `InvalidEmailMessage`: when the date string cannot be converted. """ try: date_tuple = parsedate_tz(field_value) timestamp = mktime_tz(date_tuple) return datetime.fromtimestamp(timestamp, tz=pytz.timezone('UTC')) except (TypeError, ValueError, OverflowError): raise InvalidEmailMessage('Invalid date %s' % field_value)
def parse_date(self,date): #data=parsedate(date) try: #trying to prevent the rfc822.parsedate bug with Tue,26 instead of Tue, 26 secs=mktime_tz(parsedate_tz(date)) except: secs=time() data=parsedate(ctime(secs)) if data[3]<10: ora="0"+repr(data[3]) else: ora=repr(data[3]) if data[4]<10: minuti="0"+repr(data[4]) else: minuti=repr(data[4]) return repr(data[2])+"/"+repr(data[1])+"/"+repr(data[0])+" "+ora+":"+minuti,secs
def was_modified_since(header=None, mtime=0, size=0): if header is None: return True matches = re.match(r'^([^;]+)(; length=([0-9]+))?$', header, re.IGNORECASE) header_mtime = mktime_tz(parsedate_tz(matches.group(1))) header_len = matches.group(3) if header_len and int(header_len) != size: return True if mtime > header_mtime: return True return False
def _dispose(self, mlist, msg, msgdata): # Support clobber_date, i.e. setting the date in the archive to the # received date, not the (potentially bogus) Date: header of the # original message. clobber = 0 originaldate = msg.get('date') receivedtime = formatdate(msgdata['received_time']) if not originaldate: clobber = 1 elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 1: clobber = 1 elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 2: # what's the timestamp on the original message? tup = parsedate_tz(originaldate) now = time.time() try: if not tup: clobber = 1 elif abs(now - mktime_tz(tup)) > \ mm_cfg.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW: clobber = 1 except (ValueError, OverflowError): # The likely cause of this is that the year in the Date: field # is horribly incorrect, e.g. (from SF bug # 571634): # Date: Tue, 18 Jun 0102 05:12:09 +0500 # Obviously clobber such dates. clobber = 1 if clobber: del msg['date'] del msg['x-original-date'] msg['Date'] = receivedtime if originaldate: msg['X-Original-Date'] = originaldate # Always put an indication of when we received the message. msg['X-List-Received-Date'] = receivedtime # Now try to get the list lock try: mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT) except LockFile.TimeOutError: # oh well, try again later return 1 try: mlist.ArchiveMail(msg) mlist.Save() finally: mlist.Unlock()
def parse_date(self, date): #data=parsedate(date) try: #trying to prevent the rfc822.parsedate bug with Tue,26 instead of Tue, 26 secs = mktime_tz(parsedate_tz(date)) except: secs = time() data = parsedate(ctime(secs)) if data[3] < 10: ora = "0" + repr(data[3]) else: ora = repr(data[3]) if data[4] < 10: minuti = "0" + repr(data[4]) else: minuti = repr(data[4]) return repr(data[2]) + "/" + repr(data[1]) + "/" + repr( data[0]) + " " + ora + ":" + minuti, secs
def was_modified_since(self, mtime=0, size=0): header = self.request.headers.get('HTTP_IF_MODIFIED_SINCE') if header is None: return True matches = re.match(r'^([^;]+)(; length=([0-9]+))?$', header, re.IGNORECASE) header_mtime = mktime_tz(parsedate_tz(matches.group(1))) header_len = matches.group(3) if header_len and int(header_len) != size: return True if mtime > header_mtime: return True return False
def apply_score_rules(self,xpn_article,group): score=xpn_article.get_score() i=0 time_now=mktime_tz(parsedate_tz(time.ctime())) age=(time_now-xpn_article.secs)/(24.0*60*60) #compatibility check try: xpn_article.xref except AttributeError: xpn_article.xref="" try: xpn_article.bytes except AttributeError: xpn_article.bytes=0 try: xpn_article.lines except AttributeError: xpn_article.lines=0 xpost=xpn_article.xref.count(":") fields={"subject":xpn_article.subj,"from":xpn_article.from_name,"date":xpn_article.date, "message-id":xpn_article.msgid,"references":xpn_article.ref,"age":age,"xref":xpn_article.xref, "bytes":xpn_article.bytes,"lines":xpn_article.lines,"xpost":xpost} for group_scope in self.group_scope_list: if group_scope=="*" or (group in group_scope): #current group is in this scope for rule in self.group_rules_list[i]: #let's apply rule score_mod=rule["score_mod"] if score_mod[0]=="!": #this is an action continue match_rule=self.build_match_rule(rule["match"]) if match_rule(fields[rule["header"]],rule["match"]): score_mod=rule["score_mod"] if score_mod[0]=="=": score=int(score_mod[1:]) if score>9999: score=9999 if score<-9999: score=-9999 return score else: score=score+int(score_mod) i=i+1 if score>9999: score=9999 if score<-9999: score=-9999 return score
def _getDateForComment(self, parsed_comment): """Return the correct date for a comment. :param parsed_comment: An `email.Message.Message` instance containing a parsed DebBugs comment. :return: The correct date to use for the comment contained in `parsed_comment`. If a date is specified in a Received header on `parsed_comment` that we can use, return that. Otherwise, return the Date field of `parsed_comment`. """ # Check for a Received: header on the comment and use # that to get the date, if possible. We only use the # date received by this host (nominally bugs.debian.org) # since that's the one that's likely to be correct. received_headers = parsed_comment.get_all('received') if received_headers is not None: host_name = urlsplit(self.baseurl)[1] received_headers = [ header for header in received_headers if host_name in header] # If there are too many - or too few - received headers then # something's gone wrong and we default back to using # the Date field. if received_headers is not None and len(received_headers) == 1: received_string = received_headers[0] received_by, date_string = received_string.split(';', 2) else: date_string = parsed_comment['date'] # We parse the date_string if we can, otherwise we just return # None. if date_string is not None: date_with_tz = parsedate_tz(date_string) timestamp = mktime_tz(date_with_tz) msg_date = datetime.fromtimestamp(timestamp, tz=pytz.timezone('UTC')) else: msg_date = None return msg_date
def _getDateForComment(self, parsed_comment): """Return the correct date for a comment. :param parsed_comment: An `email.Message.Message` instance containing a parsed DebBugs comment. :return: The correct date to use for the comment contained in `parsed_comment`. If a date is specified in a Received header on `parsed_comment` that we can use, return that. Otherwise, return the Date field of `parsed_comment`. """ # Check for a Received: header on the comment and use # that to get the date, if possible. We only use the # date received by this host (nominally bugs.debian.org) # since that's the one that's likely to be correct. received_headers = parsed_comment.get_all('received') if received_headers is not None: host_name = urlsplit(self.baseurl)[1] received_headers = [ header for header in received_headers if host_name in header ] # If there are too many - or too few - received headers then # something's gone wrong and we default back to using # the Date field. if received_headers is not None and len(received_headers) == 1: received_string = received_headers[0] received_by, date_string = received_string.split(';', 2) else: date_string = parsed_comment['date'] # We parse the date_string if we can, otherwise we just return # None. if date_string is not None: date_with_tz = parsedate_tz(date_string) timestamp = mktime_tz(date_with_tz) msg_date = datetime.fromtimestamp(timestamp, tz=pytz.timezone('UTC')) else: msg_date = None return msg_date
def test_CVSMaildirSource_create_change_from_cvs1_11msg(self): m = message_from_string(cvs1_11_msg) src = CVSMaildirSource('/dev/null') try: src, chdict = src.parse( m ) except: self.fail('Failed to get change from email message.') self.assertNotEqual(chdict, None) self.assertEqual(chdict['author'], 'andy') self.assertEqual(len(chdict['files']), 1) self.assertEqual(chdict['files'][0], 'base/module/src/make/GNUmakefile') self.assertEqual(chdict['comments'], 'Commented out some stuff.\n') self.assertFalse(chdict['isdir']) self.assertEqual(chdict['revision'], '2010-08-07 11:11:49') dateTuple = parsedate_tz('Sat, 07 Aug 2010 11:11:49 +0000') self.assertEqual(chdict['when'], mktime_tz(dateTuple)) self.assertEqual(chdict['branch'], None) self.assertEqual(chdict['repository'], ':ext:cvshost.example.com:/cvsroot') self.assertEqual(chdict['project'], 'MyModuleName') self.assertEqual(len(chdict['properties']), 0) self.assertEqual(src, 'cvs')
def test_CVSMaildirSource_create_change_from_cvs1_11msg(self): m = message_from_string(cvs1_11_msg) src = CVSMaildirSource('/dev/null', urlmaker=fileToUrl) try: chdict = src.parse( m ) except: self.fail('Failed to get change from email message.') self.assert_(chdict != None) self.assert_(chdict['who'] == 'andy') self.assert_(len(chdict['files']) == 1) self.assert_(chdict['files'][0] == 'base/module/src/make/GNUmakefile') self.assert_(chdict['comments'] == 'Commented out some stuff.\n') self.assert_(chdict['isdir'] == False) self.assert_(len(chdict['links']) == 1) self.assert_(chdict['revision'] == '2010-08-07 11:11:49') dateTuple = parsedate_tz('Sat, 07 Aug 2010 11:11:49 +0000') self.assert_(chdict['when'] == mktime_tz(dateTuple)) self.assert_(chdict['branch'] == None) self.assert_(chdict['repository'] == ':ext:cvshost.example.com:/cvsroot') self.assert_(chdict['project'] == 'MyModuleName') self.assert_(len(chdict['properties']) == 0)
def test_CVSMaildirSource_create_change_from_cvs1_11msg(self): m = message_from_string(cvs1_11_msg) src = CVSMaildirSource('/dev/null', urlmaker=fileToUrl) try: chdict = src.parse(m) except: self.fail('Failed to get change from email message.') self.assert_(chdict != None) self.assert_(chdict['who'] == 'andy') self.assert_(len(chdict['files']) == 1) self.assert_(chdict['files'][0] == 'base/module/src/make/GNUmakefile') self.assert_(chdict['comments'] == 'Commented out some stuff.\n') self.assert_(chdict['isdir'] == False) self.assert_(len(chdict['links']) == 1) self.assert_(chdict['revision'] == '2010-08-07 11:11:49') dateTuple = parsedate_tz('Sat, 07 Aug 2010 11:11:49 +0000') self.assert_(chdict['when'] == mktime_tz(dateTuple)) self.assert_(chdict['branch'] == None) self.assert_( chdict['repository'] == ':ext:cvshost.example.com:/cvsroot') self.assert_(chdict['project'] == 'MyModuleName') self.assert_(len(chdict['properties']) == 0)
def test_CVSMaildirSource_create_change_from_cvs1_11msg(self): m = message_from_string(cvs1_11_msg) src = CVSMaildirSource('/dev/null', urlmaker=fileToUrl) try: change = src.parse(m) except: self.fail('Failed to get change from email message.') self.assert_(change != None) self.assert_(change.who == 'andy') self.assert_(len(change.files) == 1) self.assert_(change.files[0] == 'base/module/src/make/GNUmakefile') self.assert_(change.comments == 'Commented out some stuff.\n') self.assert_(change.isdir == False) self.assert_(len(change.links) == 1) self.assert_(change.revision == '2010-08-07 11:11:49') dateTuple = parsedate_tz('Sat, 07 Aug 2010 11:11:49 +0000') self.assert_(change.when == mktime_tz(dateTuple)) self.assert_(change.branch == None) self.assert_(change.revlink == '') self.assert_(change.repository == ':ext:cvshost.example.com:/cvsroot') self.assert_(change.project == 'MyModuleName') propList = change.properties.asList() self.assert_(len(propList) == 0)
def test_CVSMaildirSource_create_change_from_cvs1_12msg(self): m = message_from_string(cvs1_12_msg) src = CVSMaildirSource('/dev/null', urlmaker=fileToUrl) try: chdict = src.parse(m) except: self.fail('Failed to get change from email message.') self.assert_(chdict != None) self.assert_(chdict['who'] == 'andy') self.assert_(len(chdict['files']) == 2) self.assert_(chdict['files'][0] == 'base/module/src/file1.cpp') self.assert_(chdict['files'][1] == 'base/module/src/file2.cpp') self.assert_(chdict['comments'] == 'Changes for changes sake\n') self.assert_(chdict['isdir'] == False) self.assert_(len(chdict['links']) == 2) self.assert_(chdict['revision'] == '2010-08-11 04:56:44') dateTuple = parsedate_tz('Wed, 11 Aug 2010 04:56:44 +0000') self.assert_(chdict['when'] == mktime_tz(dateTuple)) self.assert_(chdict['branch'] == None) self.assert_( chdict['repository'] == ':ext:cvshost.example.com:/cvsroot') self.assert_(chdict['project'] == 'MyModuleName') self.assert_(len(chdict['properties']) == 0)
def test_CVSMaildirSource_create_change_from_cvs1_12msg(self): m = message_from_string(cvs1_12_msg) src = CVSMaildirSource('/dev/null') try: src, chdict = src.parse(m) except: self.fail('Failed to get change from email message.') self.assertNotEqual(chdict, None) self.assertEqual(chdict['author'], 'andy') self.assertEqual(len(chdict['files']), 2) self.assertEqual(chdict['files'][0], 'base/module/src/file1.cpp') self.assertEqual(chdict['files'][1], 'base/module/src/file2.cpp') self.assertEqual(chdict['comments'], 'Changes for changes sake\n') self.assertFalse(chdict['isdir']) self.assertEqual(chdict['revision'], '2010-08-11 04:56:44') dateTuple = parsedate_tz('Wed, 11 Aug 2010 04:56:44 +0000') self.assertEqual(chdict['when'], mktime_tz(dateTuple)) self.assertEqual(chdict['branch'], None) self.assertEqual(chdict['repository'], ':ext:cvshost.example.com:/cvsroot') self.assertEqual(chdict['project'], 'MyModuleName') self.assertEqual(len(chdict['properties']), 0) self.assertEqual(src, 'cvs')
def test_CVSMaildirSource_create_change_from_cvs1_11msg(self): m = message_from_string(cvs1_11_msg) src = CVSMaildirSource('/dev/null') try: src, chdict = src.parse(m) except: self.fail('Failed to get change from email message.') self.assertNotEqual(chdict, None) self.assertEqual(chdict['author'], 'andy') self.assertEqual(len(chdict['files']), 1) self.assertEqual(chdict['files'][0], 'base/module/src/make/GNUmakefile') self.assertEqual(chdict['comments'], 'Commented out some stuff.\n') self.assertFalse(chdict['isdir']) self.assertEqual(chdict['revision'], '2010-08-07 11:11:49') dateTuple = parsedate_tz('Sat, 07 Aug 2010 11:11:49 +0000') self.assertEqual(chdict['when'], mktime_tz(dateTuple)) self.assertEqual(chdict['branch'], None) self.assertEqual(chdict['repository'], ':ext:cvshost.example.com:/cvsroot') self.assertEqual(chdict['project'], 'MyModuleName') self.assertEqual(len(chdict['properties']), 0) self.assertEqual(src, 'cvs')
def test_CVSMaildirSource_create_change_from_cvs1_12msg(self): m = message_from_string(cvs1_12_msg) src = CVSMaildirSource('/dev/null', urlmaker=fileToUrl) try: change = src.parse(m) except: self.fail('Failed to get change from email message.') self.assert_(change != None) self.assert_(change.who == 'andy') self.assert_(len(change.files) == 2) self.assert_(change.files[0] == 'base/module/src/file1.cpp') self.assert_(change.files[1] == 'base/module/src/file2.cpp') self.assert_(change.comments == 'Changes for changes sake\n') self.assert_(change.isdir == False) self.assert_(len(change.links) == 2) self.assert_(change.revision == '2010-08-11 04:56:44') dateTuple = parsedate_tz('Wed, 11 Aug 2010 04:56:44 +0000') self.assert_(change.when == mktime_tz(dateTuple)) self.assert_(change.branch == None) self.assert_(change.revlink == '') self.assert_(change.repository == ':ext:cvshost.example.com:/cvsroot') self.assert_(change.project == 'MyModuleName') propList = change.properties.asList() self.assert_(len(propList) == 0)
def cookieDate2Ts(self, cd): #Sat, 02-Nov-2013 11:33:51 GMT if cd.strip() != "" and cd != None: return mktime_tz(parsedate_tz(cd.strip())) return long(0)
def parse(self, m, prefix=None): """Parse messages sent by the 'buildbot-cvs-mail' program. """ # The mail is sent from the person doing the checkin. Assume that the # local username is enough to identify them (this assumes a one-server # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS # model) name, addr = parseaddr(m["from"]) if not addr: return None # no From means this message isn't from buildbot-cvs-mail at = addr.find("@") if at == -1: author = addr # might still be useful else: author = addr[:at] # CVS accecpts RFC822 dates. buildbot-cvs-mail adds the date as # part of the mail header, so use that. # This assumes cvs is being access via ssh or pserver, so the time # will be the CVS server's time. # calculate a "revision" based on that timestamp, or the current time # if we're unable to parse the date. log.msg('Processing CVS mail') dateTuple = parsedate_tz(m["date"]) if dateTuple == None: when = util.now() else: when = mktime_tz(dateTuple) theTime = datetime.datetime.utcfromtimestamp(float(when)) rev = theTime.strftime('%Y-%m-%d %H:%M:%S') catRE = re.compile('^Category:\s*(\S.*)') cvsRE = re.compile('^CVSROOT:\s*(\S.*)') cvsmodeRE = re.compile('^Cvsmode:\s*(\S.*)') filesRE = re.compile('^Files:\s*(\S.*)') modRE = re.compile('^Module:\s*(\S.*)') pathRE = re.compile('^Path:\s*(\S.*)') projRE = re.compile('^Project:\s*(\S.*)') singleFileRE = re.compile('(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)') tagRE = re.compile('^\s+Tag:\s*(\S.*)') updateRE = re.compile('^Update of:\s*(\S.*)') comments = "" branch = None cvsroot = None fileList = None files = [] isdir = 0 path = None project = None lines = list(body_line_iterator(m)) while lines: line = lines.pop(0) m = catRE.match(line) if m: category = m.group(1) continue m = cvsRE.match(line) if m: cvsroot = m.group(1) continue m = cvsmodeRE.match(line) if m: cvsmode = m.group(1) continue m = filesRE.match(line) if m: fileList = m.group(1) continue m = modRE.match(line) if m: # We don't actually use this #module = m.group(1) continue m = pathRE.match(line) if m: path = m.group(1) continue m = projRE.match(line) if m: project = m.group(1) continue m = tagRE.match(line) if m: branch = m.group(1) continue m = updateRE.match(line) if m: # We don't actually use this #updateof = m.group(1) continue if line == "Log Message:\n": break # CVS 1.11 lists files as: # repo/path file,old-version,new-version file2,old-version,new-version # Version 1.12 lists files as: # file1 old-version new-version file2 old-version new-version # # files consists of tuples of 'file-name old-version new-version' # The versions are either dotted-decimal version numbers, ie 1.1 # or NONE. New files are of the form 'NONE NUMBER', while removed # files are 'NUMBER NONE'. 'NONE' is a literal string # Parsing this instead of files list in 'Added File:' etc # makes it possible to handle files with embedded spaces, though # it could fail if the filename was 'bad 1.1 1.2' # For cvs version 1.11, we expect # my_module new_file.c,NONE,1.1 # my_module removed.txt,1.2,NONE # my_module modified_file.c,1.1,1.2 # While cvs version 1.12 gives us # new_file.c NONE 1.1 # removed.txt 1.2 NONE # modified_file.c 1.1,1.2 if fileList is None: log.msg('CVSMaildirSource Mail with no files. Ignoring') return None # We don't have any files. Email not from CVS if cvsmode == '1.11': # Please, no repo paths with spaces! m = re.search('([^ ]*) ', fileList) if m: path = m.group(1) else: log.msg( 'CVSMaildirSource can\'t get path from file list. Ignoring mail' ) return fileList = fileList[len(path):].strip() singleFileRE = re.compile( '(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)' ) elif cvsmode == '1.12': singleFileRE = re.compile( '(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)' ) if path is None: raise ValueError( 'CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config' ) else: raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode) log.msg("CVSMaildirSource processing filelist: %s" % fileList) while (fileList): m = singleFileRE.match(fileList) if m: curFile = path + '/' + m.group(1) files.append(curFile) fileList = fileList[m.end():] else: log.msg('CVSMaildirSource no files matched regex. Ignoring') return None # bail - we couldn't parse the files that changed # Now get comments while lines: line = lines.pop(0) comments += line comments = comments.rstrip() + "\n" if comments == '\n': comments = None return ('cvs', dict(author=author, files=files, comments=comments, isdir=isdir, when=when, branch=branch, revision=rev, category=category, repository=cvsroot, project=project, properties=self.properties))
def process_message(message): """ Processes the read message and decodes attachments. """ attachments = [] html_data = [] text_data = [] from_addr = get_addrs(message, 'From')[0] to_addrs = get_addrs(message, 'To') cc_addrs = get_addrs(message, 'Cc') bcc_addrs = get_addrs( message, 'Bcc') # depending on sending MTA, this can be present or not envelope_to_addrs = get_addrs(message, 'X-Original-To') + get_addrs( message, 'X-Envelope-To') # Postfix / Sendmail does this target_addrs = to_addrs + cc_addrs + bcc_addrs + envelope_to_addrs subject = decode_2044(message['Subject']) date = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(mktime_tz(parsedate_tz(message['Date'])))) log("Processing mail:\n To: %r\n From: %r\n Subject: %r" % (to_addrs[0], from_addr, subject)) for part in message.walk(): log(" Part " + repr(( part.get_charsets(), part.get_content_charset(), part.get_content_type(), part.is_multipart(), ))) ct = part.get_content_type() cs = part.get_content_charset() or "latin1" payload = part.get_payload(None, True) fn = part.get_filename() if fn is not None and fn.startswith("=?"): # heuristics ... fn = decode_2044(fn) if fn is None and part[ "Content-Disposition"] is not None and "attachment" in part[ "Content-Disposition"]: # this doesn't catch the case where there is no content-disposition but there is a file to offer to the user # i hope that this can be only found in mails that are older than 10 years, # so I won't care about it here fn = part["Content-Description"] or "NoName" if fn: a = attachment(fn, ct, payload) attachments.append(a) else: if ct == 'text/plain': text_data.append(payload.decode(cs)) log(repr(payload.decode(cs))) elif ct == 'text/html': html_data.append(payload.decode(cs)) elif not part.is_multipart(): log("Unknown mail part " + repr(( part.get_charsets(), part.get_content_charset(), part.get_content_type(), part.is_multipart(), ))) return { 'text': u"".join(text_data), 'html': u"".join(html_data), 'attachments': attachments, 'target_addrs': target_addrs, 'to_addrs': to_addrs, 'cc_addrs': cc_addrs, 'bcc_addrs': bcc_addrs, 'envelope_to_addrs': envelope_to_addrs, 'from_addr': from_addr, 'subject': subject, 'date': date }