def process(self, msg): """See `IBounceDetector`.""" addresses = set() state = ParseState.start for line in body_line_iterator(msg): line = line.strip() if state is ParseState.start: for introtag in introtags: if line.startswith(introtag.encode("latin1")): state = ParseState.intro_paragraph_seen break elif state is ParseState.intro_paragraph_seen and not line: # Looking for the end of the intro paragraph. state = ParseState.recip_paragraph_seen elif state is ParseState.recip_paragraph_seen: if line.startswith("-"): # We're looking at the break paragraph, so we're done. break # At this point we know we must be looking at a recipient # paragraph. mo = acre.match(line) if mo: addresses.add(mo.group("addr").encode("us-ascii")) # Otherwise, it must be a continuation line, so just ignore it. else: # We're not looking at anything in particular. pass return NoTemporaryFailures, addresses
def process(self, msg): """See `IBounceDetector`.""" addresses = set() # MAS: This is a mess. The outer loop used to be over the message # so we only looped through the message once. Looping through the # message for each set of patterns is obviously way more work, but # if we don't do it, problems arise because scre from the wrong # pattern set matches first and then acre doesn't match. The # alternative is to split things into separate modules, but then # we process the message multiple times anyway. for scre, ecre, acre in self.PATTERNS: state = ParseState.start for line in body_line_iterator(msg): if state is ParseState.start: if scre.search(line): state = ParseState.tag_seen if state is ParseState.tag_seen: mo = acre.search(line) if mo: address = mo.group('addr') if address: addresses.add(_quopri_decode(address)) elif ecre.search(line): break if len(addresses) > 0: break return NoTemporaryFailures, addresses
def process(self, msg): """See `IBounceDetector`.""" addresses = set() state = ParseState.start for line in body_line_iterator(msg): line = line.strip() if state is ParseState.start: for introtag in introtags: if line.startswith(introtag.encode('latin1')): state = ParseState.intro_paragraph_seen break elif state is ParseState.intro_paragraph_seen and not line: # Looking for the end of the intro paragraph. state = ParseState.recip_paragraph_seen elif state is ParseState.recip_paragraph_seen: if line.startswith('-'): # We're looking at the break paragraph, so we're done. break # At this point we know we must be looking at a recipient # paragraph. mo = acre.match(line) if mo: addresses.add(mo.group('addr').encode('us-ascii')) # Otherwise, it must be a continuation line, so just ignore it. else: # We're not looking at anything in particular. pass return NoTemporaryFailures, addresses
def process(self, msg): """See `IBounceDetector`.""" # Yahoo! bounces seem to have a known subject value and something # called an x-uidl: header, the value of which seems unimportant. sender = parseaddr(msg.get('from', '').lower())[1] or '' if not sender.startswith('mailer-daemon@yahoo'): return NoFailures addresses = set() state = _ParseState.start for line in body_line_iterator(msg): line = line.strip() if state is _ParseState.start: for cre in tcre: if cre.match(line): state = _ParseState.tag_seen break elif state is _ParseState.tag_seen: mo = acre.match(line) if mo: addresses.add(mo.group('addr').encode('us-ascii')) continue for cre in ecre: mo = cre.match(line) if mo: # We're at the end of the error response. state = _ParseState.all_done break if state is _ParseState.all_done: break return NoTemporaryFailures, addresses
def process(self, msg): """See `IBounceDetector`.""" for line in body_line_iterator(msg): mo = acre.search(line) if mo: address = mo.group('addr').encode('us-ascii') return NoTemporaryFailures, set([address]) return NoFailures
def process(self, msg): mailer = msg.get('x-mailer', '') if not mailer.startswith('<SMTP32 v'): return NoFailures addresses = set() for line in body_line_iterator(msg): if ecre.search(line): break mo = acre.search(line) if mo: addresses.add(mo.group('addr').encode('us-ascii')) return NoTemporaryFailures, addresses
def confirmation_line(msg): confirmation_lines = [] in_results = False for line in body_line_iterator(msg): line = line.strip() if in_results: if line.startswith('- Done'): break if len(line) > 0: confirmation_lines.append(line) if line.strip() == '- Results:': in_results = True # There should be exactly one confirmation line. assert len(confirmation_lines) == 1, confirmation_lines return confirmation_lines[0]
def process(self, msg): if msg.get_content_type() == 'multipart/mixed': state = ParseState.start # This format thinks it's a MIME, but it really isn't. for line in body_line_iterator(msg): line = line.strip() if state is ParseState.start and tcre.match(line): state = ParseState.tag_seen elif state is ParseState.tag_seen and line: mo = acre.match(line) if mo: return NoTemporaryFailures, set(mo.group('addr')) else: break return NoFailures
def test_double_confirmation(self): # A join request comes in using both the -join address and the word # 'subscribe' in the first line of the body. This should produce just # one subscription request and one confirmation response. msg = mfs("""\ From: [email protected] To: [email protected] subscribe """) # Adding the subaddress to the metadata dictionary mimics what happens # when the above email message is first processed by the lmtp runner. # For convenience, we skip that step in this test. self._commandq.enqueue( msg, dict(listid='test.example.com', subaddress='join')) self._runner.run() # There will be two messages in the queue. The first one is a reply # to Anne notifying her of the status of her command email. The # second one is the confirmation message of her join request. items = get_queue_messages('virgin', sort_on='subject', expected_count=2) self.assertTrue(str(items[1].msg['subject']).startswith('confirm')) self.assertEqual(items[0].msg['subject'], 'The results of your email commands') # Search the contents of the results message. There should be just # one 'Confirmation email' line. confirmation_lines = [] in_results = False for line in body_line_iterator(items[0].msg): line = line.strip() if in_results: if line.startswith('- Done'): break if len(line) > 0: confirmation_lines.append(line) if line.strip() == '- Results:': in_results = True # There should be exactly one confirmation line. self.assertEqual(len(confirmation_lines), 1) # And the confirmation line should name Anne's email address. self.assertIn('*****@*****.**', confirmation_lines[0])
def process(self, msg): """See `IBounceDetector`.""" if msg.get('from', '').lower() != '*****@*****.**': return NoFailures if not msg.is_multipart(): return NoFailures # The interesting bits are in the first text/plain multipart. part = None try: part = msg.get_payload(0) except IndexError: pass if not part: return NoFailures addresses = set() for line in body_line_iterator(part): mo = acre.match(line) if mo: addresses.add(mo.group('addr').encode('us-ascii')) return NoTemporaryFailures, addresses
def test_double_confirmation(self): # 'confirm' in the Subject and in the To header should not try to # confirm the token twice. # # Clear out the virgin queue so that the test below only sees the # reply to the confirmation message. get_queue_messages('virgin') subject = 'Re: confirm {0}'.format(self._token) to = 'test-confirm+{0}@example.com'.format(self._token) msg = mfs("""\ From: Anne Person <*****@*****.**> """) msg['Subject'] = subject msg['To'] = to self._commandq.enqueue( msg, dict(listid='test.example.com', subaddress='confirm')) self._runner.run() # Anne is now a confirmed member so her user record and email address # should exist in the database. manager = getUtility(IUserManager) user = manager.get_user('*****@*****.**') self.assertEqual(list(user.addresses)[0].email, '*****@*****.**') # Make sure that the confirmation was not attempted twice. messages = get_queue_messages('virgin') self.assertEqual(len(messages), 1) # Search the contents of the results message. There should be just # one 'Confirmation email' line. confirmation_lines = [] in_results = False for line in body_line_iterator(messages[0].msg): line = line.strip() if in_results: if line.startswith('- Done'): break if len(line) > 0: confirmation_lines.append(line) if line.strip() == '- Results:': in_results = True self.assertEqual(len(confirmation_lines), 1) self.assertFalse('did not match' in confirmation_lines[0])
def test_double_confirmation(self): # A join request comes in using both the -join address and the word # 'subscribe' in the first line of the body. This should produce just # one subscription request and one confirmation response. msg = mfs("""\ From: [email protected] To: [email protected] subscribe """) # Adding the subaddress to the metadata dictionary mimics what happens # when the above email message is first processed by the lmtp runner. # For convenience, we skip that step in this test. self._commandq.enqueue(msg, dict(listname='*****@*****.**', subaddress='join')) self._runner.run() # There will be two messages in the queue. The first one is a reply # to Anne notifying her of the status of her command email. The # second one is the confirmation message of her join request. messages = get_queue_messages('virgin', sort_on='subject') self.assertEqual(len(messages), 2) self.assertTrue(str(messages[1].msg['subject']).startswith('confirm')) self.assertEqual(messages[0].msg['subject'], 'The results of your email commands') # Search the contents of the results message. There should be just # one 'Confirmation email' line. confirmation_lines = [] in_results = False for line in body_line_iterator(messages[0].msg, decode=True): line = line.strip() if in_results: if line.startswith('- Done'): break if len(line) > 0: confirmation_lines.append(line) if line.strip() == '- Results:': in_results = True # There should be exactly one confirmation line. self.assertEqual(len(confirmation_lines), 1) # And the confirmation line should name Anne's email address. self.assertTrue('*****@*****.**' in confirmation_lines[0])
def test_double_confirmation(self): # 'confirm' in the Subject and in the To header should not try to # confirm the token twice. # # Clear out the virgin queue so that the test below only sees the # reply to the confirmation message. get_queue_messages('virgin') subject = 'Re: confirm {0}'.format(self._token) to = 'test-confirm+{0}@example.com'.format(self._token) msg = mfs("""\ From: Anne Person <*****@*****.**> """) msg['Subject'] = subject msg['To'] = to self._commandq.enqueue(msg, dict(listid='test.example.com', subaddress='confirm')) self._runner.run() # Anne is now a confirmed member so her user record and email address # should exist in the database. manager = getUtility(IUserManager) user = manager.get_user('*****@*****.**') self.assertEqual(list(user.addresses)[0].email, '*****@*****.**') # Make sure that the confirmation was not attempted twice. messages = get_queue_messages('virgin') self.assertEqual(len(messages), 1) # Search the contents of the results message. There should be just # one 'Confirmation email' line. confirmation_lines = [] in_results = False for line in body_line_iterator(messages[0].msg): line = line.strip() if in_results: if line.startswith('- Done'): break if len(line) > 0: confirmation_lines.append(line) if line.strip() == '- Results:': in_results = True self.assertEqual(len(confirmation_lines), 1) self.assertFalse('did not match' in confirmation_lines[0])
def process(self, msg): """See `IBounceDetector`.""" addresses = set() it = body_line_iterator(msg) # Find the start line. for line in it: if scre.search(line): break else: return NoFailures # Search each line until we hit the end line. for line in it: if ecre.search(line): break mo = a1cre.search(line) if not mo: mo = a2cre.search(line) if mo: # For Python 3 compatibility, the API requires bytes address = mo.group('addr').encode('us-ascii') addresses.add(address) return NoTemporaryFailures, set(addresses)
def test_join_when_already_a_member(self): anne = getUtility(IUserManager).create_user("*****@*****.**") self._mlist.subscribe(list(anne.addresses)[0]) # When someone tries to join by email and they are already a member, # ignore the request. msg = mfs( """\ From: [email protected] To: [email protected] Subject: join """ ) self._commandq.enqueue(msg, dict(listid="test.example.com")) self._runner.run() # There will be one message in the queue - a reply to Anne notifying # her of the status of her command email. Because Anne is already # subscribed to the list, she gets and needs no confirmation. messages = get_queue_messages("virgin") self.assertEqual(len(messages), 1) self.assertEqual(messages[0].msg["subject"], "The results of your email commands") # Search the contents of the results message. There should be just # one 'Confirmation email' line. confirmation_lines = [] in_results = False for line in body_line_iterator(messages[0].msg): line = line.strip() if in_results: if line.startswith("- Done"): break if len(line) > 0: confirmation_lines.append(line) if line.strip() == "- Results:": in_results = True # There should be exactly one confirmation line. self.assertEqual(len(confirmation_lines), 1) # And the confirmation line should name Anne's email address. self.assertTrue("*****@*****.**" in confirmation_lines[0])
def process(self, msg): """See `IBounceDetector`.""" # Yahoo! bounces seem to have a known subject value and something # called an x-uidl: header, the value of which seems unimportant. sender = parseaddr(msg.get('from', '').lower())[1] or '' if not sender.startswith('mailer-daemon@yahoo'): return NoFailures addresses = set() state = ParseState.start for line in body_line_iterator(msg): line = line.strip() if state is ParseState.start and tcre.match(line): state = ParseState.tag_seen elif state is ParseState.tag_seen: mo = acre.match(line) if mo: addresses.add(mo.group('addr').encode('us-ascii')) continue mo = ecre.match(line) if mo: # We're at the end of the error response. break return NoTemporaryFailures, addresses
def test_join_when_already_a_member(self): anne = getUtility(IUserManager).create_user('*****@*****.**') self._mlist.subscribe(list(anne.addresses)[0]) # When someone tries to join by email and they are already a member, # ignore the request. msg = mfs("""\ From: [email protected] To: [email protected] Subject: join """) self._commandq.enqueue(msg, dict(listid='test.example.com')) self._runner.run() # There will be one message in the queue - a reply to Anne notifying # her of the status of her command email. Because Anne is already # subscribed to the list, she gets and needs no confirmation. messages = get_queue_messages('virgin') self.assertEqual(len(messages), 1) self.assertEqual(messages[0].msg['subject'], 'The results of your email commands') # Search the contents of the results message. There should be just # one 'Confirmation email' line. confirmation_lines = [] in_results = False for line in body_line_iterator(messages[0].msg): line = line.strip() if in_results: if line.startswith('- Done'): break if len(line) > 0: confirmation_lines.append(line) if line.strip() == '- Results:': in_results = True # There should be exactly one confirmation line. self.assertEqual(len(confirmation_lines), 1) # And the confirmation line should name Anne's email address. self.assertTrue('*****@*****.**' in confirmation_lines[0])
def parse(self, m, prefix=None): """Parse messages sent by the 'buildbot-cvs-mail' program. """ # The mail is sent from the person doing the checkin. Assume that the # local username is enough to identify them (this assumes a one-server # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS # model) name, addr = parseaddr(m["from"]) if not addr: return None # no From means this message isn't from buildbot-cvs-mail at = addr.find("@") if at == -1: author = addr # might still be useful else: author = addr[:at] author = util.ascii2unicode(author) # CVS accepts RFC822 dates. buildbot-cvs-mail adds the date as # part of the mail header, so use that. # This assumes cvs is being access via ssh or pserver, so the time # will be the CVS server's time. # calculate a "revision" based on that timestamp, or the current time # if we're unable to parse the date. log.msg('Processing CVS mail') dateTuple = parsedate_tz(m["date"]) if dateTuple is None: when = util.now() else: when = mktime_tz(dateTuple) theTime = datetime.datetime.utcfromtimestamp(float(when)) rev = theTime.strftime('%Y-%m-%d %H:%M:%S') catRE = re.compile(r'^Category:\s*(\S.*)') cvsRE = re.compile(r'^CVSROOT:\s*(\S.*)') cvsmodeRE = re.compile(r'^Cvsmode:\s*(\S.*)') filesRE = re.compile(r'^Files:\s*(\S.*)') modRE = re.compile(r'^Module:\s*(\S.*)') pathRE = re.compile(r'^Path:\s*(\S.*)') projRE = re.compile(r'^Project:\s*(\S.*)') singleFileRE = re.compile(r'(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)') tagRE = re.compile(r'^\s+Tag:\s*(\S.*)') updateRE = re.compile(r'^Update of:\s*(\S.*)') comments = "" branch = None cvsroot = None fileList = None files = [] isdir = 0 path = None project = None lines = list(body_line_iterator(m)) while lines: line = lines.pop(0) m = catRE.match(line) if m: category = m.group(1) continue m = cvsRE.match(line) if m: cvsroot = m.group(1) continue m = cvsmodeRE.match(line) if m: cvsmode = m.group(1) continue m = filesRE.match(line) if m: fileList = m.group(1) continue m = modRE.match(line) if m: # We don't actually use this # module = m.group(1) continue m = pathRE.match(line) if m: path = m.group(1) continue m = projRE.match(line) if m: project = m.group(1) continue m = tagRE.match(line) if m: branch = m.group(1) continue m = updateRE.match(line) if m: # We don't actually use this # updateof = m.group(1) continue if line == "Log Message:\n": break # CVS 1.11 lists files as: # repo/path file,old-version,new-version file2,old-version,new-version # Version 1.12 lists files as: # file1 old-version new-version file2 old-version new-version # # files consists of tuples of 'file-name old-version new-version' # The versions are either dotted-decimal version numbers, ie 1.1 # or NONE. New files are of the form 'NONE NUMBER', while removed # files are 'NUMBER NONE'. 'NONE' is a literal string # Parsing this instead of files list in 'Added File:' etc # makes it possible to handle files with embedded spaces, though # it could fail if the filename was 'bad 1.1 1.2' # For cvs version 1.11, we expect # my_module new_file.c,NONE,1.1 # my_module removed.txt,1.2,NONE # my_module modified_file.c,1.1,1.2 # While cvs version 1.12 gives us # new_file.c NONE 1.1 # removed.txt 1.2 NONE # modified_file.c 1.1,1.2 if fileList is None: log.msg('CVSMaildirSource Mail with no files. Ignoring') return None # We don't have any files. Email not from CVS if cvsmode == '1.11': # Please, no repo paths with spaces! m = re.search('([^ ]*) ', fileList) if m: path = m.group(1) else: log.msg('CVSMaildirSource can\'t get path from file list. Ignoring mail') return fileList = fileList[len(path):].strip() singleFileRE = re.compile(r'(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') elif cvsmode == '1.12': singleFileRE = re.compile(r'(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') if path is None: raise ValueError('CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config') else: raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode) log.msg("CVSMaildirSource processing filelist: %s" % fileList) while(fileList): m = singleFileRE.match(fileList) if m: curFile = path + '/' + m.group(1) files.append(curFile) fileList = fileList[m.end():] else: log.msg('CVSMaildirSource no files matched regex. Ignoring') return None # bail - we couldn't parse the files that changed # Now get comments while lines: line = lines.pop(0) comments += line comments = comments.rstrip() + "\n" if comments == '\n': comments = None return ('cvs', dict(author=author, files=files, comments=comments, isdir=isdir, when=when, branch=branch, revision=rev, category=category, repository=cvsroot, project=project, properties=self.properties))
def parse(self, m, prefix=None): """Parse branch notification messages sent by Launchpad. """ subject = m["subject"] match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject) if match: repository = match.group(1) else: repository = None # Put these into a dictionary, otherwise we cannot assign them # from nested function definitions. d = {'files': [], 'comments': u""} gobbler = None rev = None author = None when = util.now() def gobble_comment(s): d['comments'] += s + "\n" def gobble_removed(s): d['files'].append('%s REMOVED' % s) def gobble_added(s): d['files'].append('%s ADDED' % s) def gobble_modified(s): d['files'].append('%s MODIFIED' % s) def gobble_renamed(s): match = re.search(r"^(.+) => (.+)$", s) if match: d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2))) else: d['files'].append('%s RENAMED' % s) lines = list(body_line_iterator(m, True)) rev = None while lines: line = unicode(lines.pop(0), "utf-8", errors="ignore") # revno: 101 match = re.search(r"^revno: ([0-9.]+)", line) if match: rev = match.group(1) # committer: Joe <*****@*****.**> match = re.search(r"^committer: (.*)$", line) if match: author = match.group(1) # timestamp: Fri 2009-05-15 10:35:43 +0200 # datetime.strptime() is supposed to support %z for time zone, but # it does not seem to work. So handle the time zone manually. match = re.search(r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line) if match: datestr = match.group(1) tz_sign = match.group(2) tz_hours = match.group(3) tz_minutes = match.group(4) when = parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes) if re.search(r"^message:\s*$", line): gobbler = gobble_comment elif re.search(r"^removed:\s*$", line): gobbler = gobble_removed elif re.search(r"^added:\s*$", line): gobbler = gobble_added elif re.search(r"^renamed:\s*$", line): gobbler = gobble_renamed elif re.search(r"^modified:\s*$", line): gobbler = gobble_modified elif re.search(r"^ ", line) and gobbler: gobbler(line[2:-1]) # Use :-1 to gobble trailing newline # Determine the name of the branch. branch = None if self.branchMap and repository: if repository in self.branchMap: branch = self.branchMap[repository] elif ("lp:" + repository) in self.branchMap: branch = self.branchMap['lp:' + repository] if not branch: if self.defaultBranch: branch = self.defaultBranch else: if repository: branch = 'lp:' + repository else: branch = None if rev and author: return ('bzr', dict(author=author, files=d['files'], comments=d['comments'], when=when, revision=rev, branch=branch, repository=repository or '')) else: return None
def parse(self, m, prefix=None): """Parse messages sent by the svn 'commit-email.pl' trigger. """ # The mail is sent from the person doing the checkin. Assume that the # local username is enough to identify them (this assumes a one-server # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS # model) name, addr = parseaddr(m["from"]) if not addr: return None # no From means this message isn't from svn at = addr.find("@") if at == -1: author = addr # might still be useful else: author = addr[:at] # we take the time of receipt as the time of checkin. Not correct (it # depends upon the email latency), but it avoids the # out-of-order-changes issue. Also syncmail doesn't give us anything # better to work with, unless you count pulling the v1-vs-v2 # timestamp out of the diffs, which would be ugly. TODO: Pulling the # 'Date:' header from the mail is a possibility, and # email.Utils.parsedate_tz may be useful. It should be configurable, # however, because there are a lot of broken clocks out there. when = util.now() files = [] comments = "" lines = list(body_line_iterator(m)) rev = None while lines: line = lines.pop(0) # "Author: jmason" match = re.search(r"^Author: (\S+)", line) if match: author = match.group(1) # "New Revision: 105955" match = re.search(r"^New Revision: (\d+)", line) if match: rev = match.group(1) # possible TODO: use "Date: ..." data here instead of time of # commit message receipt, above. however, this timestamp is # specified *without* a timezone, in the server's local TZ, so to # be accurate buildbot would need a config setting to specify the # source server's expected TZ setting! messy. # this stanza ends with the "Log:" if (line == "Log:\n"): break # commit message is terminated by the file-listing section while lines: line = lines.pop(0) if (line == "Modified:\n" or line == "Added:\n" or line == "Removed:\n"): break comments += line comments = comments.rstrip() + "\n" while lines: line = lines.pop(0) if line == "\n": break if line.find("Modified:\n") == 0: continue # ignore this line if line.find("Added:\n") == 0: continue # ignore this line if line.find("Removed:\n") == 0: continue # ignore this line line = line.strip() thesefiles = line.split(" ") for f in thesefiles: if prefix: # insist that the file start with the prefix: we may get # changes we don't care about too if f.startswith(prefix): f = f[len(prefix):] else: log.msg("ignored file from svn commit: prefix '%s' " "does not match filename '%s'" % (prefix, f)) continue # TODO: figure out how new directories are described, set # .isdir files.append(f) if not files: log.msg("no matching files found, ignoring commit") return None return ('svn', dict(author=author, files=files, comments=comments, when=when, revision=rev))
def parse(self, m, prefix=None): """Parse messages sent by the 'buildbot-cvs-mail' program. """ # The mail is sent from the person doing the checkin. Assume that the # local username is enough to identify them (this assumes a one-server # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS # model) name, addr = parseaddr(m["from"]) if not addr: # no From means this message isn't from buildbot-cvs-mail return None at = addr.find("@") if at == -1: author = addr # might still be useful else: author = addr[:at] author = util.bytes2unicode(author, encoding="ascii") # CVS accepts RFC822 dates. buildbot-cvs-mail adds the date as # part of the mail header, so use that. # This assumes cvs is being access via ssh or pserver, so the time # will be the CVS server's time. # calculate a "revision" based on that timestamp, or the current time # if we're unable to parse the date. log.msg('Processing CVS mail') dateTuple = parsedate_tz(m["date"]) if dateTuple is None: when = util.now() else: when = mktime_tz(dateTuple) theTime = datetime.datetime.utcfromtimestamp(float(when)) rev = theTime.strftime('%Y-%m-%d %H:%M:%S') catRE = re.compile(r'^Category:\s*(\S.*)') cvsRE = re.compile(r'^CVSROOT:\s*(\S.*)') cvsmodeRE = re.compile(r'^Cvsmode:\s*(\S.*)') filesRE = re.compile(r'^Files:\s*(\S.*)') modRE = re.compile(r'^Module:\s*(\S.*)') pathRE = re.compile(r'^Path:\s*(\S.*)') projRE = re.compile(r'^Project:\s*(\S.*)') singleFileRE = re.compile(r'(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)') tagRE = re.compile(r'^\s+Tag:\s*(\S.*)') updateRE = re.compile(r'^Update of:\s*(\S.*)') comments = "" branch = None cvsroot = None fileList = None files = [] isdir = 0 path = None project = None lines = list(body_line_iterator(m)) while lines: line = lines.pop(0) m = catRE.match(line) if m: category = m.group(1) continue m = cvsRE.match(line) if m: cvsroot = m.group(1) continue m = cvsmodeRE.match(line) if m: cvsmode = m.group(1) continue m = filesRE.match(line) if m: fileList = m.group(1) continue m = modRE.match(line) if m: # We don't actually use this # module = m.group(1) continue m = pathRE.match(line) if m: path = m.group(1) continue m = projRE.match(line) if m: project = m.group(1) continue m = tagRE.match(line) if m: branch = m.group(1) continue m = updateRE.match(line) if m: # We don't actually use this # updateof = m.group(1) continue if line == "Log Message:\n": break # CVS 1.11 lists files as: # repo/path file,old-version,new-version file2,old-version,new-version # Version 1.12 lists files as: # file1 old-version new-version file2 old-version new-version # # files consists of tuples of 'file-name old-version new-version' # The versions are either dotted-decimal version numbers, ie 1.1 # or NONE. New files are of the form 'NONE NUMBER', while removed # files are 'NUMBER NONE'. 'NONE' is a literal string # Parsing this instead of files list in 'Added File:' etc # makes it possible to handle files with embedded spaces, though # it could fail if the filename was 'bad 1.1 1.2' # For cvs version 1.11, we expect # my_module new_file.c,NONE,1.1 # my_module removed.txt,1.2,NONE # my_module modified_file.c,1.1,1.2 # While cvs version 1.12 gives us # new_file.c NONE 1.1 # removed.txt 1.2 NONE # modified_file.c 1.1,1.2 if fileList is None: log.msg('CVSMaildirSource Mail with no files. Ignoring') return None # We don't have any files. Email not from CVS if cvsmode == '1.11': # Please, no repo paths with spaces! m = re.search('([^ ]*) ', fileList) if m: path = m.group(1) else: log.msg( 'CVSMaildirSource can\'t get path from file list. Ignoring mail') return fileList = fileList[len(path):].strip() singleFileRE = re.compile( r'(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') elif cvsmode == '1.12': singleFileRE = re.compile( r'(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') if path is None: raise ValueError( 'CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config') else: raise ValueError( 'Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode) log.msg("CVSMaildirSource processing filelist: %s" % fileList) while(fileList): m = singleFileRE.match(fileList) if m: curFile = path + '/' + m.group(1) files.append(curFile) fileList = fileList[m.end():] else: log.msg('CVSMaildirSource no files matched regex. Ignoring') return None # bail - we couldn't parse the files that changed # Now get comments while lines: line = lines.pop(0) comments += line comments = comments.rstrip() + "\n" if comments == '\n': comments = None return ('cvs', dict(author=author, files=files, comments=comments, isdir=isdir, when=when, branch=branch, revision=rev, category=category, repository=cvsroot, project=project, properties=self.properties))
def parse(self, m, prefix=None): """Parse branch notification messages sent by Launchpad. """ subject = m["subject"] match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject) if match: repository = match.group(1) else: repository = None # Put these into a dictionary, otherwise we cannot assign them # from nested function definitions. d = {'files': [], 'comments': ""} gobbler = None rev = None author = None when = util.now() def gobble_comment(s): d['comments'] += s + "\n" def gobble_removed(s): d['files'].append('%s REMOVED' % s) def gobble_added(s): d['files'].append('%s ADDED' % s) def gobble_modified(s): d['files'].append('%s MODIFIED' % s) def gobble_renamed(s): match = re.search(r"^(.+) => (.+)$", s) if match: d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2))) else: d['files'].append('%s RENAMED' % s) lines = list(body_line_iterator(m, True)) rev = None while lines: line = text_type(lines.pop(0), "utf-8", errors="ignore") # revno: 101 match = re.search(r"^revno: ([0-9.]+)", line) if match: rev = match.group(1) # committer: Joe <*****@*****.**> match = re.search(r"^committer: (.*)$", line) if match: author = match.group(1) # timestamp: Fri 2009-05-15 10:35:43 +0200 # datetime.strptime() is supposed to support %z for time zone, but # it does not seem to work. So handle the time zone manually. match = re.search( r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line) if match: datestr = match.group(1) tz_sign = match.group(2) tz_hours = match.group(3) tz_minutes = match.group(4) when = parseLaunchpadDate( datestr, tz_sign, tz_hours, tz_minutes) if re.search(r"^message:\s*$", line): gobbler = gobble_comment elif re.search(r"^removed:\s*$", line): gobbler = gobble_removed elif re.search(r"^added:\s*$", line): gobbler = gobble_added elif re.search(r"^renamed:\s*$", line): gobbler = gobble_renamed elif re.search(r"^modified:\s*$", line): gobbler = gobble_modified elif re.search(r"^ ", line) and gobbler: gobbler(line[2:-1]) # Use :-1 to gobble trailing newline # Determine the name of the branch. branch = None if self.branchMap and repository: if repository in self.branchMap: branch = self.branchMap[repository] elif ("lp:" + repository) in self.branchMap: branch = self.branchMap['lp:' + repository] if not branch: if self.defaultBranch: branch = self.defaultBranch else: if repository: branch = 'lp:' + repository else: branch = None if rev and author: return ('bzr', dict(author=author, files=d['files'], comments=d['comments'], when=when, revision=rev, branch=branch, repository=repository or '')) return None
def parse(self, m, prefix=None): """Parse messages sent by the svn 'commit-email.pl' trigger. """ # The mail is sent from the person doing the checkin. Assume that the # local username is enough to identify them (this assumes a one-server # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS # model) name, addr = parseaddr(m["from"]) if not addr: return None # no From means this message isn't from svn at = addr.find("@") if at == -1: author = addr # might still be useful else: author = addr[:at] # we take the time of receipt as the time of checkin. Not correct (it # depends upon the email latency), but it avoids the # out-of-order-changes issue. Also syncmail doesn't give us anything # better to work with, unless you count pulling the v1-vs-v2 # timestamp out of the diffs, which would be ugly. TODO: Pulling the # 'Date:' header from the mail is a possibility, and # email.utils.parsedate_tz may be useful. It should be configurable, # however, because there are a lot of broken clocks out there. when = util.now() files = [] comments = "" lines = list(body_line_iterator(m)) rev = None while lines: line = lines.pop(0) # "Author: jmason" match = re.search(r"^Author: (\S+)", line) if match: author = match.group(1) # "New Revision: 105955" match = re.search(r"^New Revision: (\d+)", line) if match: rev = match.group(1) # possible TODO: use "Date: ..." data here instead of time of # commit message receipt, above. however, this timestamp is # specified *without* a timezone, in the server's local TZ, so to # be accurate buildbot would need a config setting to specify the # source server's expected TZ setting! messy. # this stanza ends with the "Log:" if (line == "Log:\n"): break # commit message is terminated by the file-listing section while lines: line = lines.pop(0) if line in ("Modified:\n", "Added:\n", "Removed:\n"): break comments += line comments = comments.rstrip() + "\n" while lines: line = lines.pop(0) if line == "\n": break if line.find("Modified:\n") == 0: continue # ignore this line if line.find("Added:\n") == 0: continue # ignore this line if line.find("Removed:\n") == 0: continue # ignore this line line = line.strip() thesefiles = line.split(" ") for f in thesefiles: if prefix: # insist that the file start with the prefix: we may get # changes we don't care about too if f.startswith(prefix): f = f[len(prefix):] else: log.msg("ignored file from svn commit: prefix '%s' " "does not match filename '%s'" % (prefix, f)) continue # TODO: figure out how new directories are described, set # .isdir files.append(f) if not files: log.msg("no matching files found, ignoring commit") return None return ('svn', dict(author=author, files=files, comments=comments, when=when, revision=rev))