def process(self, msg):
"""See `IBounceDetector`."""
addresses = set()
state = ParseState.start
for line in body_line_iterator(msg):
line = line.strip()
if state is ParseState.start:
for introtag in introtags:
if line.startswith(introtag.encode("latin1")):
state = ParseState.intro_paragraph_seen
break
elif state is ParseState.intro_paragraph_seen and not line:
# Looking for the end of the intro paragraph.
state = ParseState.recip_paragraph_seen
elif state is ParseState.recip_paragraph_seen:
if line.startswith("-"):
# We're looking at the break paragraph, so we're done.
break
# At this point we know we must be looking at a recipient
# paragraph.
mo = acre.match(line)
if mo:
addresses.add(mo.group("addr").encode("us-ascii"))
# Otherwise, it must be a continuation line, so just ignore it.
else:
# We're not looking at anything in particular.
pass
return NoTemporaryFailures, addresses
def process(self, msg):
"""See `IBounceDetector`."""
addresses = set()
# MAS: This is a mess. The outer loop used to be over the message
# so we only looped through the message once. Looping through the
# message for each set of patterns is obviously way more work, but
# if we don't do it, problems arise because scre from the wrong
# pattern set matches first and then acre doesn't match. The
# alternative is to split things into separate modules, but then
# we process the message multiple times anyway.
for scre, ecre, acre in self.PATTERNS:
state = ParseState.start
for line in body_line_iterator(msg):
if state is ParseState.start:
if scre.search(line):
state = ParseState.tag_seen
if state is ParseState.tag_seen:
mo = acre.search(line)
if mo:
address = mo.group('addr')
if address:
addresses.add(_quopri_decode(address))
elif ecre.search(line):
break
if len(addresses) > 0:
break
return NoTemporaryFailures, addresses
def process(self, msg):
"""See `IBounceDetector`."""
addresses = set()
# MAS: This is a mess. The outer loop used to be over the message
# so we only looped through the message once. Looping through the
# message for each set of patterns is obviously way more work, but
# if we don't do it, problems arise because scre from the wrong
# pattern set matches first and then acre doesn't match. The
# alternative is to split things into separate modules, but then
# we process the message multiple times anyway.
for scre, ecre, acre in self.PATTERNS:
state = ParseState.start
for line in body_line_iterator(msg):
if state is ParseState.start:
if scre.search(line):
state = ParseState.tag_seen
if state is ParseState.tag_seen:
mo = acre.search(line)
if mo:
address = mo.group('addr')
if address:
addresses.add(_quopri_decode(address))
elif ecre.search(line):
break
if len(addresses) > 0:
break
return NoTemporaryFailures, addresses
def process(self, msg):
"""See `IBounceDetector`."""
addresses = set()
state = ParseState.start
for line in body_line_iterator(msg):
line = line.strip()
if state is ParseState.start:
for introtag in introtags:
if line.startswith(introtag.encode('latin1')):
state = ParseState.intro_paragraph_seen
break
elif state is ParseState.intro_paragraph_seen and not line:
# Looking for the end of the intro paragraph.
state = ParseState.recip_paragraph_seen
elif state is ParseState.recip_paragraph_seen:
if line.startswith('-'):
# We're looking at the break paragraph, so we're done.
break
# At this point we know we must be looking at a recipient
# paragraph.
mo = acre.match(line)
if mo:
addresses.add(mo.group('addr').encode('us-ascii'))
# Otherwise, it must be a continuation line, so just ignore it.
else:
# We're not looking at anything in particular.
pass
return NoTemporaryFailures, addresses
def process(self, msg):
"""See `IBounceDetector`."""
# Yahoo! bounces seem to have a known subject value and something
# called an x-uidl: header, the value of which seems unimportant.
sender = parseaddr(msg.get('from', '').lower())[1] or ''
if not sender.startswith('mailer-daemon@yahoo'):
return NoFailures
addresses = set()
state = _ParseState.start
for line in body_line_iterator(msg):
line = line.strip()
if state is _ParseState.start:
for cre in tcre:
if cre.match(line):
state = _ParseState.tag_seen
break
elif state is _ParseState.tag_seen:
mo = acre.match(line)
if mo:
addresses.add(mo.group('addr').encode('us-ascii'))
continue
for cre in ecre:
mo = cre.match(line)
if mo:
# We're at the end of the error response.
state = _ParseState.all_done
break
if state is _ParseState.all_done:
break
return NoTemporaryFailures, addresses
def process(self, msg):
"""See `IBounceDetector`."""
for line in body_line_iterator(msg):
mo = acre.search(line)
if mo:
address = mo.group('addr').encode('us-ascii')
return NoTemporaryFailures, set([address])
return NoFailures
def process(self, msg):
"""See `IBounceDetector`."""
for line in body_line_iterator(msg):
mo = acre.search(line)
if mo:
address = mo.group('addr').encode('us-ascii')
return NoTemporaryFailures, set([address])
return NoFailures
def process(self, msg):
mailer = msg.get('x-mailer', '')
if not mailer.startswith('<SMTP32 v'):
return NoFailures
addresses = set()
for line in body_line_iterator(msg):
if ecre.search(line):
break
mo = acre.search(line)
if mo:
addresses.add(mo.group('addr').encode('us-ascii'))
return NoTemporaryFailures, addresses
def process(self, msg):
mailer = msg.get('x-mailer', '')
if not mailer.startswith('<SMTP32 v'):
return NoFailures
addresses = set()
for line in body_line_iterator(msg):
if ecre.search(line):
break
mo = acre.search(line)
if mo:
addresses.add(mo.group('addr').encode('us-ascii'))
return NoTemporaryFailures, addresses
def confirmation_line(msg):
confirmation_lines = []
in_results = False
for line in body_line_iterator(msg):
line = line.strip()
if in_results:
if line.startswith('- Done'):
break
if len(line) > 0:
confirmation_lines.append(line)
if line.strip() == '- Results:':
in_results = True
# There should be exactly one confirmation line.
assert len(confirmation_lines) == 1, confirmation_lines
return confirmation_lines[0]
def process(self, msg):
if msg.get_content_type() == 'multipart/mixed':
state = ParseState.start
# This format thinks it's a MIME, but it really isn't.
for line in body_line_iterator(msg):
line = line.strip()
if state is ParseState.start and tcre.match(line):
state = ParseState.tag_seen
elif state is ParseState.tag_seen and line:
mo = acre.match(line)
if mo:
return NoTemporaryFailures, set(mo.group('addr'))
else:
break
return NoFailures
def process(self, msg):
if msg.get_content_type() == 'multipart/mixed':
state = ParseState.start
# This format thinks it's a MIME, but it really isn't.
for line in body_line_iterator(msg):
line = line.strip()
if state is ParseState.start and tcre.match(line):
state = ParseState.tag_seen
elif state is ParseState.tag_seen and line:
mo = acre.match(line)
if mo:
return NoTemporaryFailures, set(mo.group('addr'))
else:
break
return NoFailures
def test_double_confirmation(self):
# A join request comes in using both the -join address and the word
# 'subscribe' in the first line of the body. This should produce just
# one subscription request and one confirmation response.
msg = mfs("""\
From: [email protected]
To: [email protected]
subscribe
""")
# Adding the subaddress to the metadata dictionary mimics what happens
# when the above email message is first processed by the lmtp runner.
# For convenience, we skip that step in this test.
self._commandq.enqueue(
msg, dict(listid='test.example.com', subaddress='join'))
self._runner.run()
# There will be two messages in the queue. The first one is a reply
# to Anne notifying her of the status of her command email. The
# second one is the confirmation message of her join request.
items = get_queue_messages('virgin',
sort_on='subject',
expected_count=2)
self.assertTrue(str(items[1].msg['subject']).startswith('confirm'))
self.assertEqual(items[0].msg['subject'],
'The results of your email commands')
# Search the contents of the results message. There should be just
# one 'Confirmation email' line.
confirmation_lines = []
in_results = False
for line in body_line_iterator(items[0].msg):
line = line.strip()
if in_results:
if line.startswith('- Done'):
break
if len(line) > 0:
confirmation_lines.append(line)
if line.strip() == '- Results:':
in_results = True
# There should be exactly one confirmation line.
self.assertEqual(len(confirmation_lines), 1)
# And the confirmation line should name Anne's email address.
self.assertIn('*****@*****.**', confirmation_lines[0])
def process(self, msg):
"""See `IBounceDetector`."""
if msg.get('from', '').lower() != '*****@*****.**':
return NoFailures
if not msg.is_multipart():
return NoFailures
# The interesting bits are in the first text/plain multipart.
part = None
try:
part = msg.get_payload(0)
except IndexError:
pass
if not part:
return NoFailures
addresses = set()
for line in body_line_iterator(part):
mo = acre.match(line)
if mo:
addresses.add(mo.group('addr').encode('us-ascii'))
return NoTemporaryFailures, addresses
def process(self, msg):
"""See `IBounceDetector`."""
if msg.get('from', '').lower() != '*****@*****.**':
return NoFailures
if not msg.is_multipart():
return NoFailures
# The interesting bits are in the first text/plain multipart.
part = None
try:
part = msg.get_payload(0)
except IndexError:
pass
if not part:
return NoFailures
addresses = set()
for line in body_line_iterator(part):
mo = acre.match(line)
if mo:
addresses.add(mo.group('addr').encode('us-ascii'))
return NoTemporaryFailures, addresses
def test_double_confirmation(self):
# 'confirm' in the Subject and in the To header should not try to
# confirm the token twice.
#
# Clear out the virgin queue so that the test below only sees the
# reply to the confirmation message.
get_queue_messages('virgin')
subject = 'Re: confirm {0}'.format(self._token)
to = 'test-confirm+{0}@example.com'.format(self._token)
msg = mfs("""\
From: Anne Person <*****@*****.**>
""")
msg['Subject'] = subject
msg['To'] = to
self._commandq.enqueue(
msg, dict(listid='test.example.com', subaddress='confirm'))
self._runner.run()
# Anne is now a confirmed member so her user record and email address
# should exist in the database.
manager = getUtility(IUserManager)
user = manager.get_user('*****@*****.**')
self.assertEqual(list(user.addresses)[0].email, '*****@*****.**')
# Make sure that the confirmation was not attempted twice.
messages = get_queue_messages('virgin')
self.assertEqual(len(messages), 1)
# Search the contents of the results message. There should be just
# one 'Confirmation email' line.
confirmation_lines = []
in_results = False
for line in body_line_iterator(messages[0].msg):
line = line.strip()
if in_results:
if line.startswith('- Done'):
break
if len(line) > 0:
confirmation_lines.append(line)
if line.strip() == '- Results:':
in_results = True
self.assertEqual(len(confirmation_lines), 1)
self.assertFalse('did not match' in confirmation_lines[0])
def test_double_confirmation(self):
# A join request comes in using both the -join address and the word
# 'subscribe' in the first line of the body. This should produce just
# one subscription request and one confirmation response.
msg = mfs("""\
From: [email protected]
To: [email protected]
subscribe
""")
# Adding the subaddress to the metadata dictionary mimics what happens
# when the above email message is first processed by the lmtp runner.
# For convenience, we skip that step in this test.
self._commandq.enqueue(msg, dict(listname='*****@*****.**',
subaddress='join'))
self._runner.run()
# There will be two messages in the queue. The first one is a reply
# to Anne notifying her of the status of her command email. The
# second one is the confirmation message of her join request.
messages = get_queue_messages('virgin', sort_on='subject')
self.assertEqual(len(messages), 2)
self.assertTrue(str(messages[1].msg['subject']).startswith('confirm'))
self.assertEqual(messages[0].msg['subject'],
'The results of your email commands')
# Search the contents of the results message. There should be just
# one 'Confirmation email' line.
confirmation_lines = []
in_results = False
for line in body_line_iterator(messages[0].msg, decode=True):
line = line.strip()
if in_results:
if line.startswith('- Done'):
break
if len(line) > 0:
confirmation_lines.append(line)
if line.strip() == '- Results:':
in_results = True
# There should be exactly one confirmation line.
self.assertEqual(len(confirmation_lines), 1)
# And the confirmation line should name Anne's email address.
self.assertTrue('*****@*****.**' in confirmation_lines[0])
def test_double_confirmation(self):
# 'confirm' in the Subject and in the To header should not try to
# confirm the token twice.
#
# Clear out the virgin queue so that the test below only sees the
# reply to the confirmation message.
get_queue_messages('virgin')
subject = 'Re: confirm {0}'.format(self._token)
to = 'test-confirm+{0}@example.com'.format(self._token)
msg = mfs("""\
From: Anne Person <*****@*****.**>
""")
msg['Subject'] = subject
msg['To'] = to
self._commandq.enqueue(msg, dict(listid='test.example.com',
subaddress='confirm'))
self._runner.run()
# Anne is now a confirmed member so her user record and email address
# should exist in the database.
manager = getUtility(IUserManager)
user = manager.get_user('*****@*****.**')
self.assertEqual(list(user.addresses)[0].email, '*****@*****.**')
# Make sure that the confirmation was not attempted twice.
messages = get_queue_messages('virgin')
self.assertEqual(len(messages), 1)
# Search the contents of the results message. There should be just
# one 'Confirmation email' line.
confirmation_lines = []
in_results = False
for line in body_line_iterator(messages[0].msg):
line = line.strip()
if in_results:
if line.startswith('- Done'):
break
if len(line) > 0:
confirmation_lines.append(line)
if line.strip() == '- Results:':
in_results = True
self.assertEqual(len(confirmation_lines), 1)
self.assertFalse('did not match' in confirmation_lines[0])
def process(self, msg):
"""See `IBounceDetector`."""
addresses = set()
it = body_line_iterator(msg)
# Find the start line.
for line in it:
if scre.search(line):
break
else:
return NoFailures
# Search each line until we hit the end line.
for line in it:
if ecre.search(line):
break
mo = a1cre.search(line)
if not mo:
mo = a2cre.search(line)
if mo:
# For Python 3 compatibility, the API requires bytes
address = mo.group('addr').encode('us-ascii')
addresses.add(address)
return NoTemporaryFailures, set(addresses)
def process(self, msg):
"""See `IBounceDetector`."""
addresses = set()
it = body_line_iterator(msg)
# Find the start line.
for line in it:
if scre.search(line):
break
else:
return NoFailures
# Search each line until we hit the end line.
for line in it:
if ecre.search(line):
break
mo = a1cre.search(line)
if not mo:
mo = a2cre.search(line)
if mo:
# For Python 3 compatibility, the API requires bytes
address = mo.group('addr').encode('us-ascii')
addresses.add(address)
return NoTemporaryFailures, set(addresses)
def test_join_when_already_a_member(self):
anne = getUtility(IUserManager).create_user("*****@*****.**")
self._mlist.subscribe(list(anne.addresses)[0])
# When someone tries to join by email and they are already a member,
# ignore the request.
msg = mfs(
"""\
From: [email protected]
To: [email protected]
Subject: join
"""
)
self._commandq.enqueue(msg, dict(listid="test.example.com"))
self._runner.run()
# There will be one message in the queue - a reply to Anne notifying
# her of the status of her command email. Because Anne is already
# subscribed to the list, she gets and needs no confirmation.
messages = get_queue_messages("virgin")
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0].msg["subject"], "The results of your email commands")
# Search the contents of the results message. There should be just
# one 'Confirmation email' line.
confirmation_lines = []
in_results = False
for line in body_line_iterator(messages[0].msg):
line = line.strip()
if in_results:
if line.startswith("- Done"):
break
if len(line) > 0:
confirmation_lines.append(line)
if line.strip() == "- Results:":
in_results = True
# There should be exactly one confirmation line.
self.assertEqual(len(confirmation_lines), 1)
# And the confirmation line should name Anne's email address.
self.assertTrue("*****@*****.**" in confirmation_lines[0])
def process(self, msg):
"""See `IBounceDetector`."""
# Yahoo! bounces seem to have a known subject value and something
# called an x-uidl: header, the value of which seems unimportant.
sender = parseaddr(msg.get('from', '').lower())[1] or ''
if not sender.startswith('mailer-daemon@yahoo'):
return NoFailures
addresses = set()
state = ParseState.start
for line in body_line_iterator(msg):
line = line.strip()
if state is ParseState.start and tcre.match(line):
state = ParseState.tag_seen
elif state is ParseState.tag_seen:
mo = acre.match(line)
if mo:
addresses.add(mo.group('addr').encode('us-ascii'))
continue
mo = ecre.match(line)
if mo:
# We're at the end of the error response.
break
return NoTemporaryFailures, addresses
def test_join_when_already_a_member(self):
anne = getUtility(IUserManager).create_user('*****@*****.**')
self._mlist.subscribe(list(anne.addresses)[0])
# When someone tries to join by email and they are already a member,
# ignore the request.
msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: join
""")
self._commandq.enqueue(msg, dict(listid='test.example.com'))
self._runner.run()
# There will be one message in the queue - a reply to Anne notifying
# her of the status of her command email. Because Anne is already
# subscribed to the list, she gets and needs no confirmation.
messages = get_queue_messages('virgin')
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0].msg['subject'],
'The results of your email commands')
# Search the contents of the results message. There should be just
# one 'Confirmation email' line.
confirmation_lines = []
in_results = False
for line in body_line_iterator(messages[0].msg):
line = line.strip()
if in_results:
if line.startswith('- Done'):
break
if len(line) > 0:
confirmation_lines.append(line)
if line.strip() == '- Results:':
in_results = True
# There should be exactly one confirmation line.
self.assertEqual(len(confirmation_lines), 1)
# And the confirmation line should name Anne's email address.
self.assertTrue('*****@*****.**' in confirmation_lines[0])
def parse(self, m, prefix=None):
"""Parse messages sent by the 'buildbot-cvs-mail' program.
"""
# The mail is sent from the person doing the checkin. Assume that the
# local username is enough to identify them (this assumes a one-server
# cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
# model)
name, addr = parseaddr(m["from"])
if not addr:
return None # no From means this message isn't from buildbot-cvs-mail
at = addr.find("@")
if at == -1:
author = addr # might still be useful
else:
author = addr[:at]
author = util.ascii2unicode(author)
# CVS accepts RFC822 dates. buildbot-cvs-mail adds the date as
# part of the mail header, so use that.
# This assumes cvs is being access via ssh or pserver, so the time
# will be the CVS server's time.
# calculate a "revision" based on that timestamp, or the current time
# if we're unable to parse the date.
log.msg('Processing CVS mail')
dateTuple = parsedate_tz(m["date"])
if dateTuple is None:
when = util.now()
else:
when = mktime_tz(dateTuple)
theTime = datetime.datetime.utcfromtimestamp(float(when))
rev = theTime.strftime('%Y-%m-%d %H:%M:%S')
catRE = re.compile(r'^Category:\s*(\S.*)')
cvsRE = re.compile(r'^CVSROOT:\s*(\S.*)')
cvsmodeRE = re.compile(r'^Cvsmode:\s*(\S.*)')
filesRE = re.compile(r'^Files:\s*(\S.*)')
modRE = re.compile(r'^Module:\s*(\S.*)')
pathRE = re.compile(r'^Path:\s*(\S.*)')
projRE = re.compile(r'^Project:\s*(\S.*)')
singleFileRE = re.compile(r'(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)')
tagRE = re.compile(r'^\s+Tag:\s*(\S.*)')
updateRE = re.compile(r'^Update of:\s*(\S.*)')
comments = ""
branch = None
cvsroot = None
fileList = None
files = []
isdir = 0
path = None
project = None
lines = list(body_line_iterator(m))
while lines:
line = lines.pop(0)
m = catRE.match(line)
if m:
category = m.group(1)
continue
m = cvsRE.match(line)
if m:
cvsroot = m.group(1)
continue
m = cvsmodeRE.match(line)
if m:
cvsmode = m.group(1)
continue
m = filesRE.match(line)
if m:
fileList = m.group(1)
continue
m = modRE.match(line)
if m:
# We don't actually use this
# module = m.group(1)
continue
m = pathRE.match(line)
if m:
path = m.group(1)
continue
m = projRE.match(line)
if m:
project = m.group(1)
continue
m = tagRE.match(line)
if m:
branch = m.group(1)
continue
m = updateRE.match(line)
if m:
# We don't actually use this
# updateof = m.group(1)
continue
if line == "Log Message:\n":
break
# CVS 1.11 lists files as:
# repo/path file,old-version,new-version file2,old-version,new-version
# Version 1.12 lists files as:
# file1 old-version new-version file2 old-version new-version
#
# files consists of tuples of 'file-name old-version new-version'
# The versions are either dotted-decimal version numbers, ie 1.1
# or NONE. New files are of the form 'NONE NUMBER', while removed
# files are 'NUMBER NONE'. 'NONE' is a literal string
# Parsing this instead of files list in 'Added File:' etc
# makes it possible to handle files with embedded spaces, though
# it could fail if the filename was 'bad 1.1 1.2'
# For cvs version 1.11, we expect
# my_module new_file.c,NONE,1.1
# my_module removed.txt,1.2,NONE
# my_module modified_file.c,1.1,1.2
# While cvs version 1.12 gives us
# new_file.c NONE 1.1
# removed.txt 1.2 NONE
# modified_file.c 1.1,1.2
if fileList is None:
log.msg('CVSMaildirSource Mail with no files. Ignoring')
return None # We don't have any files. Email not from CVS
if cvsmode == '1.11':
# Please, no repo paths with spaces!
m = re.search('([^ ]*) ', fileList)
if m:
path = m.group(1)
else:
log.msg('CVSMaildirSource can\'t get path from file list. Ignoring mail')
return
fileList = fileList[len(path):].strip()
singleFileRE = re.compile(r'(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)')
elif cvsmode == '1.12':
singleFileRE = re.compile(r'(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)')
if path is None:
raise ValueError('CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config')
else:
raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode)
log.msg("CVSMaildirSource processing filelist: %s" % fileList)
while(fileList):
m = singleFileRE.match(fileList)
if m:
curFile = path + '/' + m.group(1)
files.append(curFile)
fileList = fileList[m.end():]
else:
log.msg('CVSMaildirSource no files matched regex. Ignoring')
return None # bail - we couldn't parse the files that changed
# Now get comments
while lines:
line = lines.pop(0)
comments += line
comments = comments.rstrip() + "\n"
if comments == '\n':
comments = None
return ('cvs', dict(author=author, files=files, comments=comments,
isdir=isdir, when=when, branch=branch,
revision=rev, category=category,
repository=cvsroot, project=project,
properties=self.properties))
def parse(self, m, prefix=None):
"""Parse branch notification messages sent by Launchpad.
"""
subject = m["subject"]
match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject)
if match:
repository = match.group(1)
else:
repository = None
# Put these into a dictionary, otherwise we cannot assign them
# from nested function definitions.
d = {'files': [], 'comments': u""}
gobbler = None
rev = None
author = None
when = util.now()
def gobble_comment(s):
d['comments'] += s + "\n"
def gobble_removed(s):
d['files'].append('%s REMOVED' % s)
def gobble_added(s):
d['files'].append('%s ADDED' % s)
def gobble_modified(s):
d['files'].append('%s MODIFIED' % s)
def gobble_renamed(s):
match = re.search(r"^(.+) => (.+)$", s)
if match:
d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2)))
else:
d['files'].append('%s RENAMED' % s)
lines = list(body_line_iterator(m, True))
rev = None
while lines:
line = unicode(lines.pop(0), "utf-8", errors="ignore")
# revno: 101
match = re.search(r"^revno: ([0-9.]+)", line)
if match:
rev = match.group(1)
# committer: Joe <*****@*****.**>
match = re.search(r"^committer: (.*)$", line)
if match:
author = match.group(1)
# timestamp: Fri 2009-05-15 10:35:43 +0200
# datetime.strptime() is supposed to support %z for time zone, but
# it does not seem to work. So handle the time zone manually.
match = re.search(r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line)
if match:
datestr = match.group(1)
tz_sign = match.group(2)
tz_hours = match.group(3)
tz_minutes = match.group(4)
when = parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes)
if re.search(r"^message:\s*$", line):
gobbler = gobble_comment
elif re.search(r"^removed:\s*$", line):
gobbler = gobble_removed
elif re.search(r"^added:\s*$", line):
gobbler = gobble_added
elif re.search(r"^renamed:\s*$", line):
gobbler = gobble_renamed
elif re.search(r"^modified:\s*$", line):
gobbler = gobble_modified
elif re.search(r"^ ", line) and gobbler:
gobbler(line[2:-1]) # Use :-1 to gobble trailing newline
# Determine the name of the branch.
branch = None
if self.branchMap and repository:
if repository in self.branchMap:
branch = self.branchMap[repository]
elif ("lp:" + repository) in self.branchMap:
branch = self.branchMap['lp:' + repository]
if not branch:
if self.defaultBranch:
branch = self.defaultBranch
else:
if repository:
branch = 'lp:' + repository
else:
branch = None
if rev and author:
return ('bzr', dict(author=author, files=d['files'],
comments=d['comments'],
when=when, revision=rev,
branch=branch, repository=repository or ''))
else:
return None
def parse(self, m, prefix=None):
"""Parse messages sent by the svn 'commit-email.pl' trigger.
"""
# The mail is sent from the person doing the checkin. Assume that the
# local username is enough to identify them (this assumes a one-server
# cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
# model)
name, addr = parseaddr(m["from"])
if not addr:
return None # no From means this message isn't from svn
at = addr.find("@")
if at == -1:
author = addr # might still be useful
else:
author = addr[:at]
# we take the time of receipt as the time of checkin. Not correct (it
# depends upon the email latency), but it avoids the
# out-of-order-changes issue. Also syncmail doesn't give us anything
# better to work with, unless you count pulling the v1-vs-v2
# timestamp out of the diffs, which would be ugly. TODO: Pulling the
# 'Date:' header from the mail is a possibility, and
# email.Utils.parsedate_tz may be useful. It should be configurable,
# however, because there are a lot of broken clocks out there.
when = util.now()
files = []
comments = ""
lines = list(body_line_iterator(m))
rev = None
while lines:
line = lines.pop(0)
# "Author: jmason"
match = re.search(r"^Author: (\S+)", line)
if match:
author = match.group(1)
# "New Revision: 105955"
match = re.search(r"^New Revision: (\d+)", line)
if match:
rev = match.group(1)
# possible TODO: use "Date: ..." data here instead of time of
# commit message receipt, above. however, this timestamp is
# specified *without* a timezone, in the server's local TZ, so to
# be accurate buildbot would need a config setting to specify the
# source server's expected TZ setting! messy.
# this stanza ends with the "Log:"
if (line == "Log:\n"):
break
# commit message is terminated by the file-listing section
while lines:
line = lines.pop(0)
if (line == "Modified:\n" or
line == "Added:\n" or
line == "Removed:\n"):
break
comments += line
comments = comments.rstrip() + "\n"
while lines:
line = lines.pop(0)
if line == "\n":
break
if line.find("Modified:\n") == 0:
continue # ignore this line
if line.find("Added:\n") == 0:
continue # ignore this line
if line.find("Removed:\n") == 0:
continue # ignore this line
line = line.strip()
thesefiles = line.split(" ")
for f in thesefiles:
if prefix:
# insist that the file start with the prefix: we may get
# changes we don't care about too
if f.startswith(prefix):
f = f[len(prefix):]
else:
log.msg("ignored file from svn commit: prefix '%s' "
"does not match filename '%s'" % (prefix, f))
continue
# TODO: figure out how new directories are described, set
# .isdir
files.append(f)
if not files:
log.msg("no matching files found, ignoring commit")
return None
return ('svn', dict(author=author, files=files, comments=comments,
when=when, revision=rev))
def parse(self, m, prefix=None):
"""Parse messages sent by the 'buildbot-cvs-mail' program.
"""
# The mail is sent from the person doing the checkin. Assume that the
# local username is enough to identify them (this assumes a one-server
# cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
# model)
name, addr = parseaddr(m["from"])
if not addr:
# no From means this message isn't from buildbot-cvs-mail
return None
at = addr.find("@")
if at == -1:
author = addr # might still be useful
else:
author = addr[:at]
author = util.bytes2unicode(author, encoding="ascii")
# CVS accepts RFC822 dates. buildbot-cvs-mail adds the date as
# part of the mail header, so use that.
# This assumes cvs is being access via ssh or pserver, so the time
# will be the CVS server's time.
# calculate a "revision" based on that timestamp, or the current time
# if we're unable to parse the date.
log.msg('Processing CVS mail')
dateTuple = parsedate_tz(m["date"])
if dateTuple is None:
when = util.now()
else:
when = mktime_tz(dateTuple)
theTime = datetime.datetime.utcfromtimestamp(float(when))
rev = theTime.strftime('%Y-%m-%d %H:%M:%S')
catRE = re.compile(r'^Category:\s*(\S.*)')
cvsRE = re.compile(r'^CVSROOT:\s*(\S.*)')
cvsmodeRE = re.compile(r'^Cvsmode:\s*(\S.*)')
filesRE = re.compile(r'^Files:\s*(\S.*)')
modRE = re.compile(r'^Module:\s*(\S.*)')
pathRE = re.compile(r'^Path:\s*(\S.*)')
projRE = re.compile(r'^Project:\s*(\S.*)')
singleFileRE = re.compile(r'(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)')
tagRE = re.compile(r'^\s+Tag:\s*(\S.*)')
updateRE = re.compile(r'^Update of:\s*(\S.*)')
comments = ""
branch = None
cvsroot = None
fileList = None
files = []
isdir = 0
path = None
project = None
lines = list(body_line_iterator(m))
while lines:
line = lines.pop(0)
m = catRE.match(line)
if m:
category = m.group(1)
continue
m = cvsRE.match(line)
if m:
cvsroot = m.group(1)
continue
m = cvsmodeRE.match(line)
if m:
cvsmode = m.group(1)
continue
m = filesRE.match(line)
if m:
fileList = m.group(1)
continue
m = modRE.match(line)
if m:
# We don't actually use this
# module = m.group(1)
continue
m = pathRE.match(line)
if m:
path = m.group(1)
continue
m = projRE.match(line)
if m:
project = m.group(1)
continue
m = tagRE.match(line)
if m:
branch = m.group(1)
continue
m = updateRE.match(line)
if m:
# We don't actually use this
# updateof = m.group(1)
continue
if line == "Log Message:\n":
break
# CVS 1.11 lists files as:
# repo/path file,old-version,new-version file2,old-version,new-version
# Version 1.12 lists files as:
# file1 old-version new-version file2 old-version new-version
#
# files consists of tuples of 'file-name old-version new-version'
# The versions are either dotted-decimal version numbers, ie 1.1
# or NONE. New files are of the form 'NONE NUMBER', while removed
# files are 'NUMBER NONE'. 'NONE' is a literal string
# Parsing this instead of files list in 'Added File:' etc
# makes it possible to handle files with embedded spaces, though
# it could fail if the filename was 'bad 1.1 1.2'
# For cvs version 1.11, we expect
# my_module new_file.c,NONE,1.1
# my_module removed.txt,1.2,NONE
# my_module modified_file.c,1.1,1.2
# While cvs version 1.12 gives us
# new_file.c NONE 1.1
# removed.txt 1.2 NONE
# modified_file.c 1.1,1.2
if fileList is None:
log.msg('CVSMaildirSource Mail with no files. Ignoring')
return None # We don't have any files. Email not from CVS
if cvsmode == '1.11':
# Please, no repo paths with spaces!
m = re.search('([^ ]*) ', fileList)
if m:
path = m.group(1)
else:
log.msg(
'CVSMaildirSource can\'t get path from file list. Ignoring mail')
return
fileList = fileList[len(path):].strip()
singleFileRE = re.compile(
r'(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)')
elif cvsmode == '1.12':
singleFileRE = re.compile(
r'(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)')
if path is None:
raise ValueError(
'CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config')
else:
raise ValueError(
'Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode)
log.msg("CVSMaildirSource processing filelist: %s" % fileList)
while(fileList):
m = singleFileRE.match(fileList)
if m:
curFile = path + '/' + m.group(1)
files.append(curFile)
fileList = fileList[m.end():]
else:
log.msg('CVSMaildirSource no files matched regex. Ignoring')
return None # bail - we couldn't parse the files that changed
# Now get comments
while lines:
line = lines.pop(0)
comments += line
comments = comments.rstrip() + "\n"
if comments == '\n':
comments = None
return ('cvs', dict(author=author, files=files, comments=comments,
isdir=isdir, when=when, branch=branch,
revision=rev, category=category,
repository=cvsroot, project=project,
properties=self.properties))
def parse(self, m, prefix=None):
"""Parse branch notification messages sent by Launchpad.
"""
subject = m["subject"]
match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject)
if match:
repository = match.group(1)
else:
repository = None
# Put these into a dictionary, otherwise we cannot assign them
# from nested function definitions.
d = {'files': [], 'comments': ""}
gobbler = None
rev = None
author = None
when = util.now()
def gobble_comment(s):
d['comments'] += s + "\n"
def gobble_removed(s):
d['files'].append('%s REMOVED' % s)
def gobble_added(s):
d['files'].append('%s ADDED' % s)
def gobble_modified(s):
d['files'].append('%s MODIFIED' % s)
def gobble_renamed(s):
match = re.search(r"^(.+) => (.+)$", s)
if match:
d['files'].append('%s RENAMED %s' %
(match.group(1), match.group(2)))
else:
d['files'].append('%s RENAMED' % s)
lines = list(body_line_iterator(m, True))
rev = None
while lines:
line = text_type(lines.pop(0), "utf-8", errors="ignore")
# revno: 101
match = re.search(r"^revno: ([0-9.]+)", line)
if match:
rev = match.group(1)
# committer: Joe <*****@*****.**>
match = re.search(r"^committer: (.*)$", line)
if match:
author = match.group(1)
# timestamp: Fri 2009-05-15 10:35:43 +0200
# datetime.strptime() is supposed to support %z for time zone, but
# it does not seem to work. So handle the time zone manually.
match = re.search(
r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line)
if match:
datestr = match.group(1)
tz_sign = match.group(2)
tz_hours = match.group(3)
tz_minutes = match.group(4)
when = parseLaunchpadDate(
datestr, tz_sign, tz_hours, tz_minutes)
if re.search(r"^message:\s*$", line):
gobbler = gobble_comment
elif re.search(r"^removed:\s*$", line):
gobbler = gobble_removed
elif re.search(r"^added:\s*$", line):
gobbler = gobble_added
elif re.search(r"^renamed:\s*$", line):
gobbler = gobble_renamed
elif re.search(r"^modified:\s*$", line):
gobbler = gobble_modified
elif re.search(r"^ ", line) and gobbler:
gobbler(line[2:-1]) # Use :-1 to gobble trailing newline
# Determine the name of the branch.
branch = None
if self.branchMap and repository:
if repository in self.branchMap:
branch = self.branchMap[repository]
elif ("lp:" + repository) in self.branchMap:
branch = self.branchMap['lp:' + repository]
if not branch:
if self.defaultBranch:
branch = self.defaultBranch
else:
if repository:
branch = 'lp:' + repository
else:
branch = None
if rev and author:
return ('bzr', dict(author=author, files=d['files'],
comments=d['comments'],
when=when, revision=rev,
branch=branch, repository=repository or ''))
return None
def parse(self, m, prefix=None):
"""Parse messages sent by the svn 'commit-email.pl' trigger.
"""
# The mail is sent from the person doing the checkin. Assume that the
# local username is enough to identify them (this assumes a one-server
# cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
# model)
name, addr = parseaddr(m["from"])
if not addr:
return None # no From means this message isn't from svn
at = addr.find("@")
if at == -1:
author = addr # might still be useful
else:
author = addr[:at]
# we take the time of receipt as the time of checkin. Not correct (it
# depends upon the email latency), but it avoids the
# out-of-order-changes issue. Also syncmail doesn't give us anything
# better to work with, unless you count pulling the v1-vs-v2
# timestamp out of the diffs, which would be ugly. TODO: Pulling the
# 'Date:' header from the mail is a possibility, and
# email.utils.parsedate_tz may be useful. It should be configurable,
# however, because there are a lot of broken clocks out there.
when = util.now()
files = []
comments = ""
lines = list(body_line_iterator(m))
rev = None
while lines:
line = lines.pop(0)
# "Author: jmason"
match = re.search(r"^Author: (\S+)", line)
if match:
author = match.group(1)
# "New Revision: 105955"
match = re.search(r"^New Revision: (\d+)", line)
if match:
rev = match.group(1)
# possible TODO: use "Date: ..." data here instead of time of
# commit message receipt, above. however, this timestamp is
# specified *without* a timezone, in the server's local TZ, so to
# be accurate buildbot would need a config setting to specify the
# source server's expected TZ setting! messy.
# this stanza ends with the "Log:"
if (line == "Log:\n"):
break
# commit message is terminated by the file-listing section
while lines:
line = lines.pop(0)
if line in ("Modified:\n", "Added:\n", "Removed:\n"):
break
comments += line
comments = comments.rstrip() + "\n"
while lines:
line = lines.pop(0)
if line == "\n":
break
if line.find("Modified:\n") == 0:
continue # ignore this line
if line.find("Added:\n") == 0:
continue # ignore this line
if line.find("Removed:\n") == 0:
continue # ignore this line
line = line.strip()
thesefiles = line.split(" ")
for f in thesefiles:
if prefix:
# insist that the file start with the prefix: we may get
# changes we don't care about too
if f.startswith(prefix):
f = f[len(prefix):]
else:
log.msg("ignored file from svn commit: prefix '%s' "
"does not match filename '%s'" % (prefix, f))
continue
# TODO: figure out how new directories are described, set
# .isdir
files.append(f)
if not files:
log.msg("no matching files found, ignoring commit")
return None
return ('svn', dict(author=author, files=files, comments=comments,
when=when, revision=rev))
