def parse(self, m, prefix=None):
"""Parse branch notification messages sent by Launchpad.
"""
subject = m["subject"]
match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject)
if match:
repository = match.group(1)
else:
repository = None
# Put these into a dictionary, otherwise we cannot assign them
# from nested function definitions.
d = { 'files': [], 'comments': "" }
gobbler = None
rev = None
who = None
when = util.now()
def gobble_comment(s):
d['comments'] += s + "\n"
def gobble_removed(s):
d['files'].append('%s REMOVED' % s)
def gobble_added(s):
d['files'].append('%s ADDED' % s)
def gobble_modified(s):
d['files'].append('%s MODIFIED' % s)
def gobble_renamed(s):
match = re.search(r"^(.+) => (.+)$", s)
if match:
d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2)))
else:
d['files'].append('%s RENAMED' % s)
lines = list(body_line_iterator(m, True))
rev = None
while lines:
line = lines.pop(0)
# revno: 101
match = re.search(r"^revno: ([0-9.]+)", line)
if match:
rev = match.group(1)
# committer: Joe <*****@*****.**>
match = re.search(r"^committer: (.*)$", line)
if match:
who = match.group(1)
# timestamp: Fri 2009-05-15 10:35:43 +0200
# datetime.strptime() is supposed to support %z for time zone, but
# it does not seem to work. So handle the time zone manually.
match = re.search(r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line)
if match:
datestr = match.group(1)
tz_sign = match.group(2)
tz_hours = match.group(3)
tz_minutes = match.group(4)
when = parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes)
if re.search(r"^message:\s*$", line):
gobbler = gobble_comment
elif re.search(r"^removed:\s*$", line):
gobbler = gobble_removed
elif re.search(r"^added:\s*$", line):
gobbler = gobble_added
elif re.search(r"^renamed:\s*$", line):
gobbler = gobble_renamed
elif re.search(r"^modified:\s*$", line):
gobbler = gobble_modified
elif re.search(r"^ ", line) and gobbler:
gobbler(line[2:-1]) # Use :-1 to gobble trailing newline
# Determine the name of the branch.
branch = None
if self.branchMap and repository:
if self.branchMap.has_key(repository):
branch = self.branchMap[repository]
elif self.branchMap.has_key('lp:' + repository):
branch = self.branchMap['lp:' + repository]
if not branch:
if self.defaultBranch:
branch = self.defaultBranch
else:
if repository:
branch = 'lp:' + repository
else:
branch = None
#log.msg("parse(): rev=%s who=%s files=%s comments='%s' when=%s branch=%s" % (rev, who, d['files'], d['comments'], time.asctime(time.localtime(when)), branch))
if rev and who:
return changes.Change(who, d['files'], d['comments'],
when=when, revision=rev, branch=branch,
repository=repository or '')
else:
return None
def parse(self, m, prefix=None):
"""Parse mail sent by FreshCVS"""
# FreshCVS sets From: to "user CVS <user>", but the <> part may be
# modified by the MTA (to include a local domain)
name, addr = parseaddr(m["from"])
if not name:
return None # no From means this message isn't from FreshCVS
cvs = name.find(" CVS")
if cvs == -1:
return None # this message isn't from FreshCVS
who = name[:cvs]
# we take the time of receipt as the time of checkin. Not correct,
# but it avoids the out-of-order-changes issue. See the comment in
# parseSyncmail about using the 'Date:' header
when = util.now()
files = []
comments = ""
isdir = 0
lines = list(body_line_iterator(m))
while lines:
line = lines.pop(0)
if line == "Modified files:\n":
break
while lines:
line = lines.pop(0)
if line == "\n":
break
line = line.rstrip("\n")
linebits = line.split(None, 1)
file = linebits[0]
if prefix:
# insist that the file start with the prefix: FreshCVS sends
# changes we don't care about too
if file.startswith(prefix):
file = file[len(prefix):]
else:
continue
if len(linebits) == 1:
isdir = 1
elif linebits[1] == "0 0":
isdir = 1
files.append(file)
while lines:
line = lines.pop(0)
if line == "Log message:\n":
break
# message is terminated by "ViewCVS links:" or "Index:..." (patch)
while lines:
line = lines.pop(0)
if line == "ViewCVS links:\n":
break
if line.find("Index: ") == 0:
break
comments += line
comments = comments.rstrip() + "\n"
if not files:
return None
change = changes.Change(who, files, comments, isdir, when=when)
return change
def parse(self, m, prefix=None):
"""Parse mail sent by the Bonsai cvs loginfo script."""
# we don't care who the email came from b/c the cvs user is in the
# msg text
who = "unknown"
timestamp = None
files = []
lines = list(body_line_iterator(m))
# read the control lines (what/who/where/file/etc.)
while lines:
line = lines.pop(0)
if line == "LOGCOMMENT\n":
break;
line = line.rstrip("\n")
# we'd like to do the following but it won't work if the number of
# items doesn't match so...
# what, timestamp, user, repo, module, file = line.split( '|' )
items = line.split('|')
if len(items) < 6:
# not a valid line, assume this isn't a bonsai message
return None
try:
# just grab the bottom-most timestamp, they're probably all the
# same. TODO: I'm assuming this is relative to the epoch, but
# this needs testing.
timestamp = int(items[1])
except ValueError:
pass
user = items[2]
if user:
who = user
module = items[4]
file = items[5]
if module and file:
path = "%s/%s" % (module, file)
files.append(path)
sticky = items[7]
branch = items[8]
# if no files changed, return nothing
if not files:
return None
# read the comments
comments = ""
while lines:
line = lines.pop(0)
if line == ":ENDLOGCOMMENT\n":
break
comments += line
comments = comments.rstrip() + "\n"
# return buildbot Change object
return changes.Change(who, files, comments, when=timestamp,
branch=branch)
def parse(self, m, prefix=None):
"""Parse messages sent by the svn 'commit-email.pl' trigger.
"""
# The mail is sent from the person doing the checkin. Assume that the
# local username is enough to identify them (this assumes a one-server
# cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
# model)
name, addr = parseaddr(m["from"])
if not addr:
return None # no From means this message isn't from FreshCVS
at = addr.find("@")
if at == -1:
who = addr # might still be useful
else:
who = addr[:at]
# we take the time of receipt as the time of checkin. Not correct (it
# depends upon the email latency), but it avoids the
# out-of-order-changes issue. Also syncmail doesn't give us anything
# better to work with, unless you count pulling the v1-vs-v2
# timestamp out of the diffs, which would be ugly. TODO: Pulling the
# 'Date:' header from the mail is a possibility, and
# email.Utils.parsedate_tz may be useful. It should be configurable,
# however, because there are a lot of broken clocks out there.
when = util.now()
files = []
comments = ""
isdir = 0
lines = list(body_line_iterator(m))
rev = None
while lines:
line = lines.pop(0)
# "Author: jmason"
match = re.search(r"^Author: (\S+)", line)
if match:
who = match.group(1)
# "New Revision: 105955"
match = re.search(r"^New Revision: (\d+)", line)
if match:
rev = match.group(1)
# possible TODO: use "Date: ..." data here instead of time of
# commit message receipt, above. however, this timestamp is
# specified *without* a timezone, in the server's local TZ, so to
# be accurate buildbot would need a config setting to specify the
# source server's expected TZ setting! messy.
# this stanza ends with the "Log:"
if (line == "Log:\n"):
break
# commit message is terminated by the file-listing section
while lines:
line = lines.pop(0)
if (line == "Modified:\n" or
line == "Added:\n" or
line == "Removed:\n"):
break
comments += line
comments = comments.rstrip() + "\n"
while lines:
line = lines.pop(0)
if line == "\n":
break
if line.find("Modified:\n") == 0:
continue # ignore this line
if line.find("Added:\n") == 0:
continue # ignore this line
if line.find("Removed:\n") == 0:
continue # ignore this line
line = line.strip()
thesefiles = line.split(" ")
for f in thesefiles:
if prefix:
# insist that the file start with the prefix: we may get
# changes we don't care about too
if f.startswith(prefix):
f = f[len(prefix):]
else:
log.msg("ignored file from svn commit: prefix '%s' "
"does not match filename '%s'" % (prefix, f))
continue
# TODO: figure out how new directories are described, set
# .isdir
files.append(f)
if not files:
log.msg("no matching files found, ignoring commit")
return None
return changes.Change(who, files, comments, when=when, revision=rev)
def parse(self, m, prefix=None):
"""Parse messages sent by the 'buildbot-cvs-mail' program.
"""
# The mail is sent from the person doing the checkin. Assume that the
# local username is enough to identify them (this assumes a one-server
# cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
# model)
name, addr = parseaddr(m["from"])
if not addr:
return None # no From means this message isn't from buildbot-cvs-mail
at = addr.find("@")
if at == -1:
author = addr # might still be useful
else:
author = addr[:at]
# CVS accecpts RFC822 dates. buildbot-cvs-mail adds the date as
# part of the mail header, so use that.
# This assumes cvs is being access via ssh or pserver, so the time
# will be the CVS server's time.
# calculate a "revision" based on that timestamp, or the current time
# if we're unable to parse the date.
log.msg('Processing CVS mail')
dateTuple = parsedate_tz(m["date"])
if dateTuple == None:
when = util.now()
else:
when = mktime_tz(dateTuple)
theTime = datetime.datetime.utcfromtimestamp(float(when))
rev = theTime.strftime('%Y-%m-%d %H:%M:%S')
catRE = re.compile('^Category:\s*(\S.*)')
cvsRE = re.compile('^CVSROOT:\s*(\S.*)')
cvsmodeRE = re.compile('^Cvsmode:\s*(\S.*)')
filesRE = re.compile('^Files:\s*(\S.*)')
modRE = re.compile('^Module:\s*(\S.*)')
pathRE = re.compile('^Path:\s*(\S.*)')
projRE = re.compile('^Project:\s*(\S.*)')
singleFileRE = re.compile('(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)')
tagRE = re.compile('^\s+Tag:\s*(\S.*)')
updateRE = re.compile('^Update of:\s*(\S.*)')
comments = ""
branch = None
cvsroot = None
fileList = None
files = []
isdir = 0
path = None
project = None
lines = list(body_line_iterator(m))
while lines:
line = lines.pop(0)
m = catRE.match(line)
if m:
category = m.group(1)
continue
m = cvsRE.match(line)
if m:
cvsroot = m.group(1)
continue
m = cvsmodeRE.match(line)
if m:
cvsmode = m.group(1)
continue
m = filesRE.match(line)
if m:
fileList = m.group(1)
continue
m = modRE.match(line)
if m:
# We don't actually use this
#module = m.group(1)
continue
m = pathRE.match(line)
if m:
path = m.group(1)
continue
m = projRE.match(line)
if m:
project = m.group(1)
continue
m = tagRE.match(line)
if m:
branch = m.group(1)
continue
m = updateRE.match(line)
if m:
# We don't actually use this
#updateof = m.group(1)
continue
if line == "Log Message:\n":
break
# CVS 1.11 lists files as:
# repo/path file,old-version,new-version file2,old-version,new-version
# Version 1.12 lists files as:
# file1 old-version new-version file2 old-version new-version
#
# files consists of tuples of 'file-name old-version new-version'
# The versions are either dotted-decimal version numbers, ie 1.1
# or NONE. New files are of the form 'NONE NUMBER', while removed
# files are 'NUMBER NONE'. 'NONE' is a literal string
# Parsing this instead of files list in 'Added File:' etc
# makes it possible to handle files with embedded spaces, though
# it could fail if the filename was 'bad 1.1 1.2'
# For cvs version 1.11, we expect
# my_module new_file.c,NONE,1.1
# my_module removed.txt,1.2,NONE
# my_module modified_file.c,1.1,1.2
# While cvs version 1.12 gives us
# new_file.c NONE 1.1
# removed.txt 1.2 NONE
# modified_file.c 1.1,1.2
if fileList is None:
log.msg('CVSMaildirSource Mail with no files. Ignoring')
return None # We don't have any files. Email not from CVS
if cvsmode == '1.11':
# Please, no repo paths with spaces!
m = re.search('([^ ]*) ', fileList)
if m:
path = m.group(1)
else:
log.msg(
'CVSMaildirSource can\'t get path from file list. Ignoring mail'
)
return
fileList = fileList[len(path):].strip()
singleFileRE = re.compile(
'(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)'
)
elif cvsmode == '1.12':
singleFileRE = re.compile(
'(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)'
)
if path is None:
raise ValueError(
'CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config'
)
else:
raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' %
cvsmode)
log.msg("CVSMaildirSource processing filelist: %s" % fileList)
while (fileList):
m = singleFileRE.match(fileList)
if m:
curFile = path + '/' + m.group(1)
files.append(curFile)
fileList = fileList[m.end():]
else:
log.msg('CVSMaildirSource no files matched regex. Ignoring')
return None # bail - we couldn't parse the files that changed
# Now get comments
while lines:
line = lines.pop(0)
comments += line
comments = comments.rstrip() + "\n"
if comments == '\n':
comments = None
return ('cvs',
dict(author=author,
files=files,
comments=comments,
isdir=isdir,
when=when,
branch=branch,
revision=rev,
category=category,
repository=cvsroot,
project=project,
properties=self.properties))
def parse(self, m, prefix=None):
"""Parse messages sent by the 'syncmail' program, as suggested by the
sourceforge.net CVS Admin documentation. Syncmail is maintained at
syncmail.sf.net .
"""
# pretty much the same as freshcvs mail, not surprising since CVS is
# the one creating most of the text
# The mail is sent from the person doing the checkin. Assume that the
# local username is enough to identify them (this assumes a one-server
# cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
# model)
name, addr = parseaddr(m["from"])
if not addr:
return None # no From means this message isn't from FreshCVS
at = addr.find("@")
if at == -1:
who = addr # might still be useful
else:
who = addr[:at]
# we take the time of receipt as the time of checkin. Not correct (it
# depends upon the email latency), but it avoids the
# out-of-order-changes issue. Also syncmail doesn't give us anything
# better to work with, unless you count pulling the v1-vs-v2
# timestamp out of the diffs, which would be ugly. TODO: Pulling the
# 'Date:' header from the mail is a possibility, and
# email.Utils.parsedate_tz may be useful. It should be configurable,
# however, because there are a lot of broken clocks out there.
when = util.now()
# calculate a "revision" based on that timestamp
theCurrentTime = datetime.datetime.utcfromtimestamp(float(when))
rev = theCurrentTime.isoformat()
subject = m["subject"]
# syncmail puts the repository-relative directory in the subject:
# mprefix + "%(dir)s %(file)s,%(oldversion)s,%(newversion)s", where
# 'mprefix' is something that could be added by a mailing list
# manager.
# this is the only reasonable way to determine the directory name
space = subject.find(" ")
if space != -1:
directory = subject[:space]
else:
directory = subject
files = []
comments = ""
isdir = 0
branch = None
lines = list(body_line_iterator(m))
while lines:
line = lines.pop(0)
if (line == "Modified Files:\n" or
line == "Added Files:\n" or
line == "Removed Files:\n"):
break
while lines:
line = lines.pop(0)
if line == "\n":
break
if line == "Log Message:\n":
lines.insert(0, line)
break
line = line.lstrip()
line = line.rstrip()
# note: syncmail will send one email per directory involved in a
# commit, with multiple files if they were in the same directory.
# Unlike freshCVS, it makes no attempt to collect all related
# commits into a single message.
# note: syncmail will report a Tag underneath the ... Files: line
# e.g.: Tag: BRANCH-DEVEL
if line.startswith('Tag:'):
branch = line.split(' ')[-1].rstrip()
continue
thesefiles = line.split(" ")
for f in thesefiles:
f = directory + "/" + f
if prefix:
# insist that the file start with the prefix: we may get
# changes we don't care about too
if f.startswith(prefix):
f = f[len(prefix):]
else:
continue
break
# TODO: figure out how new directories are described, set
# .isdir
files.append(f)
if not files:
return None
while lines:
line = lines.pop(0)
if line == "Log Message:\n":
break
# message is terminated by "Index:..." (patch) or "--- NEW FILE.."
# or "--- filename DELETED ---". Sigh.
while lines:
line = lines.pop(0)
if line.find("Index: ") == 0:
break
if re.search(r"^--- NEW FILE", line):
break
if re.search(r" DELETED ---$", line):
break
comments += line
comments = comments.rstrip() + "\n"
change = changes.Change(who, files, comments, isdir, when=when,
branch=branch, revision=rev)
return change
def parse(self, m, prefix=None):
"""Parse messages sent by the svn 'commit-email.pl' trigger.
"""
# The mail is sent from the person doing the checkin. Assume that the
# local username is enough to identify them (this assumes a one-server
# cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
# model)
name, addr = parseaddr(m["from"])
if not addr:
return None # no From means this message isn't from svn
at = addr.find("@")
if at == -1:
author = addr # might still be useful
else:
author = addr[:at]
# we take the time of receipt as the time of checkin. Not correct (it
# depends upon the email latency), but it avoids the
# out-of-order-changes issue. Also syncmail doesn't give us anything
# better to work with, unless you count pulling the v1-vs-v2
# timestamp out of the diffs, which would be ugly. TODO: Pulling the
# 'Date:' header from the mail is a possibility, and
# email.Utils.parsedate_tz may be useful. It should be configurable,
# however, because there are a lot of broken clocks out there.
when = util.now()
files = []
comments = ""
lines = list(body_line_iterator(m))
rev = None
while lines:
line = lines.pop(0)
# "Author: jmason"
match = re.search(r"^Author: (\S+)", line)
if match:
author = match.group(1)
# "New Revision: 105955"
match = re.search(r"^New Revision: (\d+)", line)
if match:
rev = match.group(1)
# possible TODO: use "Date: ..." data here instead of time of
# commit message receipt, above. however, this timestamp is
# specified *without* a timezone, in the server's local TZ, so to
# be accurate buildbot would need a config setting to specify the
# source server's expected TZ setting! messy.
# this stanza ends with the "Log:"
if (line == "Log:\n"):
break
# commit message is terminated by the file-listing section
while lines:
line = lines.pop(0)
if (line == "Modified:\n" or line == "Added:\n"
or line == "Removed:\n"):
break
comments += line
comments = comments.rstrip() + "\n"
while lines:
line = lines.pop(0)
if line == "\n":
break
if line.find("Modified:\n") == 0:
continue # ignore this line
if line.find("Added:\n") == 0:
continue # ignore this line
if line.find("Removed:\n") == 0:
continue # ignore this line
line = line.strip()
thesefiles = line.split(" ")
for f in thesefiles:
if prefix:
# insist that the file start with the prefix: we may get
# changes we don't care about too
if f.startswith(prefix):
f = f[len(prefix):]
else:
log.msg("ignored file from svn commit: prefix '%s' "
"does not match filename '%s'" % (prefix, f))
continue
# TODO: figure out how new directories are described, set
# .isdir
files.append(f)
if not files:
log.msg("no matching files found, ignoring commit")
return None
return ('svn',
dict(author=author,
files=files,
comments=comments,
when=when,
revision=rev))
def parse(self, m, prefix=None):
"""Parse branch notification messages sent by Launchpad.
"""
subject = m["subject"]
match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject)
if match:
repository = match.group(1)
else:
repository = None
# Put these into a dictionary, otherwise we cannot assign them
# from nested function definitions.
d = {'files': [], 'comments': u""}
gobbler = None
rev = None
author = None
when = util.now()
def gobble_comment(s):
d['comments'] += s + "\n"
def gobble_removed(s):
d['files'].append('%s REMOVED' % s)
def gobble_added(s):
d['files'].append('%s ADDED' % s)
def gobble_modified(s):
d['files'].append('%s MODIFIED' % s)
def gobble_renamed(s):
match = re.search(r"^(.+) => (.+)$", s)
if match:
d['files'].append('%s RENAMED %s' %
(match.group(1), match.group(2)))
else:
d['files'].append('%s RENAMED' % s)
lines = list(body_line_iterator(m, True))
rev = None
while lines:
line = unicode(lines.pop(0), "utf-8", errors="ignore")
# revno: 101
match = re.search(r"^revno: ([0-9.]+)", line)
if match:
rev = match.group(1)
# committer: Joe <*****@*****.**>
match = re.search(r"^committer: (.*)$", line)
if match:
author = match.group(1)
# timestamp: Fri 2009-05-15 10:35:43 +0200
# datetime.strptime() is supposed to support %z for time zone, but
# it does not seem to work. So handle the time zone manually.
match = re.search(
r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$",
line)
if match:
datestr = match.group(1)
tz_sign = match.group(2)
tz_hours = match.group(3)
tz_minutes = match.group(4)
when = parseLaunchpadDate(datestr, tz_sign, tz_hours,
tz_minutes)
if re.search(r"^message:\s*$", line):
gobbler = gobble_comment
elif re.search(r"^removed:\s*$", line):
gobbler = gobble_removed
elif re.search(r"^added:\s*$", line):
gobbler = gobble_added
elif re.search(r"^renamed:\s*$", line):
gobbler = gobble_renamed
elif re.search(r"^modified:\s*$", line):
gobbler = gobble_modified
elif re.search(r"^ ", line) and gobbler:
gobbler(line[2:-1]) # Use :-1 to gobble trailing newline
# Determine the name of the branch.
branch = None
if self.branchMap and repository:
if self.branchMap.has_key(repository):
branch = self.branchMap[repository]
elif self.branchMap.has_key('lp:' + repository):
branch = self.branchMap['lp:' + repository]
if not branch:
if self.defaultBranch:
branch = self.defaultBranch
else:
if repository:
branch = 'lp:' + repository
else:
branch = None
if rev and author:
return ('bzr',
dict(author=author,
files=d['files'],
comments=d['comments'],
when=when,
revision=rev,
branch=branch,
repository=repository or ''))
else:
return None
def parse(self, m, prefix=None):
"""Parse mail sent by FreshCVS"""
# FreshCVS sets From: to "user CVS <user>", but the <> part may be
# modified by the MTA (to include a local domain)
name, addr = parseaddr(m["from"])
if not name:
return None # no From means this message isn't from FreshCVS
cvs = name.find(" CVS")
if cvs == -1:
return None # this message isn't from FreshCVS
who = name[:cvs]
# we take the time of receipt as the time of checkin. Not correct,
# but it avoids the out-of-order-changes issue. See the comment in
# parseSyncmail about using the 'Date:' header
when = util.now()
files = []
comments = ""
isdir = 0
lines = list(body_line_iterator(m))
while lines:
line = lines.pop(0)
if line == "Modified files:\n":
break
while lines:
line = lines.pop(0)
if line == "\n":
break
line = line.rstrip("\n")
linebits = line.split(None, 1)
file = linebits[0]
if prefix:
# insist that the file start with the prefix: FreshCVS sends
# changes we don't care about too
if file.startswith(prefix):
file = file[len(prefix):]
else:
continue
if len(linebits) == 1:
isdir = 1
elif linebits[1] == "0 0":
isdir = 1
files.append(file)
while lines:
line = lines.pop(0)
if line == "Log message:\n":
break
# message is terminated by "ViewCVS links:" or "Index:..." (patch)
while lines:
line = lines.pop(0)
if line == "ViewCVS links:\n":
break
if line.find("Index: ") == 0:
break
comments += line
comments = comments.rstrip() + "\n"
if not files:
return None
change = changes.Change(who, files, comments, isdir, when=when)
return change
def parse(self, m, prefix=None):
"""Parse messages sent by the 'buildbot-cvs-mail' program.
"""
# The mail is sent from the person doing the checkin. Assume that the
# local username is enough to identify them (this assumes a one-server
# cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
# model)
name, addr = parseaddr(m["from"])
if not addr:
return None # no From means this message isn't from buildbot-cvs-mail
at = addr.find("@")
if at == -1:
author = addr # might still be useful
else:
author = addr[:at]
# CVS accepts RFC822 dates. buildbot-cvs-mail adds the date as
# part of the mail header, so use that.
# This assumes cvs is being access via ssh or pserver, so the time
# will be the CVS server's time.
# calculate a "revision" based on that timestamp, or the current time
# if we're unable to parse the date.
log.msg('Processing CVS mail')
dateTuple = parsedate_tz(m["date"])
if dateTuple == None:
when = util.now()
else:
when = mktime_tz(dateTuple)
theTime = datetime.datetime.utcfromtimestamp(float(when))
rev = theTime.strftime('%Y-%m-%d %H:%M:%S')
catRE = re.compile( '^Category:\s*(\S.*)')
cvsRE = re.compile( '^CVSROOT:\s*(\S.*)')
cvsmodeRE = re.compile( '^Cvsmode:\s*(\S.*)')
filesRE = re.compile( '^Files:\s*(\S.*)')
modRE = re.compile( '^Module:\s*(\S.*)')
pathRE = re.compile( '^Path:\s*(\S.*)')
projRE = re.compile( '^Project:\s*(\S.*)')
singleFileRE = re.compile( '(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)')
tagRE = re.compile( '^\s+Tag:\s*(\S.*)')
updateRE = re.compile( '^Update of:\s*(\S.*)')
comments = ""
branch = None
cvsroot = None
fileList = None
files = []
isdir = 0
path = None
project = None
lines = list(body_line_iterator(m))
while lines:
line = lines.pop(0)
m = catRE.match(line)
if m:
category = m.group(1)
continue
m = cvsRE.match(line)
if m:
cvsroot = m.group(1)
continue
m = cvsmodeRE.match(line)
if m:
cvsmode = m.group(1)
continue
m = filesRE.match(line)
if m:
fileList = m.group(1)
continue
m = modRE.match(line)
if m:
# We don't actually use this
#module = m.group(1)
continue
m = pathRE.match(line)
if m:
path = m.group(1)
continue
m = projRE.match(line)
if m:
project = m.group(1)
continue
m = tagRE.match(line)
if m:
branch = m.group(1)
continue
m = updateRE.match(line)
if m:
# We don't actually use this
#updateof = m.group(1)
continue
if line == "Log Message:\n":
break
# CVS 1.11 lists files as:
# repo/path file,old-version,new-version file2,old-version,new-version
# Version 1.12 lists files as:
# file1 old-version new-version file2 old-version new-version
#
# files consists of tuples of 'file-name old-version new-version'
# The versions are either dotted-decimal version numbers, ie 1.1
# or NONE. New files are of the form 'NONE NUMBER', while removed
# files are 'NUMBER NONE'. 'NONE' is a literal string
# Parsing this instead of files list in 'Added File:' etc
# makes it possible to handle files with embedded spaces, though
# it could fail if the filename was 'bad 1.1 1.2'
# For cvs version 1.11, we expect
# my_module new_file.c,NONE,1.1
# my_module removed.txt,1.2,NONE
# my_module modified_file.c,1.1,1.2
# While cvs version 1.12 gives us
# new_file.c NONE 1.1
# removed.txt 1.2 NONE
# modified_file.c 1.1,1.2
if fileList is None:
log.msg('CVSMaildirSource Mail with no files. Ignoring')
return None # We don't have any files. Email not from CVS
if cvsmode == '1.11':
# Please, no repo paths with spaces!
m = re.search('([^ ]*) ', fileList)
if m:
path = m.group(1)
else:
log.msg('CVSMaildirSource can\'t get path from file list. Ignoring mail')
return
fileList = fileList[len(path):].strip()
singleFileRE = re.compile( '(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)')
elif cvsmode == '1.12':
singleFileRE = re.compile( '(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)')
if path is None:
raise ValueError('CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config')
else:
raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode)
log.msg("CVSMaildirSource processing filelist: %s" % fileList)
while(fileList):
m = singleFileRE.match(fileList)
if m:
curFile = path + '/' + m.group(1)
files.append( curFile )
fileList = fileList[m.end():]
else:
log.msg('CVSMaildirSource no files matched regex. Ignoring')
return None # bail - we couldn't parse the files that changed
# Now get comments
while lines:
line = lines.pop(0)
comments += line
comments = comments.rstrip() + "\n"
if comments == '\n':
comments = None
return ('cvs', dict(author=author, files=files, comments=comments,
isdir=isdir, when=when, branch=branch,
revision=rev, category=category,
repository=cvsroot, project=project,
properties=self.properties))
def parse(self, m, prefix=None):
"""Parse mail sent by the Bonsai cvs loginfo script."""
# we don't care who the email came from b/c the cvs user is in the
# msg text
who = "unknown"
timestamp = None
files = []
lines = list(body_line_iterator(m))
# read the control lines (what/who/where/file/etc.)
while lines:
line = lines.pop(0)
if line == "LOGCOMMENT\n":
break;
line = line.rstrip("\n")
# we'd like to do the following but it won't work if the number of
# items doesn't match so...
# what, timestamp, user, repo, module, file = line.split( '|' )
items = line.split('|')
if len(items) < 6:
# not a valid line, assume this isn't a bonsai message
return None
try:
# just grab the bottom-most timestamp, they're probably all the
# same. TODO: I'm assuming this is relative to the epoch, but
# this needs testing.
timestamp = int(items[1])
except ValueError:
pass
user = items[2]
if user:
who = user
module = items[4]
file = items[5]
if module and file:
path = "%s/%s" % (module, file)
files.append(path)
sticky = items[7]
branch = items[8]
# if no files changed, return nothing
if not files:
return None
# read the comments
comments = ""
while lines:
line = lines.pop(0)
if line == ":ENDLOGCOMMENT\n":
break
comments += line
comments = comments.rstrip() + "\n"
# return buildbot Change object
return changes.Change(who, files, comments, when=timestamp,
branch=branch)
def parse(self, m, prefix=None):
"""Parse messages sent by the 'syncmail' program, as suggested by the
sourceforge.net CVS Admin documentation. Syncmail is maintained at
syncmail.sf.net .
"""
# pretty much the same as freshcvs mail, not surprising since CVS is
# the one creating most of the text
# The mail is sent from the person doing the checkin. Assume that the
# local username is enough to identify them (this assumes a one-server
# cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
# model)
name, addr = parseaddr(m["from"])
if not addr:
return None # no From means this message isn't from FreshCVS
at = addr.find("@")
if at == -1:
who = addr # might still be useful
else:
who = addr[:at]
# we take the time of receipt as the time of checkin. Not correct (it
# depends upon the email latency), but it avoids the
# out-of-order-changes issue. Also syncmail doesn't give us anything
# better to work with, unless you count pulling the v1-vs-v2
# timestamp out of the diffs, which would be ugly. TODO: Pulling the
# 'Date:' header from the mail is a possibility, and
# email.Utils.parsedate_tz may be useful. It should be configurable,
# however, because there are a lot of broken clocks out there.
when = util.now()
subject = m["subject"]
# syncmail puts the repository-relative directory in the subject:
# mprefix + "%(dir)s %(file)s,%(oldversion)s,%(newversion)s", where
# 'mprefix' is something that could be added by a mailing list
# manager.
# this is the only reasonable way to determine the directory name
space = subject.find(" ")
if space != -1:
directory = subject[:space]
else:
directory = subject
files = []
comments = ""
isdir = 0
branch = None
lines = list(body_line_iterator(m))
while lines:
line = lines.pop(0)
if (line == "Modified Files:\n" or
line == "Added Files:\n" or
line == "Removed Files:\n"):
break
while lines:
line = lines.pop(0)
if line == "\n":
break
if line == "Log Message:\n":
lines.insert(0, line)
break
line = line.lstrip()
line = line.rstrip()
# note: syncmail will send one email per directory involved in a
# commit, with multiple files if they were in the same directory.
# Unlike freshCVS, it makes no attempt to collect all related
# commits into a single message.
# note: syncmail will report a Tag underneath the ... Files: line
# e.g.: Tag: BRANCH-DEVEL
if line.startswith('Tag:'):
branch = line.split(' ')[-1].rstrip()
continue
thesefiles = line.split(" ")
for f in thesefiles:
f = directory + "/" + f
if prefix:
# insist that the file start with the prefix: we may get
# changes we don't care about too
if f.startswith(prefix):
f = f[len(prefix):]
else:
continue
break
# TODO: figure out how new directories are described, set
# .isdir
files.append(f)
if not files:
return None
while lines:
line = lines.pop(0)
if line == "Log Message:\n":
break
# message is terminated by "Index:..." (patch) or "--- NEW FILE.."
# or "--- filename DELETED ---". Sigh.
while lines:
line = lines.pop(0)
if line.find("Index: ") == 0:
break
if re.search(r"^--- NEW FILE", line):
break
if re.search(r" DELETED ---$", line):
break
comments += line
comments = comments.rstrip() + "\n"
change = changes.Change(who, files, comments, isdir, when=when,
branch=branch)
return change
def parse(self, m, prefix=None):
if m is None:
# not a mail at all
return None
from_header = m['from']
if '<' in from_header:
from_email = m['from'].split('<')[1][:-1]
else:
from_email = m['from']
# From is [email protected]
name, domain = from_email.split("@")
# If this e-mail is valid, it will come from an svn/src.gnome.org email
if domain != 'src.gnome.org':
return None
# we take the time of receipt as the time of checkin. Not correct, but it
# avoids the out-of-order-changes issue. See the comment in parseSyncmail
# about using the 'Date:' header
when = util.now()
revision = None
files = []
comments = ""
isdir = 0
links = []
subject = m['subject']
if not subject.startswith('['):
# not a git message, abort
return None
# git message
revision = m.get('X-Git-Newrev')
if not revision:
# not a new git revision, may be a new tag, a new branch, etc.
return None
if revision == '0000000000000000000000000000000000000000':
# probably a deleted branch, ignore
return None
if m.get('X-Git-Refname', '').startswith('refs/tags/'):
# ignore tags
return None
try:
project = subject[1:subject.index(']')]
except ValueError:
return None # old git commit message format; ignored
if '/' in project:
# remove the branch part (ex: [anjal/inline-composer-quotes])
project = project.split('/')[0]
if ':' in project:
# remove the patch number part (ex: [anjal: 3/3])
project = project.split(':')[0]
if 'Created branch' in subject:
# new branches don't have to trigger rebuilds
return None
if 'Merge branch' in subject:
comments = subject[subject.index('Merge branch'):]
elif 'Merge commit' in subject:
comments = subject[subject.index('Merge commit'):]
else:
lines = list(body_line_iterator(m, m['Content-Transfer-Encoding']))
after_date = False
in_files = False
while lines:
line = lines.pop(0)
if line.startswith('Date:'):
after_date = True
continue
if not after_date:
continue
if len(line) > 3 and line[0] == ' ' and line[1] != ' ' and '|' in line:
in_files = True
if line.startswith('---'):
break
if in_files:
if not '|' in line:
break
files.append(line.split()[0])
else:
comments += line[4:] + '\n'
comments = unicode(comments.strip(), m.get_content_charset() or 'ascii', 'ignore')
c = changes.Change(name, files, comments, isdir, revision=revision, links=links, when=when)
c.project = project
c.git_module_name = project
# some modules may have alternate checkouts under different names, look
# for those, and create appropriate Change objects
for module in self.modules:
if hasattr(module, 'branch') and isinstance(module.branch, GitBranch):
git_module_name = module.branch.module.rsplit('/', 1)[-1]
if module.name != project and git_module_name == project:
change = changes.Change(name, files, comments, isdir,
revision=revision, links=links, when=when)
change.project = module.name
change.git_module_name = git_module_name
self.parent.addChange(change)
return c
def parse(self, m, prefix=None):
if m is None:
# not a mail at all
return None
from_header = m['from']
if '<' in from_header:
from_email = m['from'].split('<')[1][:-1]
else:
from_email = m['from']
# From is [email protected]
name, domain = from_email.split("@")
# If this e-mail is valid, it will come from an svn/src.gnome.org email
if domain != 'src.gnome.org':
return None
# we take the time of receipt as the time of checkin. Not correct, but it
# avoids the out-of-order-changes issue. See the comment in parseSyncmail
# about using the 'Date:' header
when = util.now()
revision = None
files = []
comments = ""
isdir = 0
links = []
subject = m['subject']
if not subject.startswith('['):
# not a git message, abort
return None
# git message
revision = m.get('X-Git-Newrev')
if not revision:
# not a new git revision, may be a new tag, a new branch, etc.
return None
if revision == '0000000000000000000000000000000000000000':
# probably a deleted branch, ignore
return None
if m.get('X-Git-Refname', '').startswith('refs/tags/'):
# ignore tags
return None
try:
project = subject[1:subject.index(']')]
except ValueError:
return None # old git commit message format; ignored
if '/' in project:
# remove the branch part (ex: [anjal/inline-composer-quotes])
project = project.split('/')[0]
if ':' in project:
# remove the patch number part (ex: [anjal: 3/3])
project = project.split(':')[0]
if 'Created branch' in subject:
# new branches don't have to trigger rebuilds
return None
if 'Merge branch' in subject:
comments = subject[subject.index('Merge branch'):]
elif 'Merge commit' in subject:
comments = subject[subject.index('Merge commit'):]
else:
lines = list(body_line_iterator(m, m['Content-Transfer-Encoding']))
after_date = False
in_files = False
while lines:
line = lines.pop(0)
if line.startswith('Date:'):
after_date = True
continue
if not after_date:
continue
if len(line) > 3 and line[
0] == ' ' and line[1] != ' ' and '|' in line:
in_files = True
if line.startswith('---'):
break
if in_files:
if not '|' in line:
break
files.append(line.split()[0])
else:
comments += line[4:] + '\n'
comments = unicode(comments.strip(),
m.get_content_charset() or 'ascii', 'ignore')
c = changes.Change(name,
files,
comments,
isdir,
revision=revision,
links=links,
when=when)
c.project = project
c.git_module_name = project
# some modules may have alternate checkouts under different names, look
# for those, and create appropriate Change objects
for module in self.modules:
if hasattr(module, 'branch') and isinstance(
module.branch, GitBranch):
git_module_name = module.branch.module.rsplit('/', 1)[-1]
if module.name != project and git_module_name == project:
change = changes.Change(name,
files,
comments,
isdir,
revision=revision,
links=links,
when=when)
change.project = module.name
change.git_module_name = git_module_name
self.parent.addChange(change)
return c
def parse(self, m, prefix):
"""parse email sent by cradek's EMC CVS Script 1.0"""
#log.msg("got an email!")
#log.msg(m)
# the From: should look like this: EMC CVS server <*****@*****.**>
name, addr = parseaddr(m["from"]);
expected_name = "EMC CVS server"
expected_addr = "*****@*****.**"
if name != expected_name:
log.msg("email is from \"%s\", expected \"%s\"" % (name, expected_name))
return None
if addr != expected_addr:
log.msg("email is from \"%s\", expected \"%s\"" % (addr, expected_addr))
return None
# we take the time of receipt as the time of checkin. Not correct,
# but it avoids the out-of-order-changes issue. See the comment in
# parseSyncmail about using the 'Date:' header
when = util.now()
#
# the commit notification email looks like this:
#
# <message>
# <generator>
# <name>EMC CIA script</name>
# <version>1.0</version>
# </generator>
# <source>
# <project>EMC</project>
# <module>emc2/docs/src/gui</module>
# <branch>TRUNK</branch>
# </source>
# <body>
# <commit>
# <files><file>axis.lyx</file></files>
# <author>bigjohnt</author>
# <log>minor edit
# </log>
# </commit>
# </body>
# </message>
files = []
module = ""
branch = ""
comments = ""
author = ""
isdir = 0
lines = list(body_line_iterator(m))
while lines:
line = lines.pop(0)
#log.msg("thinking about line \"%s\"" % line)
if re.search(r"\<module\>emc2", line):
module = "emc2"
continue
match = re.search(r"\<branch\>([^<]+)", line)
if match:
branch = match.group(1)
continue
match = re.search(r"\<author\>([^\<]+)\<", line)
if match:
author = match.group(1)
continue
match = re.search(r"\<files\>(.+)\<\/files\>", line)
if match:
f = match.group(1)
# f begins and ends with the separator, so the split result begins and ends with an empty string
new_files = re.split(r"(?:</?file>)+", f)
while new_files:
new_file = new_files.pop(0)
if new_file:
files.append(new_file)
continue
match = re.search(r"\<log\>(.*)", line)
if match:
comments += match.group(1)
while lines:
line = lines.pop(0)
if re.search(r"\<\/log\>", line):
break
comments += line
continue
if module != "emc2":
log.msg("email was not for the emc2 module, ignoring");
return None
if not branch:
log.msg("email contained no branch");
return None
if not files:
log.msg("email contained no files");
return None
if not author:
log.msg("email contained no author");
return None
log.msg("accepting change from email:");
log.msg(" branch = %s" % branch);
log.msg(" author = %s" % author);
log.msg(" files = %s" % files);
log.msg(" comments = %s" % comments);
# the "branch" value gets used by Buildbot as a CVS Tag, and for TRUNK the proper Tag is None
if branch == "TRUNK":
branch = None
return changes.Change(author, files, comments, when=when, branch=branch)