def copy(_):
cmd = buildstep.RemoteCommand(
'cpdir', {
'fromdir': self.srcdir,
'todir': old_workdir,
'logEnviron': self.logEnviron,
'timeout': self.timeout,
})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
return d
def _clobber(self):
cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir,
'logEnviron': self.logEnviron})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
def checkRemoval(res):
if res != 0:
raise RuntimeError("Failed to delete directory")
return res
d.addCallback(lambda _: checkRemoval(cmd.rc))
return d
def clobber(self, _):
cmd = buildstep.RemoteCommand('rmdir', {
'dir': self.workdir,
'logEnviron': self.logEnviron
})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
d.addCallback(lambda _: self._dovccmd(
['clone', '--noupdate', self.repourl, "."]))
d.addCallback(self._update)
return d
def _sourcedirIsUpdatable(self):
cmd = buildstep.RemoteCommand('stat', {'file': self.workdir + '/CVS',
'logEnviron': self.logEnviron})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
def _fail(tmp):
if cmd.rc != 0:
return False
return True
d.addCallback(_fail)
return d
def patch(self, _, patch):
diff = patch[1]
root = None
if len(patch) >= 3:
root = patch[2]
if (root and os.path.abspath(
os.path.join(self.workdir, root)).startswith(
os.path.abspath(self.workdir))):
self.workdir = os.path.join(self.workdir, root)
def _downloadFile(buf, filename):
filereader = _FileReader(StringIO.StringIO(buf))
args = {
'slavedest': filename,
'maxsize': None,
'reader': filereader,
'blocksize': 16 * 1024,
'workdir': self.workdir,
'mode': None
}
cmd = buildstep.RemoteCommand('downloadFile', args)
cmd.useLog(self.stdio_log, False)
log.msg("Downloading file: %s" % (filename))
d = self.runCommand(cmd)
def evaluateCommand(_):
if cmd.didFail():
raise buildstep.BuildStepFailed()
return cmd.rc
d.addCallback(evaluateCommand)
return d
d = _downloadFile(diff, ".buildbot-diff")
d.addCallback(
lambda _: _downloadFile("patched\n", ".buildbot-patched"))
d.addCallback(lambda _: self.applyPatch(patch))
cmd = buildstep.RemoteCommand(
'rmdir', {
'dir': os.path.join(self.workdir, ".buildbot-diff"),
'logEnviron': self.logEnviron
})
cmd.useLog(self.stdio_log, False)
d.addCallback(lambda _: self.runCommand(cmd))
def evaluateCommand(cmd):
if cmd.didFail():
raise buildstep.BuildStepFailed()
return cmd.rc
d.addCallback(lambda _: evaluateCommand(cmd))
return d
def removeFiles(self, files):
for filename in files:
cmd = buildstep.RemoteCommand('rmdir', {'dir': filename,
'logEnviron': self.logEnviron,})
cmd.useLog(self.stdio_log, False)
wfd = defer.waitForDeferred(self.runCommand(cmd))
yield wfd
wfd.getResult()
if cmd.rc != 0:
yield cmd.rc
return
yield 0
def removeFiles(self, files):
for filename in files:
cmd = buildstep.RemoteCommand('rmdir', {
'dir': filename,
'logEnviron': self.logEnviron,
})
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if cmd.rc != 0:
defer.returnValue(cmd.rc)
return
defer.returnValue(0)
def _sourcedirIsUpdatable(self):
myFileWriter = StringFileWriter()
args = {
'workdir': self.build.path_module.join(self.workdir, 'CVS'),
'writer': myFileWriter,
'maxsize': None,
'blocksize': 32 * 1024,
}
cmd = remotecommand.RemoteCommand('uploadFile',
dict(slavesrc='Root', **args),
ignore_updates=True)
yield self.runCommand(cmd)
if cmd.rc is not None and cmd.rc != 0:
defer.returnValue(False)
return
# on Windows, the cvsroot may not contain the password, so compare to
# both
cvsroot_without_pw = re.sub("(:pserver:[^:]*):[^@]*(@.*)",
r"\1\2", self.cvsroot)
if myFileWriter.buffer.strip() not in (self.cvsroot,
cvsroot_without_pw):
defer.returnValue(False)
return
myFileWriter.buffer = ""
cmd = remotecommand.RemoteCommand('uploadFile',
dict(slavesrc='Repository', **args),
ignore_updates=True)
yield self.runCommand(cmd)
if cmd.rc is not None and cmd.rc != 0:
defer.returnValue(False)
return
if myFileWriter.buffer.strip() != self.cvsmodule:
defer.returnValue(False)
return
# if there are sticky dates (from an earlier build with revision),
# we can't update (unless we remove those tags with cvs update -A)
myFileWriter.buffer = ""
cmd = buildstep.RemoteCommand('uploadFile',
dict(slavesrc='Entries', **args),
ignore_updates=True)
yield self.runCommand(cmd)
if cmd.rc is not None and cmd.rc != 0:
defer.returnValue(False)
return
if self._cvsEntriesContainStickyDates(myFileWriter.buffer):
defer.returnValue(False)
defer.returnValue(True)
def clobber(self):
cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir,
'logEnviron': self.logEnviron,})
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if cmd.rc != 0:
raise buildstep.BuildStepFailed()
checkout_cmd = ['checkout', self.repourl, '.']
if self.revision:
checkout_cmd.extend(["--revision", str(self.revision)])
yield self._dovccmd(checkout_cmd)
def _sourcedirIsUpdatable(self):
myFileWriter = StringFileWriter()
args = {
'workdir': self.build.path_module.join(self.workdir, 'CVS'),
'writer': myFileWriter,
'maxsize': None,
'blocksize': 32 * 1024,
}
cmd = buildstep.RemoteCommand('uploadFile',
dict(slavesrc='Root', **args),
ignore_updates=True)
yield self.runCommand(cmd)
if cmd.rc is not None and cmd.rc != 0:
defer.returnValue(False)
return
# on Windows, the cvsroot may not contain the password, so compare to
# both
cvsroot_without_pw = re.sub("(:pserver:[^:]*):[^@]*(@.*)", r"\1\2",
self.cvsroot)
if myFileWriter.buffer.strip() not in (self.cvsroot,
cvsroot_without_pw):
defer.returnValue(False)
return
myFileWriter.buffer = ""
cmd = buildstep.RemoteCommand('uploadFile',
dict(slavesrc='Repository', **args),
ignore_updates=True)
yield self.runCommand(cmd)
if cmd.rc is not None and cmd.rc != 0:
defer.returnValue(False)
return
if myFileWriter.buffer.strip() != self.cvsmodule:
defer.returnValue(False)
return
defer.returnValue(True)
def runRemoteCommand(self, cmd, args, abandonOnFailure=True):
"""generic RemoteCommand boilerplate"""
cmd = buildstep.RemoteCommand(cmd, args)
cmd.useLog(self.rc_log, False)
d = self.runCommand(cmd)
def commandComplete(cmd):
if abandonOnFailure and cmd.didFail():
raise buildstep.BuildStepFailed()
return cmd.didFail()
d.addCallback(lambda res: commandComplete(cmd))
return d
def copy(self):
cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir,
'logEnviron': self.logEnviron,})
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if cmd.rc != 0:
raise buildstep.BuildStepFailed()
# temporarily set workdir = 'source' and do an incremental checkout
try:
old_workdir = self.workdir
self.workdir = 'source'
yield self.incremental(None)
except: # finally doesn't work in python-2.4
self.workdir = old_workdir
raise
self.workdir = old_workdir
# if we're copying, copy; otherwise, export from source to build
if self.method == 'copy':
cmd = buildstep.RemoteCommand('cpdir',
{ 'fromdir': 'source', 'todir':self.workdir,
'logEnviron': self.logEnviron })
else:
export_cmd = ['svn', 'export']
if self.revision:
export_cmd.extend(["--revision", str(self.revision)])
export_cmd.extend(['source', self.workdir])
cmd = buildstep.RemoteShellCommand('', export_cmd,
env=self.env, logEnviron=self.logEnviron)
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if cmd.rc != 0:
raise buildstep.BuildStepFailed()
def testPyFlakes1(self):
self.masterbase = "Python.testPyFlakes1"
step = self.makeStep(python.PyFlakes)
output = \
"""pyflakes buildbot
buildbot/changes/freshcvsmail.py:5: 'FCMaildirSource' imported but unused
buildbot/clients/debug.py:9: redefinition of unused 'gtk' from line 9
buildbot/clients/debug.py:9: 'gnome' imported but unused
buildbot/scripts/runner.py:323: redefinition of unused 'run' from line 321
buildbot/scripts/runner.py:325: redefinition of unused 'run' from line 323
buildbot/scripts/imaginary.py:12: undefined name 'size'
buildbot/scripts/imaginary.py:18: 'from buildbot import *' used; unable to detect undefined names
"""
log = step.addLog("stdio")
log.addStdout(output)
log.finish()
step.createSummary(log)
desc = step.descriptionDone
self.failUnless("unused=2" in desc)
self.failUnless("undefined=1" in desc)
self.failUnless("redefs=3" in desc)
self.failUnless("import*=1" in desc)
self.failIf("misc=" in desc)
self.failUnlessEqual(step.getProperty("pyflakes-unused"), 2)
self.failUnlessEqual(step.getProperty("pyflakes-undefined"), 1)
self.failUnlessEqual(step.getProperty("pyflakes-redefs"), 3)
self.failUnlessEqual(step.getProperty("pyflakes-import*"), 1)
self.failUnlessEqual(step.getProperty("pyflakes-misc"), 0)
self.failUnlessEqual(step.getProperty("pyflakes-total"), 7)
logs = {}
for log in step.step_status.getLogs():
logs[log.getName()] = log
for name in ["unused", "undefined", "redefs", "import*"]:
self.failUnless(name in logs)
self.failIf("misc" in logs)
lines = logs["unused"].readlines()
self.failUnlessEqual(len(lines), 2)
self.failUnlessEqual(
lines[0],
"buildbot/changes/freshcvsmail.py:5: 'FCMaildirSource' imported but unused\n"
)
cmd = buildstep.RemoteCommand(None, {})
cmd.rc = 0
results = step.evaluateCommand(cmd)
self.failUnlessEqual(results, FAILURE) # because of the 'undefined'
def testCompile4(self):
# Test suppression of warnings.
self.masterbase = "Warnings.testCompile4"
step = self.makeStep(
shell.Compile,
warningPattern="^(.*?):([0-9]+): [Ww]arning: (.*)$",
warningExtractor=shell.Compile.warnExtractFromRegexpGroups,
directoryEnterPattern="make.*: Entering directory [\"`'](.*)['`\"]",
directoryLeavePattern="make.*: Leaving directory")
step.addSuppression([
(r"/subdir/", r"xyzzy", None, None),
(r"foo.c", r".*", None, 20),
(r"foo.c", r".*", 200, None),
(r"foo.c", r".*", 50, 50),
(r"xxx", r".*", None, None),
])
log = step.addLog("stdio")
output = \
"""Making all in .
make[1]: Entering directory `/abs/path/build'
foo.c:10: warning: `bar' defined but not used
foo.c:50: warning: `bar' defined but not used
make[2]: Entering directory `/abs/path/build/subdir'
baz.c:33: warning: `xyzzy' defined but not used
baz.c:34: warning: `magic' defined but not used
make[2]: Leaving directory `/abs/path/build/subdir'
foo.c:100: warning: `xyzzy' defined but not used
foo.c:200: warning: `bar' defined but not used
make[2]: Leaving directory `/abs/path/build'
"""
log.addStdout(output)
log.finish()
step.createSummary(log)
self.failUnlessEqual(step.getProperty("warnings-count"), 2)
logs = {}
for log in step.step_status.getLogs():
logs[log.getName()] = log
self.failUnless("warnings" in logs)
lines = logs["warnings"].readlines()
self.failUnlessEqual(len(lines), 2)
self.failUnlessEqual(
lines[0], "baz.c:34: warning: `magic' defined but not used\n")
self.failUnlessEqual(
lines[1], "foo.c:100: warning: `xyzzy' defined but not used\n")
cmd = buildstep.RemoteCommand(None, {})
cmd.rc = 0
results = step.evaluateCommand(cmd)
self.failUnlessEqual(results, WARNINGS)
def copy(self):
cmd = buildstep.RemoteCommand('rmdir', {'dir': 'build',
'logEnviron': self.logEnviron,})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
d.addCallback(lambda _: self.incremental())
def copy(_):
cmd = buildstep.RemoteCommand('cpdir',
{'fromdir': 'source',
'todir':'build',
'logEnviron': self.logEnviron,})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
return d
d.addCallback(copy)
return d
def start(self):
slavever = self.slaveVersion('cpdir')
if not slavever:
raise BuildSlaveTooOldError("slave is too old, does not know "
"about cpdir")
args = {'fromdir': self.src, 'todir': self.dest}
if self.timeout:
args['timeout'] = self.timeout
if self.maxTime:
args['maxTime'] = self.maxTime
cmd = buildstep.RemoteCommand('cpdir', args)
d = self.runCommand(cmd)
d.addCallback(lambda res: self.commandComplete(cmd))
d.addErrback(self.failed)
def start(self):
if self.suppressionFile == None:
return ShellCommand.start(self)
self.myFileWriter = StringFileWriter()
args = {
'slavesrc': self.suppressionFile,
'workdir': self.getWorkdir(),
'writer': self.myFileWriter,
'maxsize': None,
'blocksize': 32*1024,
}
cmd = buildstep.RemoteCommand('uploadFile', args, ignore_updates=True)
d = self.runCommand(cmd)
d.addCallback(self.uploadDone)
d.addErrback(self.failed)
def cleanWorkdir(self):
cmd = buildstep.RemoteCommand('rmdir', {
'dir': self.workdir,
'logEnviron': self.logEnviron
},
collectStdout=True)
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
def evaluateCommand(cmd):
if cmd.didFail():
log.msg("Source step failed while cleaning the workdir %s" %
cmd)
return cmd.rc
d.addCallback(lambda _: evaluateCommand(cmd))
return d
def copy(self):
yield self.runRmdir(self.workdir)
# temporarily set workdir = 'source' and do an incremental checkout
try:
old_workdir = self.workdir
self.workdir = 'source'
yield self.incremental(None)
except: # finally doesn't work in python-2.4
self.workdir = old_workdir
raise
self.workdir = old_workdir
# if we're copying, copy; otherwise, export from source to build
if self.method == 'copy':
cmd = buildstep.RemoteCommand(
'cpdir', {
'fromdir': 'source',
'todir': self.workdir,
'logEnviron': self.logEnviron
})
else:
export_cmd = ['svn', 'export']
if self.revision:
export_cmd.extend(["--revision", str(self.revision)])
if self.username:
export_cmd.extend(['--username', self.username])
if self.password:
export_cmd.extend(
['--password', ('obfuscated', self.password, 'XXXXXX')])
if self.extra_args:
export_cmd.extend(self.extra_args)
export_cmd.extend(['source', self.workdir])
cmd = buildstep.RemoteShellCommand('',
export_cmd,
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout)
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if cmd.didFail():
raise buildstep.BuildStepFailed()
def parseAndRemove(stdout):
files = []
for filename in stdout.splitlines() :
filename = self.workdir+'/'+filename
files.append(filename)
if len(files) == 0:
d = defer.succeed(0)
else:
if self.slaveVersionIsOlderThan('rmdir', '2.14'):
d = self.removeFiles(files)
else:
cmd = buildstep.RemoteCommand('rmdir', {'dir': files,
'logEnviron':
self.logEnviron,})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
d.addCallback(lambda _: cmd.rc)
return d
def _sourcedirIsUpdatable(self):
# first, perform a stat to ensure that this is really an svn directory
cmd = buildstep.RemoteCommand('stat', {'file': self.workdir + '/.svn',
'logEnviron': self.logEnviron,})
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if cmd.rc != 0:
yield defer.returnValue(False)
return
# then run 'svn info' to check that the URL matches our repourl
stdout = yield self._dovccmd(['info'], collectStdout=True)
# extract the URL, handling whitespace carefully so that \r\n works
# is a line terminator
mo = re.search('^URL:\s*(.*?)\s*$', stdout, re.M)
yield defer.returnValue(mo and mo.group(1) == self.repourl)
return
def _downloadFile(buf, filename):
filereader = _FileReader(StringIO.StringIO(buf))
args = {
'slavedest': filename,
'maxsize': None,
'reader': filereader,
'blocksize': 16*1024,
'workdir': self.workdir,
'mode' : None
}
cmd = buildstep.RemoteCommand('downloadFile', args)
cmd.useLog(self.stdio_log, False)
log.msg("Downloading file: %s" % (filename))
d = self.runCommand(cmd)
def evaluateCommand(_):
if cmd.didFail():
raise buildstep.BuildStepFailed()
return cmd.rc
d.addCallback(evaluateCommand)
return d
def findReportsDir(self, cmd):
if cmd.didFail():
self.step_status.setText(["Reports not found."])
self.finished(WARNINGS)
return
s = cmd.updates["stat"][-1]
if not stat.S_ISDIR(s[stat.ST_MODE]):
self.step_status.setText(["This not a directory"])
self.finished(WARNINGS)
return
cmd = buildstep.RemoteCommand('glob', {
'path':
os.path.join(self.getWorkdir(), self.report_dir, '*.xml')
})
d = self.runCommand(cmd)
d.addCallback(lambda res: self.findReportsFiles(cmd))
d.addErrback(self.failed)
return d
def copy(self):
yield self.runRmdir(self.workdir, abandonOnFailure=False,
timeout=self.timeout)
old_workdir = self.workdir
self.workdir = self.srcdir
try:
yield self.mode_incremental()
cmd = buildstep.RemoteCommand('cpdir',
{'fromdir': self.srcdir,
'todir': old_workdir,
'logEnviron': self.logEnviron,
'timeout': self.timeout, })
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if cmd.didFail():
raise buildstep.BuildStepFailed()
return RC_SUCCESS
finally:
self.workdir = old_workdir
def copy(self):
cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir,
'logEnviron': self.logEnviron})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
old_workdir = self.workdir
self.workdir = self.srcdir
d.addCallback(lambda _: self.incremental())
def copy(_):
cmd = buildstep.RemoteCommand('cpdir',
{'fromdir': self.srcdir,
'todir': old_workdir,
'logEnviron': self.logEnviron,})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
return d
d.addCallback(copy)
def resetWorkdir(_):
self.workdir = old_workdir
return 0
d.addCallback(resetWorkdir)
return d
def _sourcedirIsUpdatable(self):
# first, perform a stat to ensure that this is really an svn directory
cmd = buildstep.RemoteCommand('stat', {'file': self.build.path_module.join(self.workdir, '.svn'),
'logEnviron': self.logEnviron,})
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if cmd.didFail():
defer.returnValue(False)
return
# then run 'svn info --xml' to check that the URL matches our repourl
stdout = yield self._dovccmd(['info', '--xml'], collectStdout=True)
try:
stdout_xml = xml.dom.minidom.parseString(stdout)
extractedurl = stdout_xml.getElementsByTagName('url')[0].firstChild.nodeValue
except xml.parsers.expat.ExpatError:
msg = "Corrupted xml, aborting step"
self.stdio_log.addHeader(msg)
raise buildstep.BuildStepFailed()
defer.returnValue(extractedurl == self.repourl)
return
def start(self):
args = {"arg1": "value"}
cmd = buildstep.RemoteCommand("simple", args)
d = self.runCommand(cmd)
d.addCallback(lambda res: self.finished(SUCCESS))
def makeRemoteCommand(self, name, args):
return buildstep.RemoteCommand(name, args)
def start(self):
cmd = buildstep.RemoteCommand('stat', {'file': self.basetgz})
d = self.runCommand(cmd)
d.addCallback(lambda res: self.checkBasetgz(cmd))
d.addErrback(self.failed)
return d
def makeStatusRemoteCommand(step, remote_command, args):
self = buildstep.RemoteCommand(remote_command, args)
callback = lambda arg: step.step_status.addLog('stdio')
self.useLogDelayed('stdio', callback, True)
return self
