def _sourcedirIsUpdatable(self):
myFileWriter = StringFileWriter()
args = {
'workdir': self.build.path_module.join(self.workdir, 'CVS'),
'writer': myFileWriter,
'maxsize': None,
'blocksize': 32 * 1024,
}
def uploadFileArgs(source):
full_args = dict(args)
if self.workerVersionIsOlderThan('uploadFile', '3.0'):
full_args['slavesrc'] = source
else:
full_args['workersrc'] = source
return full_args
cmd = remotecommand.RemoteCommand('uploadFile',
uploadFileArgs('Root'),
ignore_updates=True)
yield self.runCommand(cmd)
if cmd.rc is not None and cmd.rc != 0:
defer.returnValue(False)
return
# on Windows, the cvsroot may not contain the password, so compare to
# both
cvsroot_without_pw = re.sub("(:pserver:[^:]*):[^@]*(@.*)", r"\1\2",
self.cvsroot)
if myFileWriter.buffer.strip() not in (self.cvsroot,
cvsroot_without_pw):
defer.returnValue(False)
return
myFileWriter.buffer = ""
cmd = remotecommand.RemoteCommand('uploadFile',
uploadFileArgs('Repository'),
ignore_updates=True)
yield self.runCommand(cmd)
if cmd.rc is not None and cmd.rc != 0:
defer.returnValue(False)
return
if myFileWriter.buffer.strip() != self.cvsmodule:
defer.returnValue(False)
return
# if there are sticky dates (from an earlier build with revision),
# we can't update (unless we remove those tags with cvs update -A)
myFileWriter.buffer = ""
cmd = buildstep.RemoteCommand('uploadFile',
uploadFileArgs('Entries'),
ignore_updates=True)
yield self.runCommand(cmd)
if cmd.rc is not None and cmd.rc != 0:
defer.returnValue(False)
return
if self._cvsEntriesContainStickyDates(myFileWriter.buffer):
defer.returnValue(False)
defer.returnValue(True)
def copy(self):
cmd = remotecommand.RemoteCommand(
'rmdir', {
'dir': self.workdir,
'logEnviron': self.logEnviron,
'timeout': self.timeout
})
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
old_workdir = self.workdir
self.workdir = self.srcdir
yield self.mode_incremental()
cmd = remotecommand.RemoteCommand(
'cpdir', {
'fromdir': self.srcdir,
'todir': old_workdir,
'logEnviron': self.logEnviron,
'timeout': self.timeout
})
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
self.workdir = old_workdir
return results.SUCCESS
def run(self):
self.checkWorkerHasCommand('stat')
statSrc = remotecommand.RemoteCommand('stat',
{'workdir': self.workdir,
'file': self.src})
statGenerated = remotecommand.RemoteCommand('stat',
{'workdir': self.workdir,
'file': self.generated})
yield self.runCommand(statSrc)
yield self.runCommand(statGenerated)
if statSrc.didFail():
# Uh oh: without source no generation
self.descriptionDone = ["source file not found."]
return util.FAILURE
if statGenerated.didFail():
# No generated file: set property to True
self.setProperty(self.property, True, self.name)
self.descriptionDone = ["generated file not found."]
return util.SUCCESS
# stat object is lost when marshalling and result is seen as a tuple, doc says st_mtime is eighth
dateSrc = statSrc.updates["stat"][-1][8]
dateGenerated = statGenerated.updates["stat"][-1][8]
log = "{0}: {1!s}\n".format(self.src, datetime.fromtimestamp(dateSrc))
log += "{0}: {1!s}\n".format(self.generated, datetime.fromtimestamp(dateGenerated))
yield self.addCompleteLog('timestamps', log)
# Set to True if older
self.setProperty(self.property, dateGenerated <= dateSrc, self.name)
self.descriptionDone = ["generated file is {0} than source file".format("older" if dateGenerated <= dateSrc else "newer")]
return util.SUCCESS
def copy(self):
cmd = remotecommand.RemoteCommand('rmdir', {'dir': 'build',
'logEnviron': self.logEnviron, })
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
yield self.mode_incremental()
cmd = remotecommand.RemoteCommand('cpdir',
{'fromdir': 'source',
'todir': 'build',
'logEnviron': self.logEnviron, })
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
def _downloadFile(buf, filename):
filereader = remotetransfer.StringFileReader(buf)
args = {
'maxsize': None,
'reader': filereader,
'blocksize': 16 * 1024,
'workdir': self.workdir,
'mode': None
}
if self.workerVersionIsOlderThan('downloadFile', '3.0'):
args['slavedest'] = filename
else:
args['workerdest'] = filename
cmd = remotecommand.RemoteCommand('downloadFile', args)
cmd.useLog(self.stdio_log, False)
log.msg("Downloading file: %s" % (filename))
d = self.runCommand(cmd)
@d.addCallback
def evaluateCommand(_):
if cmd.didFail():
raise buildstep.BuildStepFailed()
return cmd.rc
return d
def _uploadRemoteReport(self, workersrc, masterdest):
"""Upload the remote file into the master.
Args:
workersrc (str): Path of the file to upload in the worker.
masterdest (str): Path of the file to write in the master.
"""
file_writer = remotetransfer.FileWriter(masterdest, self.maxSize, None)
args = {
'workdir': self.workdir,
'writer': file_writer,
'maxsize': self.maxSize,
'blocksize': self.blockSize,
'keepstamp': False,
'workersrc': workersrc,
}
cmd = remotecommand.RemoteCommand('uploadFile', args)
self.cmd = cmd
try:
yield self.runCommand(cmd)
except Exception:
file_writer.cancel()
raise
finally:
self.cmd = None
if cmd.didFail():
file_writer.cancel()
defer.returnValue(FAILURE)
defer.returnValue(SUCCESS)
def patch(self, patch):
diff = patch[1]
root = None
if len(patch) >= 3:
root = patch[2]
if root:
workdir_root = self.build.path_module.join(self.workdir, root)
workdir_root_abspath = self.build.path_module.abspath(workdir_root)
workdir_abspath = self.build.path_module.abspath(self.workdir)
if workdir_root_abspath.startswith(workdir_abspath):
self.workdir = workdir_root
yield self.downloadFileContentToWorker('.buildbot-diff', diff)
yield self.downloadFileContentToWorker('.buildbot-patched', 'patched\n')
yield self.applyPatch(patch)
cmd = remotecommand.RemoteCommand('rmdir',
{'dir': self.build.path_module.join(self.workdir,
".buildbot-diff"),
'logEnviron': self.logEnviron})
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if cmd.didFail():
raise buildstep.BuildStepFailed()
return cmd.rc
def _removeAddedFilesAndUpdate(self, _):
command = ['locate', 'set:added()']
stdout = yield self._dovccmd(command,
collectStdout=True,
decodeRC={
0: SUCCESS,
1: SUCCESS
})
files = []
for filename in stdout.splitlines():
filename = self.workdir + '/' + filename
files.append(filename)
if files:
if self.workerVersionIsOlderThan('rmdir', '2.14'):
yield self.removeFiles(files)
else:
cmd = remotecommand.RemoteCommand(
'rmdir', {
'dir': files,
'logEnviron': self.logEnviron,
})
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
yield self._update()
def _clobber(self):
cmd = remotecommand.RemoteCommand('rmdir', {
'dir': self.workdir,
'logEnviron': self.logEnviron
})
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
def copy(self):
cmd = remotecommand.RemoteCommand(
'rmdir', {
'dir': self.workdir,
'logEnviron': self.logEnviron,
'timeout': self.timeout,
})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
old_workdir = self.workdir
self.workdir = self.srcdir
d.addCallback(lambda _: self.incremental())
def copy(_):
cmd = remotecommand.RemoteCommand(
'cpdir', {
'fromdir': self.srcdir,
'todir': old_workdir,
'logEnviron': self.logEnviron,
'timeout': self.timeout,
})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
return d
d.addCallback(copy)
def resetWorkdir(_):
self.workdir = old_workdir
return 0
d.addCallback(resetWorkdir)
return d
def copy(self):
cmd = remotecommand.RemoteCommand(
'rmdir', {
'dir': self.workdir,
'logEnviron': self.logEnviron,
'timeout': self.timeout,
})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
self.workdir = 'source'
d.addCallback(lambda _: self.incremental())
@d.addCallback
def copy(_):
cmd = remotecommand.RemoteCommand(
'cpdir', {
'fromdir': 'source',
'todir': 'build',
'logEnviron': self.logEnviron,
'timeout': self.timeout,
})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
return d
@d.addCallback
def resetWorkdir(_):
self.workdir = 'build'
return 0
return d
def start(self):
"""
Try to remove the old mock logs first.
"""
if self.resultdir:
for lname in self.mock_logfiles:
self.logfiles[lname] = self.build.path_module.join(
self.resultdir, lname)
else:
for lname in self.mock_logfiles:
self.logfiles[lname] = lname
self.addLogObserver('state.log', MockStateObserver())
cmd = remotecommand.RemoteCommand(
'rmdir', {
'dir':
map(
lambda l: self.build.path_module.join(
'build', self.logfiles[l]), self.mock_logfiles)
})
d = self.runCommand(cmd)
def removeDone(cmd):
ShellCommand.start(self)
d.addCallback(removeDone)
d.addErrback(self.failed)
def run(self):
def dCheck(d):
if not isinstance(d, defer.Deferred):
raise AssertionError("expected Deferred")
return d
# don't complete immediately, or synchronously
yield util.asyncSleep(0)
lo = TestLogObserver()
self.addLogObserver('testlog', lo)
log = yield dCheck(self.addLog('testlog'))
yield dCheck(log.addStdout(u'stdout\n'))
yield dCheck(self.addCompleteLog('obs',
'Observer saw %r' % (map(unicode, lo.observed),)))
yield dCheck(self.addHTMLLog('foo.html', '<head>\n'))
yield dCheck(self.addURL('linkie', 'http://foo'))
cmd = remotecommand.RemoteCommand('fake', {})
cmd.useLog(log)
stdio = yield dCheck(self.addLog('stdio'))
cmd.useLog(stdio)
yield dCheck(cmd.addStdout(u'stdio\n'))
yield dCheck(cmd.addStderr(u'stderr\n'))
yield dCheck(cmd.addHeader(u'hdr\n'))
yield dCheck(cmd.addToLog('testlog', 'fromcmd\n'))
yield dCheck(log.finish())
defer.returnValue(results.SUCCESS)
def _statOnRemoteReport(self, filepath):
"""Execute 'stat' command on remote file.
Returns:
The output of `stat` or None if the given file doesn't exists on
the worker.
"""
args = {
'file': filepath,
'workdir': self.workdir,
}
cmd = remotecommand.RemoteCommand('stat', args)
self.cmd = cmd
try:
yield self.runCommand(cmd)
finally:
self.cmd = None
if not cmd.didFail():
result = cmd.updates['stat'][-1]
else:
result = None
defer.returnValue(result)
def start(self):
"""
Try to remove the old mock logs first.
"""
if self.resultdir:
for lname in self.mock_logfiles:
self.logfiles[lname] = self.build.path_module.join(
self.resultdir, lname)
else:
for lname in self.mock_logfiles:
self.logfiles[lname] = lname
self.addLogObserver('state.log', MockStateObserver())
cmd = remotecommand.RemoteCommand(
'rmdir', {
'dir': [
self.build.path_module.join('build', self.logfiles[l])
for l in self.mock_logfiles
]
})
d = self.runCommand(cmd)
# must resolve super() outside of the callback context.
super_ = super()
@d.addCallback
def removeDone(cmd):
super_.start()
d.addErrback(self.failed)
def patch(self, _, patch):
diff = patch[1]
root = None
if len(patch) >= 3:
root = patch[2]
if (root and
self.build.path_module.abspath(self.build.path_module.join(self.workdir, root)
).startswith(self.build.path_module.abspath(self.workdir))):
self.workdir = self.build.path_module.join(self.workdir, root)
d = self.downloadFileContentToWorker('.buildbot-diff', diff)
d.addCallback(
lambda _: self.downloadFileContentToWorker('.buildbot-patched', 'patched\n'))
d.addCallback(lambda _: self.applyPatch(patch))
cmd = remotecommand.RemoteCommand('rmdir', {'dir': self.build.path_module.join(self.workdir, ".buildbot-diff"),
'logEnviron': self.logEnviron})
cmd.useLog(self.stdio_log, False)
d.addCallback(lambda _: self.runCommand(cmd))
@d.addCallback
def evaluateCommand(_):
if cmd.didFail():
raise buildstep.BuildStepFailed()
return cmd.rc
return d
def _sourcedirIsUpdatable(self):
if self.workerVersionIsOlderThan('listdir', '2.16'):
git_path = self.build.path_module.join(self.workdir, '.git')
exists = yield self.pathExists(git_path)
if exists:
return "update"
return "clone"
cmd = remotecommand.RemoteCommand(
'listdir', {
'dir': self.workdir,
'logEnviron': self.logEnviron,
'timeout': self.timeout,
})
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if 'files' not in cmd.updates:
# no files - directory doesn't exist
return "clone"
files = cmd.updates['files'][0]
if '.git' in files:
return "update"
elif files:
return "clobber"
else:
return "clone"
def copy(self):
cmd = remotecommand.RemoteCommand(
'rmdir', {
'dir': self.workdir,
'logEnviron': self.logEnviron,
'timeout': self.timeout
})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
old_workdir = self.workdir
self.workdir = self.srcdir
d.addCallback(self.mode_incremental)
@d.addCallback
def copy(_):
cmd = remotecommand.RemoteCommand(
'cpdir', {
'fromdir': self.srcdir,
'todir': old_workdir,
'logEnviron': self.logEnviron,
'timeout': self.timeout
})
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
return d
@d.addCallback
def resetWorkdir(_):
self.workdir = old_workdir
return 0
return d
def patch(self, _, patch):
diff = patch[1]
root = None
if len(patch) >= 3:
root = patch[2]
if (root and self.build.path_module.abspath(
self.build.path_module.join(self.workdir, root)).startswith(
self.build.path_module.abspath(self.workdir))):
self.workdir = self.build.path_module.join(self.workdir, root)
def _downloadFile(buf, filename):
filereader = remotetransfer.StringFileReader(buf)
args = {
'maxsize': None,
'reader': filereader,
'blocksize': 16 * 1024,
'workdir': self.workdir,
'mode': None
}
if self.workerVersionIsOlderThan('downloadFile', '3.0'):
args['slavedest'] = filename
else:
args['workerdest'] = filename
cmd = remotecommand.RemoteCommand('downloadFile', args)
cmd.useLog(self.stdio_log, False)
log.msg("Downloading file: %s" % (filename))
d = self.runCommand(cmd)
@d.addCallback
def evaluateCommand(_):
if cmd.didFail():
raise buildstep.BuildStepFailed()
return cmd.rc
return d
d = _downloadFile(diff, ".buildbot-diff")
d.addCallback(
lambda _: _downloadFile("patched\n", ".buildbot-patched"))
d.addCallback(lambda _: self.applyPatch(patch))
cmd = remotecommand.RemoteCommand(
'rmdir', {
'dir': self.build.path_module.join(self.workdir,
".buildbot-diff"),
'logEnviron': self.logEnviron
})
cmd.useLog(self.stdio_log, False)
d.addCallback(lambda _: self.runCommand(cmd))
@d.addCallback
def evaluateCommand(_):
if cmd.didFail():
raise buildstep.BuildStepFailed()
return cmd.rc
return d
def copy(_):
cmd = remotecommand.RemoteCommand('cpdir',
{'fromdir': 'source',
'todir': 'build',
'logEnviron': self.logEnviron, })
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
return d
def _clobber(self):
cmd = remotecommand.RemoteCommand('rmdir', {'dir': self.workdir,
'logEnviron': self.logEnviron, })
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if cmd.rc != 0:
raise RuntimeError("Failed to delete directory")
def checkBasetgz(self):
cmd = remotecommand.RemoteCommand('stat', {'file': self.basetgz})
yield self.runCommand(cmd)
if cmd.rc != 0:
log.msg("basetgz not found, initializing it.")
command = ['sudo', self.pbuilder, '--create', self.baseOption,
self.basetgz, '--distribution', self.distribution,
'--mirror', self.mirror]
if self.othermirror:
command += ['--othermirror', self.othermirror]
if self.architecture:
command += ['--architecture', self.architecture]
if self.extrapackages:
command += ['--extrapackages', " ".join(self.extrapackages)]
if self.keyring:
command += ['--debootstrapopts', "--keyring={}".format(self.keyring)]
if self.components:
command += ['--components', self.components]
cmd = remotecommand.RemoteShellCommand(self.workdir, command)
stdio_log = yield self.addLog("pbuilder")
cmd.useLog(stdio_log, True, "stdio")
self.description = ["PBuilder", "create."]
yield self.updateSummary()
yield self.runCommand(cmd)
if cmd.rc != 0:
log.msg("Failure when running {}.".format(cmd))
return results.FAILURE
return results.SUCCESS
s = cmd.updates["stat"][-1]
# basetgz will be a file when running in pbuilder
# and a directory in case of cowbuilder
if stat.S_ISREG(s[stat.ST_MODE]) or stat.S_ISDIR(s[stat.ST_MODE]):
log.msg("{} found.".format(self.basetgz))
age = time.time() - s[stat.ST_MTIME]
if age >= self.maxAge:
log.msg("basetgz outdated, updating")
command = ['sudo', self.pbuilder, '--update',
self.baseOption, self.basetgz]
cmd = remotecommand.RemoteShellCommand(self.workdir, command)
stdio_log = yield self.addLog("pbuilder")
cmd.useLog(stdio_log, True, "stdio")
yield self.runCommand(cmd)
if cmd.rc != 0:
log.msg("Failure when running {}.".format(cmd))
return results.FAILURE
return results.SUCCESS
log.msg("{} is not a file or a directory.".format(self.basetgz))
return results.FAILURE
def start(self):
slavever = self.slaveVersion('stat')
if not slavever:
raise BuildSlaveTooOldError("slave is too old, does not know "
"about stat")
cmd = remotecommand.RemoteCommand('stat', {'file': self.file})
d = self.runCommand(cmd)
d.addCallback(lambda res: self.commandComplete(cmd))
d.addErrback(self.failed)
def makeStatusRemoteCommand(step, remote_command, args):
self = remotecommand.RemoteCommand(remote_command,
args,
decodeRC={
None: SUCCESS,
0: SUCCESS
})
self.useLog(step.stdio_log)
return self
def removeFiles(self, files):
for filename in files:
cmd = remotecommand.RemoteCommand('rmdir', {'dir': filename,
'logEnviron': self.logEnviron, })
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if cmd.rc != 0:
return cmd.rc
return 0
def copy(_):
cmd = remotecommand.RemoteCommand('cpdir',
{'fromdir': self.srcdir,
'todir': old_workdir,
'logEnviron': self.logEnviron,
'timeout': self.timeout, })
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
return d
def run(self):
self.checkWorkerHasCommand('mkdir')
cmd = remotecommand.RemoteCommand('mkdir', {'dir': self.dir})
yield self.runCommand(cmd)
if cmd.didFail():
self.descriptionDone = ["Create failed."]
return FAILURE
return SUCCESS
def makeStatusRemoteCommand(step, remote_command, args):
self = remotecommand.RemoteCommand(remote_command,
args,
decodeRC={
None: SUCCESS,
0: SUCCESS
})
callback = lambda arg: step.step_status.addLog('stdio')
self.useLogDelayed('stdio', callback, True)
return self
def copy(self):
cmd = remotecommand.RemoteCommand('rmdir', {
'dir': self.workdir,
'logEnviron': self.logEnviron,
'timeout': self.timeout, })
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
self.workdir = 'source'
yield self.mode_incremental()
cmd = remotecommand.RemoteCommand('cpdir',
{'fromdir': 'source',
'todir': 'build',
'logEnviron': self.logEnviron,
'timeout': self.timeout, })
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
self.workdir = 'build'
defer.returnValue(0)
def start(self):
self.checkWorkerHasCommand('cpdir')
args = {'fromdir': self.src, 'todir': self.dest}
args['timeout'] = self.timeout
if self.maxTime:
args['maxTime'] = self.maxTime
cmd = remotecommand.RemoteCommand('cpdir', args)
d = self.runCommand(cmd)
d.addCallback(lambda res: self.commandComplete(cmd))
d.addErrback(self.failed)