def test_obfuscated_arguments(self):
command = [
"echo",
("obfuscated", "real", "fake"),
"test",
("obfuscated", "real2", "fake2"),
("not obfuscated", "a", "b"),
("obfuscated"), # not obfuscated
("obfuscated", "test"), # not obfuscated
("obfuscated", "1", "2", "3"), # not obfuscated)
]
cmd = buildstep.RemoteShellCommand("build", command)
self.assertEqual(cmd.command, command)
self.assertEqual(
cmd.fake_command,
[
"echo",
"fake",
"test",
"fake2",
("not obfuscated", "a", "b"),
("obfuscated"), # not obfuscated
("obfuscated", "test"), # not obfuscated
("obfuscated", "1", "2", "3"), # not obfuscated)
])
command = "echo test"
cmd = buildstep.RemoteShellCommand("build", command)
self.assertEqual(cmd.command, command)
self.assertEqual(cmd.fake_command, command)
def testRepr(self):
# Test for #352
rsc = buildstep.RemoteShellCommand('.', ('sh', 'make'))
testval = repr(rsc)
rsc = buildstep.RemoteShellCommand('.', ['sh', 'make'])
testval = repr(rsc)
rsc = buildstep.RemoteShellCommand('.', 'make')
testval = repr(rsc)
def checkBasetgz(self, cmd):
if cmd.rc != 0:
log.msg("basetgz not found, initializing it.")
command = [
'sudo', self.pbuilder, '--create', self.baseOption,
self.basetgz, '--distribution', self.distribution, '--mirror',
self.mirror
]
if self.architecture:
command += ['--architecture', self.architecture]
if self.extrapackages:
command += ['--extrapackages', " ".join(self.extrapackages)]
if self.keyring:
command += ['--debootstrapopts', "--keyring=%s" % self.keyring]
if self.components:
command += ['--components', self.components]
cmd = buildstep.RemoteShellCommand(self.getWorkdir(), command)
stdio_log = stdio_log = self.addLog("pbuilder")
cmd.useLog(stdio_log, True, "stdio")
d = self.runCommand(cmd)
self.step_status.setText(["PBuilder create."])
d.addCallback(lambda res: self.startBuild(cmd))
return d
s = cmd.updates["stat"][-1]
# basetgz will be a file when running in pbuilder
# and a directory in case of cowbuilder
if stat.S_ISREG(s[stat.ST_MODE]) or stat.S_ISDIR(s[stat.ST_MODE]):
log.msg("%s found." % self.basetgz)
age = time.time() - s[stat.ST_MTIME]
if age >= self.maxAge:
log.msg("basetgz outdated, updating")
command = [
'sudo', self.pbuilder, '--update', self.baseOption,
self.basetgz
]
cmd = buildstep.RemoteShellCommand(self.getWorkdir(), command)
stdio_log = stdio_log = self.addLog("pbuilder")
cmd.useLog(stdio_log, True, "stdio")
d = self.runCommand(cmd)
self.step_status.setText(["PBuilder update."])
d.addCallback(lambda res: self.startBuild(cmd))
return d
return self.startBuild(cmd)
else:
log.msg("%s is not a file or a directory." % self.basetgz)
self.finished(FAILURE)
def parseGotRevision(self, _):
# if this was a full/export, then we need to check svnversion in the
# *source* directory, not the build directory
svnversion_dir = self.workdir
if self.mode == 'full' and self.method == 'export':
svnversion_dir = 'source'
cmd = buildstep.RemoteShellCommand(svnversion_dir, ['svnversion'],
env=self.env,
logEnviron=self.logEnviron,
collectStdout=True)
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
def _setrev(_):
stdout = cmd.stdout.strip()
revision = stdout.rstrip('MSP')
revision = revision.split(':')[-1]
try:
int(revision)
except ValueError:
msg =("SVN.parseGotRevision unable to parse output "
"of svnversion: '%s'" % stdout)
log.msg(msg)
raise buildstep.BuildStepFailed()
log.msg("Got SVN revision %s" % (revision, ))
self.setProperty('got_revision', revision, 'Source')
return 0
d.addCallback(lambda _: _setrev(cmd.rc))
return d
def _dovccmd(self,
command,
collectStdout=False,
initialStdin=None,
successfulRC=(0, )):
if not command:
raise ValueError("No command specified")
cmd = buildstep.RemoteShellCommand(self.workdir,
['hg', '--verbose'] + command,
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout,
collectStdout=collectStdout,
initialStdin=initialStdin,
successfulRC=successfulRC)
cmd.useLog(self.stdio_log, False)
log.msg("Starting mercurial command : hg %s" % (" ".join(command), ))
d = self.runCommand(cmd)
def evaluateCommand(cmd):
if cmd.didFail():
log.msg("Source step failed while running command %s" % cmd)
raise buildstep.BuildStepFailed()
if collectStdout:
return cmd.stdout
else:
return cmd.rc
d.addCallback(lambda _: evaluateCommand(cmd))
return d
def parseGotRevision(self, _):
cmd = buildstep.RemoteShellCommand(self.workdir, ['svnversion'],
env=self.env,
logEnviron=self.logEnviron,
collectStdout=True)
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
def _setrev(_):
stdout = cmd.stdout.strip()
revision = stdout.rstrip('MSP')
revision = revision.split(':')[-1]
try:
int(revision)
except ValueError:
msg = ("SVN.parseGotRevision unable to parse output "
"of svnversion: '%s'" % stdout)
log.msg(msg)
raise buildstep.BuildStepFailed()
log.msg("Got SVN revision %s" % (revision, ))
self.setProperty('got_revision', revision, 'Source')
return 0
d.addCallback(lambda _: _setrev(cmd.rc))
return d
def checkP4(self):
cmd = buildstep.RemoteShellCommand(self.workdir, ['p4', '-V'],
env=self.env,
logEnviron=self.logEnviron)
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
return cmd.rc == 0
def _Cmd(self, command, abandonOnFailure=True, workdir=None, **kwargs):
if workdir is None:
workdir = self.workdir
cmd = buildstep.RemoteShellCommand(workdir,
command,
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout,
**kwargs)
self.lastCommand = cmd
# does not make sense to logEnviron for each command (just for first)
self.logEnviron = False
cmd.useLog(self.stdio_log, False)
self.stdio_log.addHeader("Starting command: {}\n".format(
" ".join(command)))
self.step_status.setText(["{}".format(" ".join(command[:2]))])
yield self.runCommand(cmd)
if abandonOnFailure and cmd.didFail():
self.descriptionDone = "repo failed at: {}".format(" ".join(
command[:2]))
self.stdio_log.addStderr(
("Source step failed while running command {}\n").format(cmd))
raise buildstep.BuildStepFailed()
return cmd.rc
def parseGotRevision(self, _):
# if this was a full/export, then we need to check svnversion in the
# *source* directory, not the build directory
svnversion_dir = self.workdir
if self.mode == 'full' and self.method == 'export':
svnversion_dir = 'source'
cmd = buildstep.RemoteShellCommand(svnversion_dir, ['svn', 'info'],
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout,
collectStdout=True)
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
def _setrev(_):
stdout = cmd.stdout
match = re.search('Revision:(.+?)\n', stdout)
try:
revision = int(match.group(1))
except (AttributeError, ValueError):
msg =("SVN.parseGotRevision unable to parse output "
"of svnversion: '%s'" % stdout)
log.msg(msg)
raise buildstep.BuildStepFailed()
log.msg("Got SVN revision %s" % (revision, ))
self.updateSourceProperty('got_revision', revision)
return 0
d.addCallback(lambda _: _setrev(cmd.rc))
return d
def start(self):
if self.autoRelease:
relfile = '%s.release' % (os.path.basename(
self.specfile).split('.')[0])
try:
with open(relfile, 'r') as rfile:
rel = int(rfile.readline().strip())
except (IOError, TypeError, ValueError):
rel = 0
self.rpmbuild = self.rpmbuild + ' --define "_release %s"' % rel
with open(relfile, 'w') as rfile:
rfile.write(str(rel + 1))
if self.vcsRevision:
revision = self.getProperty('got_revision')
# only do this in the case where there's a single codebase
if revision and not isinstance(revision, dict):
self.rpmbuild = (self.rpmbuild +
' --define "_revision %s"' % revision)
self.rpmbuild = self.rpmbuild + ' -ba %s' % self.specfile
self.command = self.rpmbuild
# create the actual RemoteShellCommand instance now
kwargs = self.remote_kwargs
kwargs['command'] = self.command
kwargs['workdir'] = self.workdir
cmd = buildstep.RemoteShellCommand(**kwargs)
self.setupEnvironment(cmd)
self.startCommand(cmd)
self.addLogObserver(
'stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
def _dovccmd(self, command, abandonOnFailure=True, collectStdout=False, initialStdin=None):
full_command = ['git']
if self.config is not None:
for name, value in self.config.iteritems():
full_command.append('-c')
full_command.append('%s=%s' % (name, value))
full_command.extend(command)
cmd = buildstep.RemoteShellCommand(self.workdir,
full_command,
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout,
collectStdout=collectStdout,
initialStdin=initialStdin)
cmd.useLog(self.stdio_log, False)
log.msg("Starting git command : git %s" % (" ".join(command), ))
d = self.runCommand(cmd)
def evaluateCommand(cmd):
if abandonOnFailure and cmd.didFail():
log.msg("Source step failed while running command %s" % cmd)
raise buildstep.BuildStepFailed()
if collectStdout:
return cmd.stdout
else:
return cmd.rc
d.addCallback(lambda _: evaluateCommand(cmd))
return d
def _dovccmd(self,
command,
workdir,
collectStdout=False,
initialStdin=None,
decodeRC=None,
abandonOnFailure=True):
if not command:
raise ValueError("No command specified")
if decodeRC is None:
decodeRC = {0: SUCCESS}
cmd = buildstep.RemoteShellCommand(workdir,
command,
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout,
collectStdout=collectStdout,
initialStdin=initialStdin,
decodeRC=decodeRC)
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if abandonOnFailure and cmd.didFail():
log.msg("Source step failed while running command %s" % cmd)
raise buildstep.BuildStepFailed()
if collectStdout:
defer.returnValue(cmd.stdout)
else:
defer.returnValue(cmd.rc)
def _dovccmd(self, command, collectStdout=False, initialStdin=None):
command = self._buildVCCommand(command)
if self.debug:
log.msg("P4:_dovccmd():workdir->{}".format(self.workdir))
cmd = buildstep.RemoteShellCommand(
self.workdir,
command,
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout,
collectStdout=collectStdout,
initialStdin=initialStdin,
)
cmd.useLog(self.stdio_log, False)
if self.debug:
log.msg("Starting p4 command : p4 {}".format(" ".join(command)))
yield self.runCommand(cmd)
if cmd.rc != 0:
if self.debug:
log.msg(
"P4:_dovccmd():Source step failed while running command {}"
.format(cmd))
raise buildstep.BuildStepFailed()
if collectStdout:
return cmd.stdout
return cmd.rc
def parseGotRevision(self):
command = self._buildVCCommand(['changes', '-m1', '#have'])
cmd = buildstep.RemoteShellCommand(self.workdir,
command,
env=self.env,
timeout=self.timeout,
logEnviron=self.logEnviron,
collectStdout=True)
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
stdout = cmd.stdout.strip()
# Example output from p4 changes -m1 #have
# Change 212798 on 2012/04/13 by user@user-unix-bldng2 'change to
# pickup build'
revision = stdout.split()[1]
try:
int(revision)
except ValueError as e:
msg = (("p4.parseGotRevision unable to parse output "
"of 'p4 changes -m1 \"#have\"': '{}'").format(stdout))
log.msg(msg)
raise buildstep.BuildStepFailed() from e
if self.debug:
log.msg("Got p4 revision {}".format(revision))
self.updateSourceProperty('got_revision', revision)
def _dovccmd(self, command, collectStdout=False, initialStdin=None):
command = self._buildVCCommand(command)
if debug_logging:
log.msg("P4:_dovccmd():workdir->%s" % self.workdir)
cmd = buildstep.RemoteShellCommand(
self.workdir,
command,
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout,
collectStdout=collectStdout,
initialStdin=initialStdin,
)
cmd.useLog(self.stdio_log, False)
if debug_logging:
log.msg("Starting p4 command : p4 %s" % (" ".join(command), ))
d = self.runCommand(cmd)
def evaluateCommand(cmd):
if cmd.rc != 0:
if debug_logging:
log.msg(
"P4:_dovccmd():Source step failed while running command %s"
% cmd)
raise buildstep.BuildStepFailed()
if collectStdout:
return cmd.stdout
else:
return cmd.rc
d.addCallback(lambda _: evaluateCommand(cmd))
return d
def parseGotRevision(self, _):
command = self._buildVCCommand(['changes', '-m1', '#have'])
cmd = buildstep.RemoteShellCommand(self.workdir,
command,
env=self.env,
timeout=self.timeout,
logEnviron=self.logEnviron,
collectStdout=True)
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
def _setrev(_):
stdout = cmd.stdout.strip()
# Example output from p4 changes -m1 #have
# Change 212798 on 2012/04/13 by user@user-unix-bldng2 'change to pickup build'
revision = stdout.split()[1]
try:
int(revision)
except ValueError:
msg = ("p4.parseGotRevision unable to parse output "
"of 'p4 changes -m1 \"#have\"': '%s'" % stdout)
log.msg(msg)
raise buildstep.BuildStepFailed()
if debug_logging:
log.msg("Got p4 revision %s" % (revision, ))
self.updateSourceProperty('got_revision', revision)
return 0
d.addCallback(lambda _: _setrev(cmd.rc))
return d
def _Cmd(self, command, abandonOnFailure=True, workdir=None, **kwargs):
if workdir is None:
workdir = self.workdir
self.cmd = cmd = buildstep.RemoteShellCommand(
workdir,
command,
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout,
**kwargs)
# does not make sense to logEnviron for each command (just for first)
self.logEnviron = False
cmd.useLog(self.stdio_log, False)
self.stdio_log.addHeader("Starting command: %s\n" %
(" ".join(command), ))
self.step_status.setText(["%s" % (" ".join(command[:2]))])
d = self.runCommand(cmd)
def evaluateCommand(cmd):
if abandonOnFailure and cmd.didFail():
self.step_status.setText(
["repo failed at: %s" % (" ".join(command[:2]))])
self.stdio_log.addStderr(
"Source step failed while running command %s\n" % cmd)
raise buildstep.BuildStepFailed()
return cmd.rc
d.addCallback(lambda _: evaluateCommand(cmd))
return d
def _dovccmd(self,
command,
collectStdout=False,
initialStdin=None,
decodeRC={0: SUCCESS},
abandonOnFailure=True,
wkdir=None):
if not command:
raise ValueError("No command specified")
workdir = wkdir or self.workdir
cmd = buildstep.RemoteShellCommand(workdir,
command,
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout,
collectStdout=collectStdout,
initialStdin=initialStdin,
decodeRC=decodeRC)
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
def evaluateCommand(cmd):
if abandonOnFailure and cmd.didFail():
log.msg("Source step failed while running command %s" % cmd)
raise buildstep.BuildStepFailed()
if collectStdout:
return cmd.stdout
else:
return cmd.rc
d.addCallback(lambda _: evaluateCommand(cmd))
return d
def _dovccmd(self, command, collectStdout=False):
assert command, "No command specified"
command.extend(['--non-interactive', '--no-auth-cache'])
if self.username:
command.extend(['--username', self.username])
if self.password:
command.extend(['--password', self.password])
if self.depth:
command.extend(['--depth', self.depth])
if self.extra_args:
command.extend(self.extra_args)
cmd = buildstep.RemoteShellCommand(self.workdir, ['svn'] + command,
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout,
collectStdout=collectStdout)
cmd.useLog(self.stdio_log, False)
log.msg("Starting SVN command : svn %s" % (" ".join(command), ))
d = self.runCommand(cmd)
def evaluateCommand(cmd):
if cmd.didFail():
log.msg("Source step failed while running command %s" % cmd)
raise buildstep.BuildStepFailed()
if collectStdout:
return cmd.stdout
else:
return cmd.rc
d.addCallback(lambda _: evaluateCommand(cmd))
return d
def _dovccmd(self,
command,
abandonOnFailure=True,
collectStdout=False,
initialStdin=None):
cmd = buildstep.RemoteShellCommand(self.workdir, ['git'] + command,
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout,
collectStdout=collectStdout,
initialStdin=initialStdin)
cmd.useLog(self.stdio_log, False)
log.msg("Starting git command : git %s" % (" ".join(command), ))
d = self.runCommand(cmd)
def evaluateCommand(cmd):
if abandonOnFailure and cmd.didFail():
log.msg("Source step failed while running command %s" % cmd)
raise buildstep.BuildStepFailed()
if collectStdout:
return cmd.stdout
else:
return cmd.rc
d.addCallback(lambda _: evaluateCommand(cmd))
return d
def start(self):
if self.autoRelease:
relfile = '%s.release' % (os.path.basename(
self.specfile).split('.')[0])
try:
with open(relfile, 'r') as rfile:
rel = int(rfile.readline().strip())
except:
rel = 0
self.rpmbuild = self.rpmbuild + ' --define "_release %s"' % rel
with open(relfile, 'w') as rfile:
rfile.write(str(rel + 1))
if self.vcsRevision:
self.rpmbuild = self.rpmbuild + ' --define "_revision %s"' % \
self.getProperty('got_revision')
self.rpmbuild = self.rpmbuild + ' -ba %s' % self.specfile
self.command = self.rpmbuild
# create the actual RemoteShellCommand instance now
kwargs = self.remote_kwargs
kwargs['command'] = self.command
cmd = buildstep.RemoteShellCommand(**kwargs)
self.setupEnvironment(cmd)
self.startCommand(cmd)
def copy(self):
yield self.runRmdir(self.workdir)
# temporarily set workdir = 'source' and do an incremental checkout
try:
old_workdir = self.workdir
self.workdir = 'source'
yield self.incremental(None)
except: # finally doesn't work in python-2.4
self.workdir = old_workdir
raise
self.workdir = old_workdir
# if we're copying, copy; otherwise, export from source to build
if self.method == 'copy':
cmd = buildstep.RemoteCommand('cpdir',
{ 'fromdir': 'source', 'todir':self.workdir,
'logEnviron': self.logEnviron })
else:
export_cmd = ['svn', 'export']
if self.revision:
export_cmd.extend(["--revision", str(self.revision)])
export_cmd.extend(['source', self.workdir])
cmd = buildstep.RemoteShellCommand('', export_cmd,
env=self.env, logEnviron=self.logEnviron, timeout=self.timeout)
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
if cmd.didFail():
raise buildstep.BuildStepFailed()
def test_runCommand(self):
bs = buildstep.BuildStep()
bs.buildslave = slave.FakeSlave()
bs.remote = 'dummy'
cmd = buildstep.RemoteShellCommand("build", ["echo", "hello"])
cmd.run = lambda self, remote: SUCCESS
bs.runCommand(cmd)
self.assertEqual(bs.cmd, cmd)
def checkSvn(self):
cmd = buildstep.RemoteShellCommand(self.workdir, ['svn', '--version'],
env=self.env,
logEnviron=self.logEnviron)
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
def evaluate(cmd):
if cmd.rc != 0:
return False
return True
d.addCallback(lambda _: evaluate(cmd))
return d
def getStepConfig(self):
log = self.addLog(".travis.yml")
cmd = self.cmd = buildstep.RemoteShellCommand(workdir="build", command=["cat", ".travis.yml"])
cmd.useLog(log, False, "stdio")
yield self.runCommand(cmd)
self.cmd = None
if cmd.rc != 0:
raise buildstep.BuildStepFailed()
config = TravisYml()
config.parse(log.getText())
defer.returnValue(config)
def checkP4(self):
cmd = buildstep.RemoteShellCommand(self.workdir, ['p4', '-V'],
env=self.env,
logEnviron=self.logEnviron)
cmd.useLog(self.stdio_log, False)
d = self.runCommand(cmd)
@d.addCallback
def evaluate(_):
if cmd.rc != 0:
return False
return True
return d
def start(self):
# this block is specific to ShellCommands. subclasses that don't need
# to set up an argv array, an environment, or extra logfiles= (like
# the Source subclasses) can just skip straight to startCommand()
warnings = []
# create the actual RemoteShellCommand instance now
kwargs = self.buildCommandKwargs(warnings)
cmd = buildstep.RemoteShellCommand(**kwargs)
self.setupEnvironment(cmd)
self.startCommand(cmd, warnings)
def parseGotRevision(self, _):
# if this was a full/export, then we need to check svnversion in the
# *source* directory, not the build directory
svnversion_dir = self.workdir
if self.mode == 'full' and self.method == 'export':
svnversion_dir = 'source'
cmd = buildstep.RemoteShellCommand(svnversion_dir,
['svn', 'info', '--xml'],
env=self.env,
logEnviron=self.logEnviron,
timeout=self.timeout,
collectStdout=True)
cmd.useLog(self.stdio_log, False)
yield self.runCommand(cmd)
stdout = cmd.stdout
try:
stdout_xml = xml.dom.minidom.parseString(stdout)
except xml.parsers.expat.ExpatError:
msg = "Corrupted xml, aborting step"
self.stdio_log.addHeader(msg)
raise buildstep.BuildStepFailed()
revision = None
if self.preferLastChangedRev:
try:
revision = stdout_xml.getElementsByTagName(
'commit')[0].attributes['revision'].value
except (KeyError, IndexError):
msg = (
"SVN.parseGotRevision unable to detect Last Changed Rev in"
" output of svn info")
log.msg(msg)
# fall through and try to get 'Revision' instead
if revision is None:
try:
revision = stdout_xml.getElementsByTagName(
'entry')[0].attributes['revision'].value
except (KeyError, IndexError):
msg = ("SVN.parseGotRevision unable to detect revision in"
" output of svn info")
log.msg(msg)
raise buildstep.BuildStepFailed()
msg = "Got SVN revision %s" % (revision, )
self.stdio_log.addHeader(msg)
self.updateSourceProperty('got_revision', revision)
defer.returnValue(cmd.rc)
def startVC(self, branch, revision, patch):
command = ['jhbuild']
if (self.moduleset is not None):
command += ['--moduleset=' + self.moduleset]
command += ['bot', '--step', 'updateone', self.module]
properties = self.build.getProperties()
kwargs = {}
kwargs['workdir'] = './'
#kwargs = properties.render(self.remote_kwargs)
kwargs['command'] = properties.render(command)
kwargs['env'] = {}
kwargs['timeout'] = 60 * 60
cmd = buildstep.RemoteShellCommand(**kwargs)
self.startCommand(cmd)
def _dovccmd(self, command, abandonOnFailure=True):
cmd = buildstep.RemoteShellCommand(self.workdir, ['git'] + command)
cmd.useLog(self.stdio_log, False)
log.msg("Starting git command : git %s" % (" ".join(command), ))
d = self.runCommand(cmd)
def evaluateCommand(cmd):
if abandonOnFailure and cmd.rc != 0:
log.msg("Source step failed while running command %s" % cmd)
raise failure.Failure(cmd.rc)
return cmd.rc
d.addCallback(lambda _: evaluateCommand(cmd))
return d
