def test_obfuscated_arguments(self): command = [ "echo", ("obfuscated", "real", "fake"), "test", ("obfuscated", "real2", "fake2"), ("not obfuscated", "a", "b"), ("obfuscated"), # not obfuscated ("obfuscated", "test"), # not obfuscated ("obfuscated", "1", "2", "3"), # not obfuscated) ] cmd = buildstep.RemoteShellCommand("build", command) self.assertEqual(cmd.command, command) self.assertEqual( cmd.fake_command, [ "echo", "fake", "test", "fake2", ("not obfuscated", "a", "b"), ("obfuscated"), # not obfuscated ("obfuscated", "test"), # not obfuscated ("obfuscated", "1", "2", "3"), # not obfuscated) ]) command = "echo test" cmd = buildstep.RemoteShellCommand("build", command) self.assertEqual(cmd.command, command) self.assertEqual(cmd.fake_command, command)
def testRepr(self): # Test for #352 rsc = buildstep.RemoteShellCommand('.', ('sh', 'make')) testval = repr(rsc) rsc = buildstep.RemoteShellCommand('.', ['sh', 'make']) testval = repr(rsc) rsc = buildstep.RemoteShellCommand('.', 'make') testval = repr(rsc)
def checkBasetgz(self, cmd): if cmd.rc != 0: log.msg("basetgz not found, initializing it.") command = [ 'sudo', self.pbuilder, '--create', self.baseOption, self.basetgz, '--distribution', self.distribution, '--mirror', self.mirror ] if self.architecture: command += ['--architecture', self.architecture] if self.extrapackages: command += ['--extrapackages', " ".join(self.extrapackages)] if self.keyring: command += ['--debootstrapopts', "--keyring=%s" % self.keyring] if self.components: command += ['--components', self.components] cmd = buildstep.RemoteShellCommand(self.getWorkdir(), command) stdio_log = stdio_log = self.addLog("pbuilder") cmd.useLog(stdio_log, True, "stdio") d = self.runCommand(cmd) self.step_status.setText(["PBuilder create."]) d.addCallback(lambda res: self.startBuild(cmd)) return d s = cmd.updates["stat"][-1] # basetgz will be a file when running in pbuilder # and a directory in case of cowbuilder if stat.S_ISREG(s[stat.ST_MODE]) or stat.S_ISDIR(s[stat.ST_MODE]): log.msg("%s found." % self.basetgz) age = time.time() - s[stat.ST_MTIME] if age >= self.maxAge: log.msg("basetgz outdated, updating") command = [ 'sudo', self.pbuilder, '--update', self.baseOption, self.basetgz ] cmd = buildstep.RemoteShellCommand(self.getWorkdir(), command) stdio_log = stdio_log = self.addLog("pbuilder") cmd.useLog(stdio_log, True, "stdio") d = self.runCommand(cmd) self.step_status.setText(["PBuilder update."]) d.addCallback(lambda res: self.startBuild(cmd)) return d return self.startBuild(cmd) else: log.msg("%s is not a file or a directory." % self.basetgz) self.finished(FAILURE)
def parseGotRevision(self, _): # if this was a full/export, then we need to check svnversion in the # *source* directory, not the build directory svnversion_dir = self.workdir if self.mode == 'full' and self.method == 'export': svnversion_dir = 'source' cmd = buildstep.RemoteShellCommand(svnversion_dir, ['svnversion'], env=self.env, logEnviron=self.logEnviron, collectStdout=True) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def _setrev(_): stdout = cmd.stdout.strip() revision = stdout.rstrip('MSP') revision = revision.split(':')[-1] try: int(revision) except ValueError: msg =("SVN.parseGotRevision unable to parse output " "of svnversion: '%s'" % stdout) log.msg(msg) raise buildstep.BuildStepFailed() log.msg("Got SVN revision %s" % (revision, )) self.setProperty('got_revision', revision, 'Source') return 0 d.addCallback(lambda _: _setrev(cmd.rc)) return d
def _dovccmd(self, command, collectStdout=False, initialStdin=None, successfulRC=(0, )): if not command: raise ValueError("No command specified") cmd = buildstep.RemoteShellCommand(self.workdir, ['hg', '--verbose'] + command, env=self.env, logEnviron=self.logEnviron, timeout=self.timeout, collectStdout=collectStdout, initialStdin=initialStdin, successfulRC=successfulRC) cmd.useLog(self.stdio_log, False) log.msg("Starting mercurial command : hg %s" % (" ".join(command), )) d = self.runCommand(cmd) def evaluateCommand(cmd): if cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: return cmd.stdout else: return cmd.rc d.addCallback(lambda _: evaluateCommand(cmd)) return d
def parseGotRevision(self, _): cmd = buildstep.RemoteShellCommand(self.workdir, ['svnversion'], env=self.env, logEnviron=self.logEnviron, collectStdout=True) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def _setrev(_): stdout = cmd.stdout.strip() revision = stdout.rstrip('MSP') revision = revision.split(':')[-1] try: int(revision) except ValueError: msg = ("SVN.parseGotRevision unable to parse output " "of svnversion: '%s'" % stdout) log.msg(msg) raise buildstep.BuildStepFailed() log.msg("Got SVN revision %s" % (revision, )) self.setProperty('got_revision', revision, 'Source') return 0 d.addCallback(lambda _: _setrev(cmd.rc)) return d
def checkP4(self): cmd = buildstep.RemoteShellCommand(self.workdir, ['p4', '-V'], env=self.env, logEnviron=self.logEnviron) cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) return cmd.rc == 0
def _Cmd(self, command, abandonOnFailure=True, workdir=None, **kwargs): if workdir is None: workdir = self.workdir cmd = buildstep.RemoteShellCommand(workdir, command, env=self.env, logEnviron=self.logEnviron, timeout=self.timeout, **kwargs) self.lastCommand = cmd # does not make sense to logEnviron for each command (just for first) self.logEnviron = False cmd.useLog(self.stdio_log, False) self.stdio_log.addHeader("Starting command: {}\n".format( " ".join(command))) self.step_status.setText(["{}".format(" ".join(command[:2]))]) yield self.runCommand(cmd) if abandonOnFailure and cmd.didFail(): self.descriptionDone = "repo failed at: {}".format(" ".join( command[:2])) self.stdio_log.addStderr( ("Source step failed while running command {}\n").format(cmd)) raise buildstep.BuildStepFailed() return cmd.rc
def parseGotRevision(self, _): # if this was a full/export, then we need to check svnversion in the # *source* directory, not the build directory svnversion_dir = self.workdir if self.mode == 'full' and self.method == 'export': svnversion_dir = 'source' cmd = buildstep.RemoteShellCommand(svnversion_dir, ['svn', 'info'], env=self.env, logEnviron=self.logEnviron, timeout=self.timeout, collectStdout=True) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def _setrev(_): stdout = cmd.stdout match = re.search('Revision:(.+?)\n', stdout) try: revision = int(match.group(1)) except (AttributeError, ValueError): msg =("SVN.parseGotRevision unable to parse output " "of svnversion: '%s'" % stdout) log.msg(msg) raise buildstep.BuildStepFailed() log.msg("Got SVN revision %s" % (revision, )) self.updateSourceProperty('got_revision', revision) return 0 d.addCallback(lambda _: _setrev(cmd.rc)) return d
def start(self): if self.autoRelease: relfile = '%s.release' % (os.path.basename( self.specfile).split('.')[0]) try: with open(relfile, 'r') as rfile: rel = int(rfile.readline().strip()) except (IOError, TypeError, ValueError): rel = 0 self.rpmbuild = self.rpmbuild + ' --define "_release %s"' % rel with open(relfile, 'w') as rfile: rfile.write(str(rel + 1)) if self.vcsRevision: revision = self.getProperty('got_revision') # only do this in the case where there's a single codebase if revision and not isinstance(revision, dict): self.rpmbuild = (self.rpmbuild + ' --define "_revision %s"' % revision) self.rpmbuild = self.rpmbuild + ' -ba %s' % self.specfile self.command = self.rpmbuild # create the actual RemoteShellCommand instance now kwargs = self.remote_kwargs kwargs['command'] = self.command kwargs['workdir'] = self.workdir cmd = buildstep.RemoteShellCommand(**kwargs) self.setupEnvironment(cmd) self.startCommand(cmd) self.addLogObserver( 'stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
def _dovccmd(self, command, abandonOnFailure=True, collectStdout=False, initialStdin=None): full_command = ['git'] if self.config is not None: for name, value in self.config.iteritems(): full_command.append('-c') full_command.append('%s=%s' % (name, value)) full_command.extend(command) cmd = buildstep.RemoteShellCommand(self.workdir, full_command, env=self.env, logEnviron=self.logEnviron, timeout=self.timeout, collectStdout=collectStdout, initialStdin=initialStdin) cmd.useLog(self.stdio_log, False) log.msg("Starting git command : git %s" % (" ".join(command), )) d = self.runCommand(cmd) def evaluateCommand(cmd): if abandonOnFailure and cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: return cmd.stdout else: return cmd.rc d.addCallback(lambda _: evaluateCommand(cmd)) return d
def _dovccmd(self, command, workdir, collectStdout=False, initialStdin=None, decodeRC=None, abandonOnFailure=True): if not command: raise ValueError("No command specified") if decodeRC is None: decodeRC = {0: SUCCESS} cmd = buildstep.RemoteShellCommand(workdir, command, env=self.env, logEnviron=self.logEnviron, timeout=self.timeout, collectStdout=collectStdout, initialStdin=initialStdin, decodeRC=decodeRC) cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) if abandonOnFailure and cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: defer.returnValue(cmd.stdout) else: defer.returnValue(cmd.rc)
def _dovccmd(self, command, collectStdout=False, initialStdin=None): command = self._buildVCCommand(command) if self.debug: log.msg("P4:_dovccmd():workdir->{}".format(self.workdir)) cmd = buildstep.RemoteShellCommand( self.workdir, command, env=self.env, logEnviron=self.logEnviron, timeout=self.timeout, collectStdout=collectStdout, initialStdin=initialStdin, ) cmd.useLog(self.stdio_log, False) if self.debug: log.msg("Starting p4 command : p4 {}".format(" ".join(command))) yield self.runCommand(cmd) if cmd.rc != 0: if self.debug: log.msg( "P4:_dovccmd():Source step failed while running command {}" .format(cmd)) raise buildstep.BuildStepFailed() if collectStdout: return cmd.stdout return cmd.rc
def parseGotRevision(self): command = self._buildVCCommand(['changes', '-m1', '#have']) cmd = buildstep.RemoteShellCommand(self.workdir, command, env=self.env, timeout=self.timeout, logEnviron=self.logEnviron, collectStdout=True) cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) stdout = cmd.stdout.strip() # Example output from p4 changes -m1 #have # Change 212798 on 2012/04/13 by user@user-unix-bldng2 'change to # pickup build' revision = stdout.split()[1] try: int(revision) except ValueError as e: msg = (("p4.parseGotRevision unable to parse output " "of 'p4 changes -m1 \"#have\"': '{}'").format(stdout)) log.msg(msg) raise buildstep.BuildStepFailed() from e if self.debug: log.msg("Got p4 revision {}".format(revision)) self.updateSourceProperty('got_revision', revision)
def _dovccmd(self, command, collectStdout=False, initialStdin=None): command = self._buildVCCommand(command) if debug_logging: log.msg("P4:_dovccmd():workdir->%s" % self.workdir) cmd = buildstep.RemoteShellCommand( self.workdir, command, env=self.env, logEnviron=self.logEnviron, timeout=self.timeout, collectStdout=collectStdout, initialStdin=initialStdin, ) cmd.useLog(self.stdio_log, False) if debug_logging: log.msg("Starting p4 command : p4 %s" % (" ".join(command), )) d = self.runCommand(cmd) def evaluateCommand(cmd): if cmd.rc != 0: if debug_logging: log.msg( "P4:_dovccmd():Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: return cmd.stdout else: return cmd.rc d.addCallback(lambda _: evaluateCommand(cmd)) return d
def parseGotRevision(self, _): command = self._buildVCCommand(['changes', '-m1', '#have']) cmd = buildstep.RemoteShellCommand(self.workdir, command, env=self.env, timeout=self.timeout, logEnviron=self.logEnviron, collectStdout=True) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def _setrev(_): stdout = cmd.stdout.strip() # Example output from p4 changes -m1 #have # Change 212798 on 2012/04/13 by user@user-unix-bldng2 'change to pickup build' revision = stdout.split()[1] try: int(revision) except ValueError: msg = ("p4.parseGotRevision unable to parse output " "of 'p4 changes -m1 \"#have\"': '%s'" % stdout) log.msg(msg) raise buildstep.BuildStepFailed() if debug_logging: log.msg("Got p4 revision %s" % (revision, )) self.updateSourceProperty('got_revision', revision) return 0 d.addCallback(lambda _: _setrev(cmd.rc)) return d
def _Cmd(self, command, abandonOnFailure=True, workdir=None, **kwargs): if workdir is None: workdir = self.workdir self.cmd = cmd = buildstep.RemoteShellCommand( workdir, command, env=self.env, logEnviron=self.logEnviron, timeout=self.timeout, **kwargs) # does not make sense to logEnviron for each command (just for first) self.logEnviron = False cmd.useLog(self.stdio_log, False) self.stdio_log.addHeader("Starting command: %s\n" % (" ".join(command), )) self.step_status.setText(["%s" % (" ".join(command[:2]))]) d = self.runCommand(cmd) def evaluateCommand(cmd): if abandonOnFailure and cmd.didFail(): self.step_status.setText( ["repo failed at: %s" % (" ".join(command[:2]))]) self.stdio_log.addStderr( "Source step failed while running command %s\n" % cmd) raise buildstep.BuildStepFailed() return cmd.rc d.addCallback(lambda _: evaluateCommand(cmd)) return d
def _dovccmd(self, command, collectStdout=False, initialStdin=None, decodeRC={0: SUCCESS}, abandonOnFailure=True, wkdir=None): if not command: raise ValueError("No command specified") workdir = wkdir or self.workdir cmd = buildstep.RemoteShellCommand(workdir, command, env=self.env, logEnviron=self.logEnviron, timeout=self.timeout, collectStdout=collectStdout, initialStdin=initialStdin, decodeRC=decodeRC) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def evaluateCommand(cmd): if abandonOnFailure and cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: return cmd.stdout else: return cmd.rc d.addCallback(lambda _: evaluateCommand(cmd)) return d
def _dovccmd(self, command, collectStdout=False): assert command, "No command specified" command.extend(['--non-interactive', '--no-auth-cache']) if self.username: command.extend(['--username', self.username]) if self.password: command.extend(['--password', self.password]) if self.depth: command.extend(['--depth', self.depth]) if self.extra_args: command.extend(self.extra_args) cmd = buildstep.RemoteShellCommand(self.workdir, ['svn'] + command, env=self.env, logEnviron=self.logEnviron, timeout=self.timeout, collectStdout=collectStdout) cmd.useLog(self.stdio_log, False) log.msg("Starting SVN command : svn %s" % (" ".join(command), )) d = self.runCommand(cmd) def evaluateCommand(cmd): if cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: return cmd.stdout else: return cmd.rc d.addCallback(lambda _: evaluateCommand(cmd)) return d
def _dovccmd(self, command, abandonOnFailure=True, collectStdout=False, initialStdin=None): cmd = buildstep.RemoteShellCommand(self.workdir, ['git'] + command, env=self.env, logEnviron=self.logEnviron, timeout=self.timeout, collectStdout=collectStdout, initialStdin=initialStdin) cmd.useLog(self.stdio_log, False) log.msg("Starting git command : git %s" % (" ".join(command), )) d = self.runCommand(cmd) def evaluateCommand(cmd): if abandonOnFailure and cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: return cmd.stdout else: return cmd.rc d.addCallback(lambda _: evaluateCommand(cmd)) return d
def start(self): if self.autoRelease: relfile = '%s.release' % (os.path.basename( self.specfile).split('.')[0]) try: with open(relfile, 'r') as rfile: rel = int(rfile.readline().strip()) except: rel = 0 self.rpmbuild = self.rpmbuild + ' --define "_release %s"' % rel with open(relfile, 'w') as rfile: rfile.write(str(rel + 1)) if self.vcsRevision: self.rpmbuild = self.rpmbuild + ' --define "_revision %s"' % \ self.getProperty('got_revision') self.rpmbuild = self.rpmbuild + ' -ba %s' % self.specfile self.command = self.rpmbuild # create the actual RemoteShellCommand instance now kwargs = self.remote_kwargs kwargs['command'] = self.command cmd = buildstep.RemoteShellCommand(**kwargs) self.setupEnvironment(cmd) self.startCommand(cmd)
def copy(self): yield self.runRmdir(self.workdir) # temporarily set workdir = 'source' and do an incremental checkout try: old_workdir = self.workdir self.workdir = 'source' yield self.incremental(None) except: # finally doesn't work in python-2.4 self.workdir = old_workdir raise self.workdir = old_workdir # if we're copying, copy; otherwise, export from source to build if self.method == 'copy': cmd = buildstep.RemoteCommand('cpdir', { 'fromdir': 'source', 'todir':self.workdir, 'logEnviron': self.logEnviron }) else: export_cmd = ['svn', 'export'] if self.revision: export_cmd.extend(["--revision", str(self.revision)]) export_cmd.extend(['source', self.workdir]) cmd = buildstep.RemoteShellCommand('', export_cmd, env=self.env, logEnviron=self.logEnviron, timeout=self.timeout) cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) if cmd.didFail(): raise buildstep.BuildStepFailed()
def test_runCommand(self): bs = buildstep.BuildStep() bs.buildslave = slave.FakeSlave() bs.remote = 'dummy' cmd = buildstep.RemoteShellCommand("build", ["echo", "hello"]) cmd.run = lambda self, remote: SUCCESS bs.runCommand(cmd) self.assertEqual(bs.cmd, cmd)
def checkSvn(self): cmd = buildstep.RemoteShellCommand(self.workdir, ['svn', '--version'], env=self.env, logEnviron=self.logEnviron) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def evaluate(cmd): if cmd.rc != 0: return False return True d.addCallback(lambda _: evaluate(cmd)) return d
def getStepConfig(self): log = self.addLog(".travis.yml") cmd = self.cmd = buildstep.RemoteShellCommand(workdir="build", command=["cat", ".travis.yml"]) cmd.useLog(log, False, "stdio") yield self.runCommand(cmd) self.cmd = None if cmd.rc != 0: raise buildstep.BuildStepFailed() config = TravisYml() config.parse(log.getText()) defer.returnValue(config)
def checkP4(self): cmd = buildstep.RemoteShellCommand(self.workdir, ['p4', '-V'], env=self.env, logEnviron=self.logEnviron) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) @d.addCallback def evaluate(_): if cmd.rc != 0: return False return True return d
def start(self): # this block is specific to ShellCommands. subclasses that don't need # to set up an argv array, an environment, or extra logfiles= (like # the Source subclasses) can just skip straight to startCommand() warnings = [] # create the actual RemoteShellCommand instance now kwargs = self.buildCommandKwargs(warnings) cmd = buildstep.RemoteShellCommand(**kwargs) self.setupEnvironment(cmd) self.startCommand(cmd, warnings)
def parseGotRevision(self, _): # if this was a full/export, then we need to check svnversion in the # *source* directory, not the build directory svnversion_dir = self.workdir if self.mode == 'full' and self.method == 'export': svnversion_dir = 'source' cmd = buildstep.RemoteShellCommand(svnversion_dir, ['svn', 'info', '--xml'], env=self.env, logEnviron=self.logEnviron, timeout=self.timeout, collectStdout=True) cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) stdout = cmd.stdout try: stdout_xml = xml.dom.minidom.parseString(stdout) except xml.parsers.expat.ExpatError: msg = "Corrupted xml, aborting step" self.stdio_log.addHeader(msg) raise buildstep.BuildStepFailed() revision = None if self.preferLastChangedRev: try: revision = stdout_xml.getElementsByTagName( 'commit')[0].attributes['revision'].value except (KeyError, IndexError): msg = ( "SVN.parseGotRevision unable to detect Last Changed Rev in" " output of svn info") log.msg(msg) # fall through and try to get 'Revision' instead if revision is None: try: revision = stdout_xml.getElementsByTagName( 'entry')[0].attributes['revision'].value except (KeyError, IndexError): msg = ("SVN.parseGotRevision unable to detect revision in" " output of svn info") log.msg(msg) raise buildstep.BuildStepFailed() msg = "Got SVN revision %s" % (revision, ) self.stdio_log.addHeader(msg) self.updateSourceProperty('got_revision', revision) defer.returnValue(cmd.rc)
def startVC(self, branch, revision, patch): command = ['jhbuild'] if (self.moduleset is not None): command += ['--moduleset=' + self.moduleset] command += ['bot', '--step', 'updateone', self.module] properties = self.build.getProperties() kwargs = {} kwargs['workdir'] = './' #kwargs = properties.render(self.remote_kwargs) kwargs['command'] = properties.render(command) kwargs['env'] = {} kwargs['timeout'] = 60 * 60 cmd = buildstep.RemoteShellCommand(**kwargs) self.startCommand(cmd)
def _dovccmd(self, command, abandonOnFailure=True): cmd = buildstep.RemoteShellCommand(self.workdir, ['git'] + command) cmd.useLog(self.stdio_log, False) log.msg("Starting git command : git %s" % (" ".join(command), )) d = self.runCommand(cmd) def evaluateCommand(cmd): if abandonOnFailure and cmd.rc != 0: log.msg("Source step failed while running command %s" % cmd) raise failure.Failure(cmd.rc) return cmd.rc d.addCallback(lambda _: evaluateCommand(cmd)) return d