def __init__(self, branch, clobber_url, clobberer_path, clobberTime=None, timeout=3600, workdir='..', command=[], **kwargs): command = ['python', clobberer_path, '-s', 'tools'] if clobberTime: command.extend(['-t', str(clobberTime)]) command.extend([ clobber_url, branch, WithProperties("%(buildername)s"), WithProperties("%(slavebuilddir)s"), WithProperties("%(slavename)s"), WithProperties("%(master)s"), ]) self.super_class = ShellCommand self.super_class.__init__(self, command=command, timeout=timeout, workdir=workdir, **kwargs) self.addFactoryArguments(branch=branch, clobber_url=clobber_url, clobberer_path=clobberer_path, clobberTime=clobberTime)
def GetCompilerArtifacts(f): f.addStep( buildbot.steps.shell.ShellCommand( name='rm.host-compiler', command=['rm', '-rfv', 'host-compiler', 'host-compiler.tar.gz'], haltOnFailure=False, description=['rm', 'host-compiler'], workdir=WithProperties('%(builddir)s'))) f.addStep( buildbot.steps.shell.ShellCommand( name='download.artifacts', command=[ 'curl', curl_flags, 'host-compiler.tar.gz', WithProperties('%(get_curl)s', get_curl=determine_url) ], haltOnFailure=True, description=['download build artifacts'], workdir=WithProperties('%(builddir)s'))) f.addStep( buildbot.steps.shell.ShellCommand( name='unzip', command=['tar', '-zxvf', '../host-compiler.tar.gz'], haltOnFailure=True, description=['extract', 'host-compiler'], workdir='host-compiler')) return f
def GetCompilerRoot(f): # The following steps are used to retrieve a compiler archive # clean out any existing archives f.addStep( buildbot.steps.shell.ShellCommand( name='rm.host-compiler', command=['rm', '-rfv', 'host-compiler', 'host-compiler.tar.gz'], haltOnFailure=False, description=['rm', 'host-compiler'], workdir=WithProperties('%(builddir)s'))) setProperty( f, 'rootURL', WithProperties(base_download_url + '/%(getpath)s/%(getname)s', getpath=_determine_compiler_path, getname=_determine_archive_name)) # curl down the archive f.addStep( buildbot.steps.shell.ShellCommand( name='download.artifacts', command=[ 'curl', curl_flags, 'host-compiler.tar.gz', WithProperties('%(rootURL)s') ], haltOnFailure=True, description=['download build artifacts'], workdir=WithProperties('%(builddir)s'))) # extract the compiler root from the archive f.addStep( buildbot.steps.shell.ShellCommand( name='unzip', command=['tar', '-zxvf', '../host-compiler.tar.gz'], haltOnFailure=True, description=['extract', 'host-compiler'], workdir='host-compiler')) return f
def testSourceStamp(self): c = ShellCommand(workdir=dir, command=[ "touch", WithProperties("%s-dir", "branch"), WithProperties("%s-rev", "revision"), ]) c.setBuild(self.build) cmd = c._interpolateProperties(c.command) self.failUnlessEqual(cmd, ["touch", "branch2-dir", "1234-rev"])
def testBuildNumber(self): c = ShellCommand(workdir=dir, command=[ "touch", WithProperties("build-%d", "buildnumber"), WithProperties("builder-%s", "buildername"), ]) c.setBuild(self.build) cmd = c._interpolateProperties(c.command) self.failUnlessEqual(cmd, ["touch", "build-5", "builder-fakebuilder"])
def __init__(self, MAKE, *args, **kwargs): ShellCommand.__init__(self, *args, **kwargs) self.addFactoryArguments(MAKE=MAKE) self.command = [ MAKE, "upload-coverage", WithProperties("UPLOAD_TARGET=%s" % self.UPLOAD_TARGET, "buildnumber"), "UPLOAD_HOST=%s" % self.UPLOAD_HOST, WithProperties("COVERAGEDIR=%s" % self.COVERAGEDIR, "buildnumber"), ]
def addGetSourcecodeForProject(self, project, name=None, src_dir=None, **kwargs): # Remove 'is_legacy_mode' if it leaked in to kwargs. kwargs.pop('is_legacy_mode', None) # Bail out if we are in the legacy mode and SVN checkout is required. if self.is_legacy_mode: workdir, baseURL = svn_repos[project] if not name: name = 'svn-%s' % project # Check out to the given directory if any. # Otherwise this is a part of the unified source tree. if src_dir is None: src_dir = workdir % {'llvm_srcdir': self.llvm_srcdir} if not kwargs.get('mode', None): kwargs['mode'] = 'update' if not kwargs.get('defaultBranch', None): kwargs['defaultBranch'] = 'trunk' self.addStep( SVN(name=name, workdir=src_dir, baseURL=WithProperties(baseURL), **kwargs)) else: # project contains a repo name which is not a part of the monorepo. # We do not enforce it here, though. _repourl = kwargs.pop('repourl', None) if not _repourl: _repourl = self.repourl_prefix + "llvm-%s.git" % project if not name: name = 'Checkout %s' % project # Check out to the given directory if any. # Otherwise this is a part of the unified source tree. if src_dir is None: src_dir = 'llvm-%s' % project # Ignore workdir if given. We check out to src_dir. kwargs.pop('workdir', None) self.addStep( Git(name=name, repourl=_repourl, progress=True, workdir=WithProperties(src_dir), **kwargs))
def __init__(self, variant, symbols_path=None, testPath=None, xrePath='../hostutils/xre', testManifest=None, utilityPath='../hostutils/bin', certificatePath='certs', app='org.mozilla.fennec', consoleLevel='INFO', totalChunks=None, thisChunk=None, **kwargs): self.super_class = ShellCommandReportTimeout ShellCommandReportTimeout.__init__(self, **kwargs) if totalChunks: assert 1 <= thisChunk <= totalChunks self.addFactoryArguments(variant=variant, symbols_path=symbols_path, testPath=testPath, xrePath=xrePath, testManifest=testManifest, utilityPath=utilityPath, certificatePath=certificatePath, app=app, consoleLevel=consoleLevel, totalChunks=totalChunks, thisChunk=thisChunk) self.name = 'mochitest-%s' % variant self.command = [ 'python', 'mochitest/runtestsremote.py', '--deviceIP', WithProperties('%(sut_ip)s'), '--xre-path', xrePath, '--utility-path', utilityPath, '--certificate-path', certificatePath, '--app', app, '--console-level', consoleLevel, '--http-port', WithProperties('%(http_port)s'), '--ssl-port', WithProperties('%(ssl_port)s'), '--pidfile', WithProperties('%(basedir)s/../runtestsremote.pid') ] self.command.extend(self.getVariantOptions(variant)) if testPath: self.command.extend(['--test-path', testPath]) if testManifest: self.command.extend(['--run-only-tests', testManifest]) if symbols_path: self.command.append( WithProperties("--symbols-path=../%s" % symbols_path)) self.command.extend(self.getChunkOptions(totalChunks, thisChunk))
def __init__(self, variant='plain', symbols_path=None, leakThreshold=None, chunkByDir=None, totalChunks=None, thisChunk=None, testPath=None, **kwargs): self.super_class = ShellCommandReportTimeout ShellCommandReportTimeout.__init__(self, **kwargs) if totalChunks: assert 1 <= thisChunk <= totalChunks self.addFactoryArguments(variant=variant, symbols_path=symbols_path, leakThreshold=leakThreshold, chunkByDir=chunkByDir, totalChunks=totalChunks, thisChunk=thisChunk, testPath=testPath) if totalChunks: self.name = 'mochitest-%s-%i' % (variant, thisChunk) else: self.name = 'mochitest-%s' % variant self.command = [ 'python', 'mochitest/runtests.py', WithProperties('--appname=%(exepath)s'), '--utility-path=bin', WithProperties('--extra-profile-file=bin/plugins'), '--certificate-path=certs', '--autorun', '--close-when-done', '--console-level=INFO' ] if testPath: self.command.append("--test-path=%s" % testPath) if symbols_path: self.command.append( WithProperties("--symbols-path=%s" % symbols_path)) if leakThreshold: self.command.append('--leak-threshold=%d' % leakThreshold) self.command.extend( self.getChunkOptions(totalChunks, thisChunk, chunkByDir)) self.command.extend(self.getVariantOptions(variant))
def __init__(self, platform, symbols_path=None, **kwargs): self.super_class = ShellCommandReportTimeout ShellCommandReportTimeout.__init__(self, **kwargs) self.addFactoryArguments(platform=platform, symbols_path=symbols_path) bin_extension = "" if platform.startswith('win'): bin_extension = ".exe" script = " && ".join(["if [ ! -d %(xredir)s/plugins ]; then mkdir %(xredir)s/plugins; fi", "if [ ! -d %(xredir)s/components ]; then mkdir %(xredir)s/components; fi", "if [ ! -d %(xredir)s/extensions ]; then mkdir %(xredir)s/extensions; fi", "cp bin/xpcshell" + bin_extension + " %(exedir)s", "cp bin/ssltunnel" + bin_extension + " %(exedir)s", "cp -R bin/components/* %(xredir)s/components/", "cp -R bin/plugins/* %(xredir)s/plugins/", "if [ -d extensions ]; then cp -R extensions/* %(xredir)s/extensions/; fi", "python -u xpcshell/runxpcshelltests.py"]) if symbols_path: script += " --symbols-path=%s" % symbols_path script += " --manifest=xpcshell/tests/all-test-dirs.list %(exedir)s/xpcshell" + bin_extension self.command = ['bash', '-c', WithProperties(script)]
def addGetSourcecodeForProject(self, project, name=None, src_dir=None, **kwargs): # project contains a repo name which is not a part of the monorepo. # We do not enforce it here, though. _repourl = kwargs.pop('repourl', None) if not _repourl: _repourl = self.repourl_prefix + "llvm-%s.git" % project if not name: name = 'Checkout %s' % project # Check out to the given directory if any. # Otherwise this is a part of the unified source tree. if src_dir is None: src_dir = 'llvm-%s' % project # Ignore workdir if given. We check out to src_dir. kwargs.pop('workdir', None) self.addStep( Git(name=name, repourl=_repourl, progress=True, workdir=WithProperties(src_dir), **kwargs))
def __init__(self, suite, symbols_path=None, leakThreshold=None, **kwargs): self.super_class = ShellCommandReportTimeout ShellCommandReportTimeout.__init__(self, **kwargs) self.addFactoryArguments(suite=suite, symbols_path=symbols_path, leakThreshold=leakThreshold) self.name = suite self.command = [ 'python', WithProperties("%(toolsdir)s/buildfarm/utils/run_jetpack.py"), '-p', WithProperties("%(platform)s") ]
def start(self): try: props = self.build.getProperties() branch = props.render(self.branch) revision = props.render(self.revision) comments = props.render(self.comments) files = props.render(self.files) user = props.render(self.user) sendchange_props = [] for key, value in self.sendchange_props.iteritems(): sendchange_props.append((key, props.render(value))) self.addCompleteLog( "sendchange", """\ master: %s branch: %s revision: %s comments: %s user: %s files: %s properties: %s""" % (self.master, branch, revision, comments, user, files, sendchange_props)) bb_cmd = [ 'buildbot', 'sendchange', '--master', self.master, '--username', user, '--branch', branch, '--revision', revision ] if isinstance(comments, basestring): if re.search('try: ', comments, re.MULTILINE): comments = 'try: ' + ' '.join(processMessage(comments)) else: try: comments = comments.splitlines()[0] except IndexError: comments = '' comments = re.sub(r'[\r\n^<>|;&"\'%$]', '_', comments) comments = comments.encode('ascii', 'replace') if comments: bb_cmd.extend(['--comments', comments]) for key, value in sendchange_props: bb_cmd.extend(['--property', '%s:%s' % (key, value)]) if files: bb_cmd.extend(self.files) cmd = [ 'python', WithProperties("%(toolsdir)s/buildfarm/utils/retry.py"), '-s', str(self.sleepTime), '-t', str(self.timeout), '-r', str(self.retries), '--stdout-regexp', 'change sent successfully' ] cmd.extend(bb_cmd) self.setCommand(cmd) self.super_class.start(self) except KeyError: self.addCompleteLog("errors", str(Failure())) return self.finished(FAILURE)
def addGetSourcecodeSteps(self, **kwargs): # Checkout the monorepo. self.addStep( Git(name='Checkout the source code', repourl=self.repourl_prefix + "llvm-project.git", progress=True, workdir=WithProperties(self.monorepo_dir), **kwargs))
def testWorkdir(self): self.build.setProperty("revision", 47) self.failUnlessEqual(self.build_status.getProperty("revision"), 47) c = ShellCommand(command=["tar", "czf", "foo.tar.gz", "source"]) c.setBuild(self.build) workdir = WithProperties("workdir-%d", "revision") workdir = c._interpolateWorkdir(workdir) self.failUnlessEqual(workdir, "workdir-47")
def testSlaveName(self): c = ShellCommand(workdir=dir, command=[ "touch", WithProperties("%s-slave", "slavename"), ]) c.setBuild(self.build) cmd = c._interpolateProperties(c.command) self.failUnlessEqual(cmd, ["touch", "bot12-slave"])
def addUnpackBuildSteps(self): self.addStep(UnpackFile( filename=WithProperties("%(filename)s"), workdir=self.base_dir, haltOnFailure=True, timeout=60*60, name="unpack_build", description=['unpack', 'build'], ))
def set_mock_command(self): if self.properties_rendered: return self.properties_rendered = True # We need to have all commands as a string. We'll # convert argv commands into string commands if isinstance(self.command, list): string_list = [] for arg in self.command: if issubclass(arg.__class__, WithProperties): string_list.append(quote(str(arg.fmtstring))) else: string_list.append(quote(str(arg))) string_command = ' '.join(string_list) elif issubclass(self.command.__class__, WithProperties): string_command = self.command.fmtstring else: string_command = self.command mock_workdir = self.mock_workdir_mutator( self.remote_kwargs['workdir']) # If the workdir is a WithProperties instance, we need to get the # format string and wrap it in another WithProperties if issubclass(mock_workdir.__class__, WithProperties): mock_workdir = mock_workdir.fmtstring if self.mock_workdir_prefix is not None: mock_workdir = self.mock_workdir_prefix + mock_workdir if 'env' in self.remote_kwargs: pre_render_env = self.remote_kwargs['env'] properties = self.build.getProperties() rendered_env = properties.render(pre_render_env) environment = ' '.join('%s="%s"' % (k, rendered_env[k]) for k in rendered_env.keys()) else: environment = '' self.command = [self.mock_login, '-r', self.target, '--cwd', WithProperties(mock_workdir)] + \ self.mock_args + ['--shell'] + \ [WithProperties('/usr/bin/env %s %s' % (environment, string_command))]
def testWithPropertiesEmpty(self): self.build.setProperty("empty", None) c = ShellCommand(workdir=dir, command=[ "tar", "czf", WithProperties("build-%(empty)s.tar.gz"), "source" ]) c.setBuild(self.build) cmd = c._interpolateProperties(c.command) self.failUnlessEqual(cmd, ["tar", "czf", "build-.tar.gz", "source"])
def GetLatestValidated(f): import zorg.buildbot.util.artifacts as artifacts f.addStep(buildbot.steps.shell.ShellCommand( name='rm.host-compiler', command=['rm', '-rfv', 'host-compiler', 'host-compiler.tar.gz'], haltOnFailure=False, description=['rm', 'host-compiler'], workdir=WithProperties('%(builddir)s'))) latest_url = artifacts.base_download_url latest_url += '/validated_builds/clang-x86_64-darwin11-R.tar.gz' f.addStep(buildbot.steps.shell.ShellCommand( name='download.artifacts', command=['curl', '-fvLo', 'host-compiler.tar.gz', latest_url], haltOnFailure=True, description=['download build artifacts'], workdir=WithProperties('%(builddir)s'))) f.addStep(buildbot.steps.shell.ShellCommand( name='unzip', command=['tar', '-zxvf','../host-compiler.tar.gz'], haltOnFailure=True, description=['extract', 'host-compiler'], workdir='host-compiler')) return f
def testWithPropertiesDict(self): self.build.setProperty("other", "foo") self.build.setProperty("missing", None) c = ShellCommand(workdir=dir, command=[ "tar", "czf", WithProperties("build-%(other)s.tar.gz"), "source" ]) c.setBuild(self.build) cmd = c._interpolateProperties(c.command) self.failUnlessEqual(cmd, ["tar", "czf", "build-foo.tar.gz", "source"])
def __init__(self, suite, symbols_path=None, xrePath='../hostutils/xre', utilityPath='../hostutils/bin', app='org.mozilla.fennec', totalChunks=None, thisChunk=None, cmdOptions=None, **kwargs): self.super_class = ShellCommandReportTimeout ShellCommandReportTimeout.__init__(self, **kwargs) self.addFactoryArguments(suite=suite, xrePath=xrePath, symbols_path=symbols_path, utilityPath=utilityPath, app=app, totalChunks=totalChunks, thisChunk=thisChunk, cmdOptions=cmdOptions) self.name = suite if totalChunks: self.name += '-%i' % thisChunk self.command = [ 'python', 'reftest/remotereftest.py', '--deviceIP', WithProperties('%(sut_ip)s'), '--xre-path', xrePath, '--utility-path', utilityPath, '--app', app, '--http-port', WithProperties('%(http_port)s'), '--ssl-port', WithProperties('%(ssl_port)s'), '--pidfile', WithProperties('%(basedir)s/../remotereftest.pid'), '--enable-privilege' ] if cmdOptions: self.command.extend(cmdOptions) self.command.extend(self.getChunkOptions(totalChunks, thisChunk)) self.command.extend(self.getSuiteOptions(suite)) if symbols_path: self.command.append( WithProperties("--symbols-path=../%s" % symbols_path))
def __init__(self, suite, symbols_path=None, leakThreshold=None, **kwargs): self.super_class = ShellCommandReportTimeout ShellCommandReportTimeout.__init__(self, **kwargs) self.addFactoryArguments(suite=suite, symbols_path=symbols_path, leakThreshold=leakThreshold) self.name = suite self.command = [ 'python', 'reftest/runreftest.py', WithProperties('--appname=%(exepath)s'), '--utility-path=bin', '--extra-profile-file=bin/plugins', ] if symbols_path: self.command.append( WithProperties("--symbols-path=%s" % symbols_path)) if leakThreshold: self.command.append('--leak-threshold=%d' % leakThreshold) self.command.extend(self.getSuiteOptions(suite))
def testWithProperties(self): self.build.setProperty("revision", 47) self.failUnlessEqual(self.build_status.getProperty("revision"), 47) c = ShellCommand(workdir=dir, command=[ "tar", "czf", WithProperties("build-%s.tar.gz", "revision"), "source" ]) c.setBuild(self.build) cmd = c._interpolateProperties(c.command) self.failUnlessEqual(cmd, ["tar", "czf", "build-47.tar.gz", "source"])
def _step_Archive(self): # make the tarball self.addStep( ShellCommand(name="makedist", description=["zipfile", "package"], workdir="build/build/root", command=[ "zip", "-r", "-9", "-v", WithProperties("../../%(filename)s"), ".", "-x", ".svn" ], haltOnFailure=True))
def addObtainBuildSteps(self): self.addDownloadBuildStep() self.addUnpackBuildSteps() #If people *hate* lambdas, i could define simple functions mozilla_fn = lambda x,y,z: parse_build_info(' '.join([y,z]), 'mozilla') mobile_fn = lambda x,y,z: parse_build_info(' '.join([y,z]), 'mobile') self.addStep(SetProperty( command=['cat', 'platform.ini'], workdir='%s/fennec' % self.base_dir, extract_fn=mozilla_fn, description=['get', 'mobile', 'revision'], name='mozilla_rev', )) self.addStep(SetProperty( command=['cat', 'application.ini'], workdir='%s/fennec' % self.base_dir, extract_fn=mobile_fn, description=['get', 'mobile', 'revision'], name='mobile_rev', )) self.addStep(SetProperty( command=['echo', WithProperties('%(mozilla_changeset)s:%(mobile_changeset)s')], property='got_revision', name='set_got_revision', description=['set_got_revision'] )) self.addStep(ShellCommand( command=['echo', 'TinderboxPrint:', WithProperties('<a href=%(mozilla_repository)s/rev/%(mozilla_changeset)s' + 'title="Built from Mozilla revision %(mozilla_changeset)s">' + 'moz:%(mozilla_changeset)s</a> <br />' + '<a href=%(mobile_repository)s/rev/%(mobile_changeset)s' + 'title="Built from Mobile revision %(mobile_changeset)s">' + 'mobile:%(mobile_changeset)s</a>')], description=['list', 'revisions'], name='rev_info', ))
def __init__(self, suite, symbols_path=None, leakThreshold=None, **kwargs): self.super_class = ShellCommandReportTimeout ShellCommandReportTimeout.__init__(self, **kwargs) self.addFactoryArguments(suite=suite, symbols_path=symbols_path, leakThreshold=leakThreshold) self.name = suite self.command = [ 'python', 'jetpack/bin/cfx', WithProperties('--binary=%(exepath)s'), '--parseable', suite ]
def addObtainBuildSteps(self): MobileTalosFactory.addObtainBuildSteps(self) self.addStep(SetProperty( command=['echo', WithProperties("%(fileURL)s")], extract_fn=download_dir, name='download_dir', description=['download', 'directory', 'property'], )) self.addStep(ShellCommand( command=['wget', WithProperties("%(download_dir)s/fennec-%(version)s.en-US.linux-gnueabi-arm.tests.zip"), '-O', 'fennec-tests.zip'], workdir=self.base_dir, haltOnFailure=True, name="get_tests", description=['get', 'unit', 'tests'], )) self.addStep(ShellCommand( command=['unzip', '%s/fennec-tests.zip' % self.base_dir], workdir="%s/fennec" % self.base_dir, name='unpack_test', description=['unpack', 'tests'], haltOnFailure=True, ))
def GetCXXFromCompilerArtifacts(f, base_dir): def get_cxx(status, stdin, stdout): lines = filter(bool, stdin.split('\n')) for line in lines: if 'bin/clang++' in line: cxx_path = line return {'cxx_path': cxx_path} return {} f.addStep( buildbot.steps.shell.SetProperty( name='find.cxx', command=['find', base_dir, '-name', 'clang++'], extract_fn=get_cxx, workdir=WithProperties('%(builddir)s'))) return f
def addGetSourcecodeSteps(self, **kwargs): # Remove 'is_legacy_mode' if it leaked in to kwargs. kwargs.pop('is_legacy_mode', None) # Bail out if we are in the legacy mode and SVN checkout is required. if self.is_legacy_mode: self.addSVNSteps(**kwargs) return # Checkout the monorepo. self.addStep( Git(name='Checkout the source code', repourl=self.repourl_prefix + "llvm-project.git", progress=True, workdir=WithProperties(self.monorepo_dir), **kwargs))