def __init__(self,
branch,
clobber_url,
clobberer_path,
clobberTime=None,
timeout=3600,
workdir='..',
command=[],
**kwargs):
command = ['python', clobberer_path, '-s', 'tools']
if clobberTime:
command.extend(['-t', str(clobberTime)])
command.extend([
clobber_url,
branch,
WithProperties("%(buildername)s"),
WithProperties("%(slavebuilddir)s"),
WithProperties("%(slavename)s"),
WithProperties("%(master)s"),
])
self.super_class = ShellCommand
self.super_class.__init__(self,
command=command,
timeout=timeout,
workdir=workdir,
**kwargs)
self.addFactoryArguments(branch=branch,
clobber_url=clobber_url,
clobberer_path=clobberer_path,
clobberTime=clobberTime)
def GetCompilerArtifacts(f):
f.addStep(
buildbot.steps.shell.ShellCommand(
name='rm.host-compiler',
command=['rm', '-rfv', 'host-compiler', 'host-compiler.tar.gz'],
haltOnFailure=False,
description=['rm', 'host-compiler'],
workdir=WithProperties('%(builddir)s')))
f.addStep(
buildbot.steps.shell.ShellCommand(
name='download.artifacts',
command=[
'curl', curl_flags, 'host-compiler.tar.gz',
WithProperties('%(get_curl)s', get_curl=determine_url)
],
haltOnFailure=True,
description=['download build artifacts'],
workdir=WithProperties('%(builddir)s')))
f.addStep(
buildbot.steps.shell.ShellCommand(
name='unzip',
command=['tar', '-zxvf', '../host-compiler.tar.gz'],
haltOnFailure=True,
description=['extract', 'host-compiler'],
workdir='host-compiler'))
return f
def GetCompilerRoot(f):
# The following steps are used to retrieve a compiler archive
# clean out any existing archives
f.addStep(
buildbot.steps.shell.ShellCommand(
name='rm.host-compiler',
command=['rm', '-rfv', 'host-compiler', 'host-compiler.tar.gz'],
haltOnFailure=False,
description=['rm', 'host-compiler'],
workdir=WithProperties('%(builddir)s')))
setProperty(
f, 'rootURL',
WithProperties(base_download_url + '/%(getpath)s/%(getname)s',
getpath=_determine_compiler_path,
getname=_determine_archive_name))
# curl down the archive
f.addStep(
buildbot.steps.shell.ShellCommand(
name='download.artifacts',
command=[
'curl', curl_flags, 'host-compiler.tar.gz',
WithProperties('%(rootURL)s')
],
haltOnFailure=True,
description=['download build artifacts'],
workdir=WithProperties('%(builddir)s')))
# extract the compiler root from the archive
f.addStep(
buildbot.steps.shell.ShellCommand(
name='unzip',
command=['tar', '-zxvf', '../host-compiler.tar.gz'],
haltOnFailure=True,
description=['extract', 'host-compiler'],
workdir='host-compiler'))
return f
def testSourceStamp(self):
c = ShellCommand(workdir=dir,
command=[
"touch",
WithProperties("%s-dir", "branch"),
WithProperties("%s-rev", "revision"),
])
c.setBuild(self.build)
cmd = c._interpolateProperties(c.command)
self.failUnlessEqual(cmd, ["touch", "branch2-dir", "1234-rev"])
def testBuildNumber(self):
c = ShellCommand(workdir=dir,
command=[
"touch",
WithProperties("build-%d", "buildnumber"),
WithProperties("builder-%s", "buildername"),
])
c.setBuild(self.build)
cmd = c._interpolateProperties(c.command)
self.failUnlessEqual(cmd, ["touch", "build-5", "builder-fakebuilder"])
def __init__(self, MAKE, *args, **kwargs):
ShellCommand.__init__(self, *args, **kwargs)
self.addFactoryArguments(MAKE=MAKE)
self.command = [
MAKE,
"upload-coverage",
WithProperties("UPLOAD_TARGET=%s" % self.UPLOAD_TARGET,
"buildnumber"),
"UPLOAD_HOST=%s" % self.UPLOAD_HOST,
WithProperties("COVERAGEDIR=%s" % self.COVERAGEDIR, "buildnumber"),
]
def addGetSourcecodeForProject(self,
project,
name=None,
src_dir=None,
**kwargs):
# Remove 'is_legacy_mode' if it leaked in to kwargs.
kwargs.pop('is_legacy_mode', None)
# Bail out if we are in the legacy mode and SVN checkout is required.
if self.is_legacy_mode:
workdir, baseURL = svn_repos[project]
if not name:
name = 'svn-%s' % project
# Check out to the given directory if any.
# Otherwise this is a part of the unified source tree.
if src_dir is None:
src_dir = workdir % {'llvm_srcdir': self.llvm_srcdir}
if not kwargs.get('mode', None):
kwargs['mode'] = 'update'
if not kwargs.get('defaultBranch', None):
kwargs['defaultBranch'] = 'trunk'
self.addStep(
SVN(name=name,
workdir=src_dir,
baseURL=WithProperties(baseURL),
**kwargs))
else:
# project contains a repo name which is not a part of the monorepo.
# We do not enforce it here, though.
_repourl = kwargs.pop('repourl', None)
if not _repourl:
_repourl = self.repourl_prefix + "llvm-%s.git" % project
if not name:
name = 'Checkout %s' % project
# Check out to the given directory if any.
# Otherwise this is a part of the unified source tree.
if src_dir is None:
src_dir = 'llvm-%s' % project
# Ignore workdir if given. We check out to src_dir.
kwargs.pop('workdir', None)
self.addStep(
Git(name=name,
repourl=_repourl,
progress=True,
workdir=WithProperties(src_dir),
**kwargs))
def __init__(self,
variant,
symbols_path=None,
testPath=None,
xrePath='../hostutils/xre',
testManifest=None,
utilityPath='../hostutils/bin',
certificatePath='certs',
app='org.mozilla.fennec',
consoleLevel='INFO',
totalChunks=None,
thisChunk=None,
**kwargs):
self.super_class = ShellCommandReportTimeout
ShellCommandReportTimeout.__init__(self, **kwargs)
if totalChunks:
assert 1 <= thisChunk <= totalChunks
self.addFactoryArguments(variant=variant,
symbols_path=symbols_path,
testPath=testPath,
xrePath=xrePath,
testManifest=testManifest,
utilityPath=utilityPath,
certificatePath=certificatePath,
app=app,
consoleLevel=consoleLevel,
totalChunks=totalChunks,
thisChunk=thisChunk)
self.name = 'mochitest-%s' % variant
self.command = [
'python', 'mochitest/runtestsremote.py', '--deviceIP',
WithProperties('%(sut_ip)s'), '--xre-path', xrePath,
'--utility-path', utilityPath, '--certificate-path',
certificatePath, '--app', app, '--console-level', consoleLevel,
'--http-port',
WithProperties('%(http_port)s'), '--ssl-port',
WithProperties('%(ssl_port)s'), '--pidfile',
WithProperties('%(basedir)s/../runtestsremote.pid')
]
self.command.extend(self.getVariantOptions(variant))
if testPath:
self.command.extend(['--test-path', testPath])
if testManifest:
self.command.extend(['--run-only-tests', testManifest])
if symbols_path:
self.command.append(
WithProperties("--symbols-path=../%s" % symbols_path))
self.command.extend(self.getChunkOptions(totalChunks, thisChunk))
def __init__(self,
variant='plain',
symbols_path=None,
leakThreshold=None,
chunkByDir=None,
totalChunks=None,
thisChunk=None,
testPath=None,
**kwargs):
self.super_class = ShellCommandReportTimeout
ShellCommandReportTimeout.__init__(self, **kwargs)
if totalChunks:
assert 1 <= thisChunk <= totalChunks
self.addFactoryArguments(variant=variant,
symbols_path=symbols_path,
leakThreshold=leakThreshold,
chunkByDir=chunkByDir,
totalChunks=totalChunks,
thisChunk=thisChunk,
testPath=testPath)
if totalChunks:
self.name = 'mochitest-%s-%i' % (variant, thisChunk)
else:
self.name = 'mochitest-%s' % variant
self.command = [
'python', 'mochitest/runtests.py',
WithProperties('--appname=%(exepath)s'), '--utility-path=bin',
WithProperties('--extra-profile-file=bin/plugins'),
'--certificate-path=certs', '--autorun', '--close-when-done',
'--console-level=INFO'
]
if testPath:
self.command.append("--test-path=%s" % testPath)
if symbols_path:
self.command.append(
WithProperties("--symbols-path=%s" % symbols_path))
if leakThreshold:
self.command.append('--leak-threshold=%d' % leakThreshold)
self.command.extend(
self.getChunkOptions(totalChunks, thisChunk, chunkByDir))
self.command.extend(self.getVariantOptions(variant))
def __init__(self, platform, symbols_path=None, **kwargs):
self.super_class = ShellCommandReportTimeout
ShellCommandReportTimeout.__init__(self, **kwargs)
self.addFactoryArguments(platform=platform, symbols_path=symbols_path)
bin_extension = ""
if platform.startswith('win'):
bin_extension = ".exe"
script = " && ".join(["if [ ! -d %(xredir)s/plugins ]; then mkdir %(xredir)s/plugins; fi",
"if [ ! -d %(xredir)s/components ]; then mkdir %(xredir)s/components; fi",
"if [ ! -d %(xredir)s/extensions ]; then mkdir %(xredir)s/extensions; fi",
"cp bin/xpcshell" +
bin_extension + " %(exedir)s",
"cp bin/ssltunnel" +
bin_extension + " %(exedir)s",
"cp -R bin/components/* %(xredir)s/components/",
"cp -R bin/plugins/* %(xredir)s/plugins/",
"if [ -d extensions ]; then cp -R extensions/* %(xredir)s/extensions/; fi",
"python -u xpcshell/runxpcshelltests.py"])
if symbols_path:
script += " --symbols-path=%s" % symbols_path
script += " --manifest=xpcshell/tests/all-test-dirs.list %(exedir)s/xpcshell" + bin_extension
self.command = ['bash', '-c', WithProperties(script)]
def addGetSourcecodeForProject(self,
project,
name=None,
src_dir=None,
**kwargs):
# project contains a repo name which is not a part of the monorepo.
# We do not enforce it here, though.
_repourl = kwargs.pop('repourl', None)
if not _repourl:
_repourl = self.repourl_prefix + "llvm-%s.git" % project
if not name:
name = 'Checkout %s' % project
# Check out to the given directory if any.
# Otherwise this is a part of the unified source tree.
if src_dir is None:
src_dir = 'llvm-%s' % project
# Ignore workdir if given. We check out to src_dir.
kwargs.pop('workdir', None)
self.addStep(
Git(name=name,
repourl=_repourl,
progress=True,
workdir=WithProperties(src_dir),
**kwargs))
def __init__(self, suite, symbols_path=None, leakThreshold=None, **kwargs):
self.super_class = ShellCommandReportTimeout
ShellCommandReportTimeout.__init__(self, **kwargs)
self.addFactoryArguments(suite=suite,
symbols_path=symbols_path,
leakThreshold=leakThreshold)
self.name = suite
self.command = [
'python',
WithProperties("%(toolsdir)s/buildfarm/utils/run_jetpack.py"),
'-p',
WithProperties("%(platform)s")
]
def start(self):
try:
props = self.build.getProperties()
branch = props.render(self.branch)
revision = props.render(self.revision)
comments = props.render(self.comments)
files = props.render(self.files)
user = props.render(self.user)
sendchange_props = []
for key, value in self.sendchange_props.iteritems():
sendchange_props.append((key, props.render(value)))
self.addCompleteLog(
"sendchange", """\
master: %s
branch: %s
revision: %s
comments: %s
user: %s
files: %s
properties: %s""" % (self.master, branch, revision, comments, user, files,
sendchange_props))
bb_cmd = [
'buildbot', 'sendchange', '--master', self.master,
'--username', user, '--branch', branch, '--revision', revision
]
if isinstance(comments, basestring):
if re.search('try: ', comments, re.MULTILINE):
comments = 'try: ' + ' '.join(processMessage(comments))
else:
try:
comments = comments.splitlines()[0]
except IndexError:
comments = ''
comments = re.sub(r'[\r\n^<>|;&"\'%$]', '_', comments)
comments = comments.encode('ascii', 'replace')
if comments:
bb_cmd.extend(['--comments', comments])
for key, value in sendchange_props:
bb_cmd.extend(['--property', '%s:%s' % (key, value)])
if files:
bb_cmd.extend(self.files)
cmd = [
'python',
WithProperties("%(toolsdir)s/buildfarm/utils/retry.py"), '-s',
str(self.sleepTime), '-t',
str(self.timeout), '-r',
str(self.retries), '--stdout-regexp',
'change sent successfully'
]
cmd.extend(bb_cmd)
self.setCommand(cmd)
self.super_class.start(self)
except KeyError:
self.addCompleteLog("errors", str(Failure()))
return self.finished(FAILURE)
def addGetSourcecodeSteps(self, **kwargs):
# Checkout the monorepo.
self.addStep(
Git(name='Checkout the source code',
repourl=self.repourl_prefix + "llvm-project.git",
progress=True,
workdir=WithProperties(self.monorepo_dir),
**kwargs))
def testWorkdir(self):
self.build.setProperty("revision", 47)
self.failUnlessEqual(self.build_status.getProperty("revision"), 47)
c = ShellCommand(command=["tar", "czf", "foo.tar.gz", "source"])
c.setBuild(self.build)
workdir = WithProperties("workdir-%d", "revision")
workdir = c._interpolateWorkdir(workdir)
self.failUnlessEqual(workdir, "workdir-47")
def testSlaveName(self):
c = ShellCommand(workdir=dir,
command=[
"touch",
WithProperties("%s-slave", "slavename"),
])
c.setBuild(self.build)
cmd = c._interpolateProperties(c.command)
self.failUnlessEqual(cmd, ["touch", "bot12-slave"])
def addUnpackBuildSteps(self):
self.addStep(UnpackFile(
filename=WithProperties("%(filename)s"),
workdir=self.base_dir,
haltOnFailure=True,
timeout=60*60,
name="unpack_build",
description=['unpack', 'build'],
))
def set_mock_command(self):
if self.properties_rendered:
return
self.properties_rendered = True
# We need to have all commands as a string. We'll
# convert argv commands into string commands
if isinstance(self.command, list):
string_list = []
for arg in self.command:
if issubclass(arg.__class__, WithProperties):
string_list.append(quote(str(arg.fmtstring)))
else:
string_list.append(quote(str(arg)))
string_command = ' '.join(string_list)
elif issubclass(self.command.__class__, WithProperties):
string_command = self.command.fmtstring
else:
string_command = self.command
mock_workdir = self.mock_workdir_mutator(
self.remote_kwargs['workdir'])
# If the workdir is a WithProperties instance, we need to get the
# format string and wrap it in another WithProperties
if issubclass(mock_workdir.__class__, WithProperties):
mock_workdir = mock_workdir.fmtstring
if self.mock_workdir_prefix is not None:
mock_workdir = self.mock_workdir_prefix + mock_workdir
if 'env' in self.remote_kwargs:
pre_render_env = self.remote_kwargs['env']
properties = self.build.getProperties()
rendered_env = properties.render(pre_render_env)
environment = ' '.join('%s="%s"' % (k, rendered_env[k])
for k in rendered_env.keys())
else:
environment = ''
self.command = [self.mock_login, '-r', self.target,
'--cwd', WithProperties(mock_workdir)] + \
self.mock_args + ['--shell'] + \
[WithProperties('/usr/bin/env %s %s' % (environment,
string_command))]
def testWithPropertiesEmpty(self):
self.build.setProperty("empty", None)
c = ShellCommand(workdir=dir,
command=[
"tar", "czf",
WithProperties("build-%(empty)s.tar.gz"), "source"
])
c.setBuild(self.build)
cmd = c._interpolateProperties(c.command)
self.failUnlessEqual(cmd, ["tar", "czf", "build-.tar.gz", "source"])
def GetLatestValidated(f):
import zorg.buildbot.util.artifacts as artifacts
f.addStep(buildbot.steps.shell.ShellCommand(
name='rm.host-compiler',
command=['rm', '-rfv', 'host-compiler', 'host-compiler.tar.gz'],
haltOnFailure=False, description=['rm', 'host-compiler'],
workdir=WithProperties('%(builddir)s')))
latest_url = artifacts.base_download_url
latest_url += '/validated_builds/clang-x86_64-darwin11-R.tar.gz'
f.addStep(buildbot.steps.shell.ShellCommand(
name='download.artifacts',
command=['curl', '-fvLo', 'host-compiler.tar.gz', latest_url],
haltOnFailure=True, description=['download build artifacts'],
workdir=WithProperties('%(builddir)s')))
f.addStep(buildbot.steps.shell.ShellCommand(
name='unzip', command=['tar', '-zxvf','../host-compiler.tar.gz'],
haltOnFailure=True, description=['extract', 'host-compiler'],
workdir='host-compiler'))
return f
def testWithPropertiesDict(self):
self.build.setProperty("other", "foo")
self.build.setProperty("missing", None)
c = ShellCommand(workdir=dir,
command=[
"tar", "czf",
WithProperties("build-%(other)s.tar.gz"), "source"
])
c.setBuild(self.build)
cmd = c._interpolateProperties(c.command)
self.failUnlessEqual(cmd, ["tar", "czf", "build-foo.tar.gz", "source"])
def __init__(self,
suite,
symbols_path=None,
xrePath='../hostutils/xre',
utilityPath='../hostutils/bin',
app='org.mozilla.fennec',
totalChunks=None,
thisChunk=None,
cmdOptions=None,
**kwargs):
self.super_class = ShellCommandReportTimeout
ShellCommandReportTimeout.__init__(self, **kwargs)
self.addFactoryArguments(suite=suite,
xrePath=xrePath,
symbols_path=symbols_path,
utilityPath=utilityPath,
app=app,
totalChunks=totalChunks,
thisChunk=thisChunk,
cmdOptions=cmdOptions)
self.name = suite
if totalChunks:
self.name += '-%i' % thisChunk
self.command = [
'python', 'reftest/remotereftest.py', '--deviceIP',
WithProperties('%(sut_ip)s'), '--xre-path', xrePath,
'--utility-path', utilityPath, '--app', app, '--http-port',
WithProperties('%(http_port)s'), '--ssl-port',
WithProperties('%(ssl_port)s'), '--pidfile',
WithProperties('%(basedir)s/../remotereftest.pid'),
'--enable-privilege'
]
if cmdOptions:
self.command.extend(cmdOptions)
self.command.extend(self.getChunkOptions(totalChunks, thisChunk))
self.command.extend(self.getSuiteOptions(suite))
if symbols_path:
self.command.append(
WithProperties("--symbols-path=../%s" % symbols_path))
def __init__(self, suite, symbols_path=None, leakThreshold=None, **kwargs):
self.super_class = ShellCommandReportTimeout
ShellCommandReportTimeout.__init__(self, **kwargs)
self.addFactoryArguments(suite=suite,
symbols_path=symbols_path,
leakThreshold=leakThreshold)
self.name = suite
self.command = [
'python',
'reftest/runreftest.py',
WithProperties('--appname=%(exepath)s'),
'--utility-path=bin',
'--extra-profile-file=bin/plugins',
]
if symbols_path:
self.command.append(
WithProperties("--symbols-path=%s" % symbols_path))
if leakThreshold:
self.command.append('--leak-threshold=%d' % leakThreshold)
self.command.extend(self.getSuiteOptions(suite))
def testWithProperties(self):
self.build.setProperty("revision", 47)
self.failUnlessEqual(self.build_status.getProperty("revision"), 47)
c = ShellCommand(workdir=dir,
command=[
"tar", "czf",
WithProperties("build-%s.tar.gz", "revision"),
"source"
])
c.setBuild(self.build)
cmd = c._interpolateProperties(c.command)
self.failUnlessEqual(cmd, ["tar", "czf", "build-47.tar.gz", "source"])
def _step_Archive(self):
# make the tarball
self.addStep(
ShellCommand(name="makedist",
description=["zipfile", "package"],
workdir="build/build/root",
command=[
"zip", "-r", "-9", "-v",
WithProperties("../../%(filename)s"), ".", "-x",
".svn"
],
haltOnFailure=True))
def addObtainBuildSteps(self):
self.addDownloadBuildStep()
self.addUnpackBuildSteps()
#If people *hate* lambdas, i could define simple functions
mozilla_fn = lambda x,y,z: parse_build_info(' '.join([y,z]), 'mozilla')
mobile_fn = lambda x,y,z: parse_build_info(' '.join([y,z]), 'mobile')
self.addStep(SetProperty(
command=['cat', 'platform.ini'],
workdir='%s/fennec' % self.base_dir,
extract_fn=mozilla_fn,
description=['get', 'mobile', 'revision'],
name='mozilla_rev',
))
self.addStep(SetProperty(
command=['cat', 'application.ini'],
workdir='%s/fennec' % self.base_dir,
extract_fn=mobile_fn,
description=['get', 'mobile', 'revision'],
name='mobile_rev',
))
self.addStep(SetProperty(
command=['echo',
WithProperties('%(mozilla_changeset)s:%(mobile_changeset)s')],
property='got_revision',
name='set_got_revision',
description=['set_got_revision']
))
self.addStep(ShellCommand(
command=['echo', 'TinderboxPrint:',
WithProperties('<a href=%(mozilla_repository)s/rev/%(mozilla_changeset)s' +
'title="Built from Mozilla revision %(mozilla_changeset)s">' +
'moz:%(mozilla_changeset)s</a> <br />' +
'<a href=%(mobile_repository)s/rev/%(mobile_changeset)s' +
'title="Built from Mobile revision %(mobile_changeset)s">' +
'mobile:%(mobile_changeset)s</a>')],
description=['list', 'revisions'],
name='rev_info',
))
def __init__(self, suite, symbols_path=None, leakThreshold=None, **kwargs):
self.super_class = ShellCommandReportTimeout
ShellCommandReportTimeout.__init__(self, **kwargs)
self.addFactoryArguments(suite=suite, symbols_path=symbols_path,
leakThreshold=leakThreshold)
self.name = suite
self.command = [
'python', 'jetpack/bin/cfx',
WithProperties('--binary=%(exepath)s'),
'--parseable', suite
]
def addObtainBuildSteps(self):
MobileTalosFactory.addObtainBuildSteps(self)
self.addStep(SetProperty(
command=['echo', WithProperties("%(fileURL)s")],
extract_fn=download_dir,
name='download_dir',
description=['download', 'directory', 'property'],
))
self.addStep(ShellCommand(
command=['wget',
WithProperties("%(download_dir)s/fennec-%(version)s.en-US.linux-gnueabi-arm.tests.zip"),
'-O', 'fennec-tests.zip'],
workdir=self.base_dir,
haltOnFailure=True,
name="get_tests",
description=['get', 'unit', 'tests'],
))
self.addStep(ShellCommand(
command=['unzip', '%s/fennec-tests.zip' % self.base_dir],
workdir="%s/fennec" % self.base_dir,
name='unpack_test',
description=['unpack', 'tests'],
haltOnFailure=True,
))
def GetCXXFromCompilerArtifacts(f, base_dir):
def get_cxx(status, stdin, stdout):
lines = filter(bool, stdin.split('\n'))
for line in lines:
if 'bin/clang++' in line:
cxx_path = line
return {'cxx_path': cxx_path}
return {}
f.addStep(
buildbot.steps.shell.SetProperty(
name='find.cxx',
command=['find', base_dir, '-name', 'clang++'],
extract_fn=get_cxx,
workdir=WithProperties('%(builddir)s')))
return f
def addGetSourcecodeSteps(self, **kwargs):
# Remove 'is_legacy_mode' if it leaked in to kwargs.
kwargs.pop('is_legacy_mode', None)
# Bail out if we are in the legacy mode and SVN checkout is required.
if self.is_legacy_mode:
self.addSVNSteps(**kwargs)
return
# Checkout the monorepo.
self.addStep(
Git(name='Checkout the source code',
repourl=self.repourl_prefix + "llvm-project.git",
progress=True,
workdir=WithProperties(self.monorepo_dir),
**kwargs))
