def __init__(self, channel, options, *args, **kwargs):
channel_filename = 'channel-%s.yaml' % channel
util.BuildFactory.__init__(self, *args, **kwargs)
self.addSteps([
steps.ShellCommand(
name='install tools',
haltOnFailure=True,
logEnviron=False,
command=['sudo', 'dnf', 'install', '-y', 'flatpak', 'flatpak-builder', 'python3-PyYAML'],
),
FlatpakGPGStep(name='setup gpg keys'),
steps.Git(
name='checkout sources',
codebase=util.Property('codebase'),
repourl=util.Property('repository'),
branch=util.Property('branch'),
mode='incremental',
submodules=True,
shallow=True,
),
FlatpakPullStep(name='pull from master'),
steps.ShellCommand(
name='build',
haltOnFailure=True,
command=['./flatpak-build', '--repo=repo', '--channel=' + channel_filename, '--jobs=1', '--export', '--gpg-homedir=flatpak-gpg', '--gpg-sign=' + options['gpg-key']],
),
FlatpakRefStep(name='copy flatpakref files', channel=channel),
FlatpakSyncStep(name='sync repo'),
])
class CppBenchmark(DockerBuilder):
"""Run C++ benchmarks via the Archery CLI tool
This builder is parametrized with builtbot properties which are set by
the github hook, for more see commands.py
"""
tags = ['arrow', 'cpp', 'benchmark']
properties = dict(
CMAKE_INSTALL_PREFIX='/usr/local',
CMAKE_INSTALL_LIBDIR='lib'
)
steps = [
checkout_arrow,
Pip(['install', '-e', '.'], workdir='dev/archery'),
Archery(
args=util.FlattenList([
'benchmark',
'diff',
'--output=diff.json',
util.Property('benchmark_options', []),
'WORKSPACE',
util.Property('benchmark_baseline', 'master')
]),
result_file='diff.json'
)
]
image_filter = Filter(
name='cpp-benchmark',
tag='worker',
variant=None, # plain linux images, not conda
platform=Filter(
arch='amd64', # until ARROW-5382: SSE on ARM NEON gets resolved
distro='ubuntu'
)
)
class CppBenchmark(DockerBuilder):
tags = ['arrow', 'cpp', 'benchmark']
properties = {
'CMAKE_INSTALL_PREFIX': '/usr/local',
'CMAKE_INSTALL_LIBDIR': 'lib'
}
steps = [
checkout_arrow,
Pip(['install', '-e', '.'], workdir='dev/archery'),
Archery(
args=util.FlattenList([
'benchmark',
'diff',
'--output=diff.json',
util.Property('benchmark_options', []),
'WORKSPACE',
util.Property('benchmark_baseline', 'master')
]),
result_file='diff.json'
)
]
images = images.filter(
name='cpp-benchmark',
os=startswith('ubuntu'),
arch='amd64', # until ARROW-5382: SSE on ARM NEON gets resolved
variant=None, # plain linux images, not conda
tag='worker'
)
def create_factory():
factory = BuildFactory()
factory.addSteps(common.initTargetProperty())
factory.addSteps(common.initNameProperty())
factory.addStep(
steps.Trigger(name="Call the 'build' scheduler",
schedulerNames=['build'],
waitForFinish=True,
haltOnFailure=True,
copy_properties=COMMON_PROPERTIES,
set_properties={
'virtual_builder_name':
util.Interpolate('Build for %(prop:box)s'),
}))
factory.addStep(
steps.Trigger(name="Call the 'run_test' scheduler",
schedulerNames=['run_test'],
waitForFinish=True,
copy_properties=COMMON_PROPERTIES,
set_properties={
'test_branch': util.Property('branch'),
"test_set": common.renderTestSet,
"use_valgrind": util.Property("use_valgrind"),
"use_callgrind": util.Property("use_callgrind"),
"backend_ssl": util.Property("backend_ssl")
}))
return factory
def build_coverage():
remove_build = steps.RemoveDirectory("build")
create_build = steps.MakeDirectory("build")
cmake_step = steps.CMake(path=util.Property("src_dir"),
definitions=util.Property("cmake_defs", {}),
options=util.Property("cmake_opts", []),
workdir="build",
env=env)
@util.renderer
def join_make_opts(props):
make_opts = props.getProperty("make_opts", [])
return ["make"] + make_opts
make_step = steps.Compile(command=join_make_opts, workdir="build", env=env)
test_coverage = steps.ShellCommand(command=["make", "coverage"],
workdir="build")
upload_coverage_data = steps.ShellCommand(command=[
"bash", "-c",
util.Interpolate("bash <(curl -s https://codecov.io/bash) -t " +
tokens.codecovToken +
" -C %(prop:revision)s -f coverage.info.cleaned")
],
workdir="build")
factory = util.BuildFactory()
factory.addStep(remove_build)
factory.addStep(create_build)
factory.addStep(cmake_step)
factory.addStep(make_step)
factory.addStep(test_coverage)
factory.addStep(upload_coverage_data)
return factory
def createBuildSteps():
buildSteps = []
buildSteps.extend(common.configureMdbciVmPathProperty())
buildSteps.append(steps.SetProperties(properties=configureBuildProperties))
buildSteps.extend(common.cloneRepository())
buildSteps.append(steps.ShellCommand(
name="Build MaxScale using MDBCI",
command=['/bin/bash', '-c', 'BUILD/mdbci/build.sh || BUILD/mdbci/build.sh'],
timeout=3600,
workdir=util.Interpolate("%(prop:builddir)s/build")
))
buildSteps.extend(common.destroyVirtualMachine())
buildSteps.append(common.runSshCommand(
name="Make dir for build results on the repo server",
host=util.Property("upload_server"),
command=["mkdir", "-p", util.Interpolate(constants.UPLOAD_PATH + '/%(prop:target)s')],
))
buildSteps.append(common.rsyncViaSsh(
name="Rsync builds results to the repo server",
local=util.Interpolate("%(prop:builddir)s/repository/%(prop:target)s/mariadb-maxscale/"),
remote=util.Interpolate("%(prop:upload_server)s:" + constants.UPLOAD_PATH + "/%(prop:target)s/")
))
buildSteps.append(common.generateMdbciRepositoryForTarget())
buildSteps.extend(common.syncRepod())
buildSteps.append(steps.ShellCommand(
name="Upgrade test",
command=['BUILD/mdbci/upgrade_test.sh'],
timeout=1800,
doStepIf=(util.Property('run_upgrade_test') == 'yes'),
workdir=util.Interpolate("%(prop:builddir)s/build")
))
buildSteps.extend(common.cleanBuildDir())
return buildSteps
class CrossbowTrigger(DockerBuilder):
tags = ['crossbow']
steps = [
GitHub(name='Clone Arrow',
repourl=util.Property('repository'),
workdir='arrow',
mode='full'),
GitHub(
name='Clone Crossbow',
# TODO(kszucs): read it from the comment and set as a property
repourl='https://github.com/ursa-labs/crossbow',
workdir='crossbow',
branch='master',
mode='full',
# quite misleasing option, but it prevents checking out the branch
# set in the sourcestamp by the pull request, which refers to arrow
alwaysUseLatest=True),
Crossbow(args=util.FlattenList([
'--github-token',
util.Secret('ursabot/github_token'), 'submit', '--output',
'job.yml', '--job-prefix', 'ursabot', '--arrow-remote',
util.Property('repository'),
util.Property('crossbow_args', [])
]),
workdir='arrow/dev/tasks',
result_file='job.yml')
]
images = images.filter(name='crossbow', tag='worker')
def __init__(self, datadir, **kwargs):
"""Simply initialize MasterShellCommand with the arguments to call the script."""
super().__init__(command=None, **kwargs)
self.command = [os.path.expanduser('~/gcc-buildbot/scripts/regression-analysis.py'),
"--data-dir", util.Interpolate("%(kw:datadir)s", datadir=datadir),
"--builder", util.Property('buildername'),
"--branch", util.Property('branch'),
util.Property('got_revision')]
def build_volk_PR():
create_src = steps.MakeDirectory(name="create src directory", dir="volk")
clone_step = steps.GitHub(name="fetch PR source",
repourl=util.Property("repository"),
mode="full",
method="fresh",
submodules=True,
retryFetch=True,
clobberOnFailure=True,
workdir="volk")
rm_src_dir = steps.RemoveDirectory(
dir=util.Interpolate(
os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s",
"%(prop:github.base.ref)s")),
hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS,
)
copy_src = steps.CopyDirectory(
name="copy src to srcdir",
src="volk",
dest=util.Interpolate(
os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s",
"%(prop:github.base.ref)s"), ),
hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS,
)
# load builders.json with definitions on how to build things
parent_path = os.path.dirname(__file__)
with open(os.path.join(parent_path, "volk_builders.json"),
"r") as builders_file:
build_config = json.loads(builders_file.read())
trigger_builds = custom_steps.BuildTrigger(
name="trigger the right builders",
build_config=build_config,
schedulerNames=["trigger"],
runner="pull",
set_properties={
"pr_base":
util.Property("github.base.ref"),
"src_dir":
util.Interpolate(
os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s"))
},
test_merge=False,
updateSourceStamp=False,
waitForFinish=True)
factory = util.BuildFactory()
factory.addStep(create_src)
factory.addStep(clone_step)
factory.addStep(rm_src_dir)
factory.addStep(copy_src)
factory.addStep(trigger_builds)
return factory
def cloneRepository():
"""Clone MaxScale repository using default configuration options"""
return [
steps.Git(name=util.Interpolate(
"Clone repository '%(prop:repository)s', branch '%(prop:branch)s'"
),
repourl=util.Property('repository'),
branch=util.Property('branch'),
mode='incremental',
haltOnFailure=True)
]
def _doRepoSync(self, env: Optional[Dict[str, str]] = None):
"""Do repo init and sync via the API provided by Buildbot that neatly
abstract the use of repo command line (which can be picky to use)."""
self.addStep(steps.Repo(
name="repo init and sync",
description="synchronize the CLIP OS source tree with repo",
haltOnFailure=True,
manifestURL=util.Property("repository"),
manifestBranch=util.Property("branch"),
jobs=4,
depth=0, # Never shallow sync!
syncAllBranches=True, # All branches must be present for some features.
env=env,
))
def build_coverity():
remove_build = steps.RemoveDirectory("build")
remove_src = steps.RemoveDirectory("src")
create_build = steps.MakeDirectory("build")
download_src_archive = steps.FileDownload(
mastersrc=util.Property("src_archive"),
workerdest="src.tar.xz",
workdir="src")
extract_src_archive = steps.ShellCommand(
name="Extract source archive",
command=["tar", "xJf", "src.tar.xz"],
workdir="src")
cmake_step = steps.CMake(path="../src/",
definitions=util.Property("cmake_defs", {}),
options=util.Property("cmake_opts", []),
workdir="build",
env=env)
make_step = steps.Compile(command=[
"cov-build", "--dir", "cov-int", "make", "-j", "16", "-l", "32"
],
workdir="build",
env=env)
compress = steps.ShellCommand(
command=["tar", "czvf", "gnuradio.tgz", "cov-int"], workdir="build")
upload = steps.ShellCommand(command=[
"curl", "--form", "token=" + tokens.coverityToken, "--form",
"[email protected]", "--form", "[email protected]", "--form",
util.Interpolate("version=%(prop:revision)s"), "--form",
util.Interpolate(
"description=\"Weekly Buildbot submission for %(prop:branch)s branch \""
), "https://scan.coverity.com/builds?project=GNURadio"
],
workdir="build")
factory = util.BuildFactory()
factory.addStep(remove_build)
factory.addStep(remove_src)
factory.addStep(create_build)
factory.addStep(download_src_archive)
factory.addStep(extract_src_archive)
factory.addStep(cmake_step)
factory.addStep(make_step)
factory.addStep(compress)
factory.addStep(upload)
return factory
def uploadTestRunsToReportPortal(**kwargs):
"""Save test results to the report portal"""
return steps.ShellCommand(
name="Send test results to the Report Portal",
command=[
util.Interpolate(
"%(prop:HOME)s/mdbci/scripts/build_parser/report_portal/bin/upload_testrun.rb"
),
util.Property("jsonResultsFile"),
util.Interpolate("%(prop:HOME)s/report-portal-config.yml")
],
env={
"LAST_WRITE_BUILD_RESULTS_ID":
util.Property("LAST_WRITE_BUILD_RESULTS_ID")
},
**kwargs)
def writeBuildResultsToDatabase(**kwargs):
"""Call the script to save results to the database"""
return [
steps.SetPropertyFromCommand(
name="Save test results to the database",
command=[
util.Interpolate(
"%(prop:builddir)s/scripts/write_build_results.py"),
"--run-id",
util.Property("buildId"),
util.Property("jsonResultsFile"), "--database-info",
util.Secret("dataBaseInfo.json")
],
extract_fn=extractDatabaseBuildid,
**kwargs)
]
def addConfigureSteps(self,
f,
platform,
*,
env,
configure_path,
additional_args=None):
if additional_args is None:
additional_args = []
f.addStep(
scummsteps.SetPropertyIfOlder(
name="check {0} freshness".format(
self.CONFIGURE_GENERATED_FILE),
src=configure_path,
generated=self.CONFIGURE_GENERATED_FILE,
property="do_configure"))
command = [configure_path]
command.extend(additional_args)
if self.verbose_build and self.VERBOSE_BUILD_FLAG:
command.append(self.VERBOSE_BUILD_FLAG)
command.extend(platform.getConfigureArgs(self))
f.addStep(
steps.Configure(command=command,
doStepIf=util.Property("do_configure",
default=True,
defaultWhenFalse=False),
env=env))
def initTargetProperty():
"""
Sets 'target' property of the build to:
- <branch>-buildbot-<starttime> if it isn't set yet or property 'targetInitMode' is TargetInitOptions.GENERATE;
- <branch> if property 'targetInitMode' is TargetInitOptions.SET_FROM_BRANCH.
:return: list of steps
"""
return [
steps.SetProperty(
name=util.Interpolate("Set 'target' property"),
property="target",
value=util.Interpolate(
"%(prop:branch)s-buildbot-%(kw:startTime)s",
startTime=getFormattedDateTime("%Y-%b-%d-%H-%M-%S")),
doStepIf=lambda step: step.build.getProperty('target') is None and
step.build.getProperty('targetInitMode') is None or step.build.
getProperty('targetInitMode') == TargetInitOptions.GENERATE,
hideStepIf=lambda results, s: results == SKIPPED),
steps.SetProperty(
name=util.Interpolate("Set 'target' property"),
property="target",
value=util.Property("branch"),
doStepIf=lambda step: step.build.getProperty(
'targetInitMode') == TargetInitOptions.SET_FROM_BRANCH,
hideStepIf=lambda results, s: results == SKIPPED)
]
class CrossbowReport(CrossbowBuilder):
steps = Extend([
SetPropertyFromCommand(
'crossbow_job_id',
extract_fn=lambda stdout, stderr: stdout.strip(),
command=Crossbow(args=[
'--github-token',
util.Secret('ursabot/github_token'), 'latest-prefix',
crossbow_prefix
]),
workdir='arrow/dev/tasks'),
Crossbow(name='Generate and send nightly report',
args=util.FlattenList([
'--github-token',
util.Secret('ursabot/github_token'), 'report', '--send',
'--poll', '--poll-max-minutes', 120,
'--poll-interval-minutes', 15, '--sender-name',
'Crossbow', '--sender-email', '*****@*****.**',
'--recipient-email', '*****@*****.**',
'--smtp-user',
util.Secret('crossbow/smtp_user'), '--smtp-password',
util.Secret('crossbow/smtp_password'),
util.Property('crossbow_job_id')
]),
workdir='arrow/dev/tasks'),
Crossbow(name="Update Crossbow's Github page",
args=util.FlattenList([
'--github-token',
util.Secret('ursabot/github_token'), 'github-page',
'generate', '-n', 20, '--github-push-token',
util.Secret('kszucs/github_status_token')
]),
workdir='arrow/dev/tasks')
])
def test_constructor_all_docker_parameters(self):
# Volumes have their own tests
bs = self.setupWorker('bot',
'pass',
'unix:///var/run/docker.sock',
'worker_img', ['/bin/sh'],
dockerfile="FROM ubuntu",
version='1.9',
tls=True,
hostconfig={
'network_mode': 'fake',
'dns': ['1.1.1.1', '1.2.3.4']
})
assert bs.workername == 'bot'
assert bs.password == 'pass'
assert bs.image == 'worker_img'
assert bs.command == ['/bin/sh']
assert bs.dockerfile == "FROM ubuntu"
assert bs.volumes, util.Property('docker_image', default=[])
assert bs.client_args == {
'base_url': 'unix:///var/run/docker.sock',
'version': '1.9',
'tls': True
}
assert isinstance(bs.hostconfig, util.Transform)
def create_factory():
factory = BuildFactory()
factory.addSteps(common.initTargetProperty())
factory.addStep(
steps.Trigger(name="Call the 'build' scheduler",
schedulerNames=['build'],
waitForFinish=True,
haltOnFailure=True,
copy_properties=COMMON_PROPERTIES,
set_properties={
'virtual_builder_name':
util.Interpolate('Build for %(prop:box)s'),
'box':
'ubuntu_bionic_libvirt',
'try_already_running':
'no',
}))
factory.addStep(
steps.Trigger(name="Call the 'run_performance_test' scheduler",
schedulerNames=['run_performance_test_trigger'],
waitForFinish=True,
copy_properties=COMMON_PROPERTIES,
set_properties={
'test_branch': util.Property('branch'),
'host': 'max-tst-01',
}))
return factory
def getCmakeWithNinjaWithMSVCBuildFactory(
depends_on_projects=None,
llvm_srcdir=None,
obj_dir=None,
checks=None,
install_dir=None,
clean=False,
extra_configure_args=None,
# VS tools environment variable if using MSVC. For example,
# %VS140COMNTOOLS% selects the 2015 toolchain.
vs=None,
target_arch=None,
env=None,
**kwargs):
assert not env, "Can't have custom builder env vars with MSVC build"
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if checks is None:
checks = ['check-all']
f = getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
**kwargs) # Pass through all the extra arguments.
f.addStep(
SetPropertyFromCommand(
command=builders_util.getVisualStudioEnvironment(vs, target_arch),
extract_fn=builders_util.extractVSEnvironment))
env = util.Property('vs_env')
cleanBuildRequested = lambda step: step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj")) or clean
addCmakeSteps(f,
generator='Ninja',
cleanBuildRequested=cleanBuildRequested,
obj_dir=f.obj_dir,
install_dir=f.install_dir,
extra_configure_args=cmake_args,
env=env,
**kwargs)
addNinjaSteps(f,
obj_dir=obj_dir,
checks=checks,
install_dir=f.install_dir,
env=env,
**kwargs)
return f
def test_constructor_noimage_nodockerfile(self):
bs = self.setupWorker('bot', 'pass', 'unix:///var/run/docker.sock')
assert bs.image == util.Property('docker_image')
assert bs.max_builds == 1
with pytest.raises(ConfigErrors):
self.setupWorker('bot', 'pass', 'unix:///var/run/docker.sock',
max_builds=2)
def build_and_test():
remove_build = steps.RemoveDirectory("build")
remove_src = steps.RemoveDirectory("src")
create_build = steps.MakeDirectory("build")
download_src_archive = steps.FileDownload(
mastersrc=util.Property("src_archive"),
workerdest="src.tar.xz",
workdir="src")
extract_src_archive = steps.ShellCommand(
name="Extract source archive",
command=["tar", "xJf", "src.tar.xz"],
workdir="src")
cmake_step = steps.CMake(path="../src/",
definitions=util.Property("cmake_defs", {}),
options=util.Property("cmake_opts", []),
workdir="build",
env=env)
@util.renderer
def join_make_opts(props):
make_opts = props.getProperty("make_opts", [])
return ["make"] + make_opts
make_step = steps.Compile(command=join_make_opts, workdir="build", env=env)
@util.renderer
def parse_test_excludes(props):
command = ["ctest", "--output-on-failure", "--timeout", "120"]
excludes = props.getProperty('test_excludes', [])
excludes.append("qtgui")
if excludes is not None:
command += ["-E", "|".join(excludes)]
return command
test_step = steps.Test(command=parse_test_excludes, workdir="build")
factory = util.BuildFactory()
factory.addStep(remove_build)
factory.addStep(remove_src)
factory.addStep(create_build)
factory.addStep(download_src_archive)
factory.addStep(extract_src_archive)
factory.addStep(cmake_step)
factory.addStep(make_step)
factory.addStep(test_step)
return factory
def cleanBuildDir():
"""Clean the build directory after the worker have completed the task"""
return [
steps.ShellCommand(name="Clean build directory using 'git clean -fd'",
command=["rm", "-rf",
util.Property('builddir')],
alwaysRun=True)
]
def buildMdbci():
"""This script will be run on the worker"""
return steps.ShellCommand(
name="Build MDBCI",
command=["./package/build.sh",
util.Property("buildnumber")],
haltOnFailure=True,
)
class Test(DockerBuilder):
properties = {'A': util.Property('builddir')}
hostconfig = {'shm_size': util.Transform(to_gigabytes, 2 * 1024**3)}
volumes = [
util.Interpolate('%(prop:builddir)s:/root/.ccache:rw'),
util.Interpolate(
'%(prop:builddir)s/subdir:%(prop:docker_workdir)s/subdir:rw')
]
def __init__(self, **kwargs):
steps.MasterShellCommand.__init__(self, command=None, **kwargs)
self.command = [
os.path.expanduser("~/scripts/update-logs.sh"),
"--commit",
util.Property("got_revision"),
"--builder",
util.Property("buildername"),
"--base-directory",
get_web_base(),
"--branch",
util.Property("branch"),
"--is-try-sched",
util.Property("isTrySched", default="no"),
"--try-count",
util.Property("try_count", default="0"),
]
def Strip(command, **kwargs):
return steps.ShellCommand(
name = "strip",
description = "stripping",
descriptionDone = "strip",
command = command,
doStepIf = util.Property("package", default=False),
**kwargs)
def createFactory():
factory = BuildFactory()
factory.addSteps(common.initTargetProperty())
factory.addStep(steps.Trigger(
name="Call the 'build' scheduler. Build CentOS",
schedulerNames=['build'],
waitForFinish=True,
haltOnFailure=True,
copy_properties=COMMON_BUILD_AND_TEST_SNAPSHOT_PROPERTIES,
set_properties={
"box": util.Property("box"),
'try_already_running': 'yes',
"target": util.Property("target"),
'virtual_builder_name': util.Interpolate('Build for %(prop:box)s'),
}
))
factory.addStep(steps.Trigger(
name="Call the 'run_test_snapshot' scheduler. Run functional tests",
schedulerNames=['run_test_snapshot'],
waitForFinish=True,
copy_properties=COMMON_BUILD_AND_TEST_SNAPSHOT_PROPERTIES,
set_properties={
"box": util.Property("box"),
"target": util.Property("target"),
"test_branch": util.Property("branch"),
"test_set": common.renderTestSet,
"backend_ssl": util.Property("backend_ssl"),
"use_valgrind": util.Property("use_valgrind"),
}
))
return factory
def checkConfig(self, name, password, docker_host, image=None,
command=None, volumes=None, hostconfig=None, **kwargs):
# Bypass the validation implemented in the parent class.
if image is None:
image = util.Property('docker_image', default=image)
super().checkConfig(
name, password, docker_host, image=image, command=command,
volumes=volumes, hostconfig=hostconfig, **kwargs
)
def asan(stage_name):
return steps.ShellCommand(
command=['./qa/zcash/full_test_suite.py', stage_name],
name=stage_name,
env={
'ASAN_OPTIONS': 'symbolize=1:report_globals=1:check_initialization_order=true:detect_stack_use_after_return=true',
'ASAN_SYMBOLIZER_PATH': util.Property('llvm-symbolizer', default='/usr/bin/llvm-symbolizer-3.5'),
},
)