def build_coverity():
remove_build = steps.RemoveDirectory("build")
remove_src = steps.RemoveDirectory("src")
create_build = steps.MakeDirectory("build")
download_src_archive = steps.FileDownload(
mastersrc=util.Property("src_archive"),
workerdest="src.tar.xz",
workdir="src")
extract_src_archive = steps.ShellCommand(
name="Extract source archive",
command=["tar", "xJf", "src.tar.xz"],
workdir="src")
cmake_step = steps.CMake(path="../src/",
definitions=util.Property("cmake_defs", {}),
options=util.Property("cmake_opts", []),
workdir="build",
env=env)
make_step = steps.Compile(command=[
"cov-build", "--dir", "cov-int", "make", "-j", "16", "-l", "32"
],
workdir="build",
env=env)
compress = steps.ShellCommand(
command=["tar", "czvf", "gnuradio.tgz", "cov-int"], workdir="build")
upload = steps.ShellCommand(command=[
"curl", "--form", "token=" + tokens.coverityToken, "--form",
"[email protected]", "--form", "[email protected]", "--form",
util.Interpolate("version=%(prop:revision)s"), "--form",
util.Interpolate(
"description=\"Weekly Buildbot submission for %(prop:branch)s branch \""
), "https://scan.coverity.com/builds?project=GNURadio"
],
workdir="build")
factory = util.BuildFactory()
factory.addStep(remove_build)
factory.addStep(remove_src)
factory.addStep(create_build)
factory.addStep(download_src_archive)
factory.addStep(extract_src_archive)
factory.addStep(cmake_step)
factory.addStep(make_step)
factory.addStep(compress)
factory.addStep(upload)
return factory
def getLLVMBuildFactoryAndPrepareForSourcecodeSteps(depends_on_projects=None,
llvm_srcdir=None,
obj_dir=None,
install_dir=None,
cleanBuildRequested=None,
env=None,
**kwargs):
def cleanBuildRequestedByProperty(step):
return step.build.getProperty("clean")
if cleanBuildRequested is None:
# We want a clean checkout only if requested by the property.
cleanBuildRequested = cleanBuildRequestedByProperty
f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
cleanBuildRequested=cleanBuildRequested,
**kwargs) # Pass through all the extra arguments.
# Remove the source code for a clean checkout if requested by property.
# TODO: Some Windows workers do not handle RemoveDirectory command well.
# So, consider running "rmdir /S /Q <dir>" if the build runs on Windows.
f.addStep(
steps.RemoveDirectory(
name='clean-src-dir',
dir=f.monorepo_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequestedByProperty,
))
return f
def build_coverage():
remove_build = steps.RemoveDirectory("build")
create_build = steps.MakeDirectory("build")
cmake_step = steps.CMake(path=util.Property("src_dir"),
definitions=util.Property("cmake_defs", {}),
options=util.Property("cmake_opts", []),
workdir="build",
env=env)
@util.renderer
def join_make_opts(props):
make_opts = props.getProperty("make_opts", [])
return ["make"] + make_opts
make_step = steps.Compile(command=join_make_opts, workdir="build", env=env)
test_coverage = steps.ShellCommand(command=["make", "coverage"],
workdir="build")
upload_coverage_data = steps.ShellCommand(command=[
"bash", "-c",
util.Interpolate("bash <(curl -s https://codecov.io/bash) -t " +
tokens.codecovToken +
" -C %(prop:revision)s -f coverage.info.cleaned")
],
workdir="build")
factory = util.BuildFactory()
factory.addStep(remove_build)
factory.addStep(create_build)
factory.addStep(cmake_step)
factory.addStep(make_step)
factory.addStep(test_coverage)
factory.addStep(upload_coverage_data)
return factory
def __init__(self):
ZcashBaseFactory.__init__(self)
self.addSteps([
steps.MakeDirectory(
dir='build/ibd-datadir',
name='Create datadir',
),
sh('touch', 'ibd-datadir/zcash.conf', name='Create zcash.conf'),
InitialBlockDownloadTimeRunner(
'ibd-datadir',
name='time-InitialBlockDownload',
),
steps.RemoveDirectory(
dir='build/ibd-datadir',
name='Remove datadir',
alwaysRun=True,
),
steps.POST(urllib.parse.urljoin(CODESPEED_URL, '/result/add/json/'),
data={'json': getPerfJson},
auth=('buildbot', CODESPEED_PASS),
verify=CODESPEED_CERT,
doStepIf=lambda s: s.getProperty('publish', False),
hideStepIf=lambda results, s: results==util.SKIPPED,
),
])
def build_and_test():
remove_build = steps.RemoveDirectory("build")
remove_src = steps.RemoveDirectory("src")
create_build = steps.MakeDirectory("build")
download_src_archive = steps.FileDownload(
mastersrc=util.Property("src_archive"),
workerdest="src.tar.xz",
workdir="src")
extract_src_archive = steps.ShellCommand(
name="Extract source archive",
command=["tar", "xJf", "src.tar.xz"],
workdir="src")
cmake_step = steps.CMake(path="../src/",
definitions=util.Property("cmake_defs", {}),
options=util.Property("cmake_opts", []),
workdir="build",
env=env)
@util.renderer
def join_make_opts(props):
make_opts = props.getProperty("make_opts", [])
return ["make"] + make_opts
make_step = steps.Compile(command=join_make_opts, workdir="build", env=env)
@util.renderer
def parse_test_excludes(props):
command = ["ctest", "--output-on-failure", "--timeout", "120"]
excludes = props.getProperty('test_excludes', [])
excludes.append("qtgui")
if excludes is not None:
command += ["-E", "|".join(excludes)]
return command
test_step = steps.Test(command=parse_test_excludes, workdir="build")
factory = util.BuildFactory()
factory.addStep(remove_build)
factory.addStep(remove_src)
factory.addStep(create_build)
factory.addStep(download_src_archive)
factory.addStep(extract_src_archive)
factory.addStep(cmake_step)
factory.addStep(make_step)
factory.addStep(test_step)
return factory
def build_volk_PR():
create_src = steps.MakeDirectory(name="create src directory", dir="volk")
clone_step = steps.GitHub(name="fetch PR source",
repourl=util.Property("repository"),
mode="full",
method="fresh",
submodules=True,
retryFetch=True,
clobberOnFailure=True,
workdir="volk")
rm_src_dir = steps.RemoveDirectory(
dir=util.Interpolate(
os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s",
"%(prop:github.base.ref)s")),
hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS,
)
copy_src = steps.CopyDirectory(
name="copy src to srcdir",
src="volk",
dest=util.Interpolate(
os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s",
"%(prop:github.base.ref)s"), ),
hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS,
)
# load builders.json with definitions on how to build things
parent_path = os.path.dirname(__file__)
with open(os.path.join(parent_path, "volk_builders.json"),
"r") as builders_file:
build_config = json.loads(builders_file.read())
trigger_builds = custom_steps.BuildTrigger(
name="trigger the right builders",
build_config=build_config,
schedulerNames=["trigger"],
runner="pull",
set_properties={
"pr_base":
util.Property("github.base.ref"),
"src_dir":
util.Interpolate(
os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s"))
},
test_merge=False,
updateSourceStamp=False,
waitForFinish=True)
factory = util.BuildFactory()
factory.addStep(create_src)
factory.addStep(clone_step)
factory.addStep(rm_src_dir)
factory.addStep(copy_src)
factory.addStep(trigger_builds)
return factory
def build_and_test():
remove_build = steps.RemoveDirectory("build")
create_build = steps.MakeDirectory("build")
cmake_step = steps.CMake(path=util.Property("src_dir"),
definitions=util.Property("cmake_defs", {}),
options=util.Property("cmake_opts", []),
workdir="build",
env=env)
@util.renderer
def join_make_opts(props):
make_opts = props.getProperty("make_opts", [])
return ["make"] + make_opts
make_step = steps.Compile(command=join_make_opts, workdir="build", env=env)
def parse_exclude_file(rc, stdout, stderr):
exclude_tests = json.loads(stdout)
return {"test_excludes": exclude_tests}
load_exclude_file = steps.SetPropertyFromCommand(
command=["cat", os.path.join("/config", "test_excludes.json")],
extract_fn=parse_exclude_file,
doStepIf=lambda steps: steps.getProperty("exclude_file", False))
@util.renderer
def parse_test_excludes(props):
command = ["ctest", "--output-on-failure", "--timeout", "10"]
excludes = props.getProperty("test_excludes", None)
if excludes is not None:
command += ["-E", "|".join(excludes)]
return command
test_step = steps.Test(command=parse_test_excludes, workdir="build")
factory = util.BuildFactory()
factory.addStep(remove_build)
factory.addStep(create_build)
factory.addStep(load_exclude_file)
factory.addStep(cmake_step)
factory.addStep(make_step)
factory.addStep(test_step)
return factory
def _addSteps4StagedCompiler(f,
stage_idx=1,
use_stage_idx=-1,
jobs=None,
extra_configure_args=None,
env=None):
if use_stage_idx < 0:
use_stage_idx = stage_idx - 1
# Index is zero-based, so we want to use a human friendly number instead.
stage_num = stage_idx + 1
# Directories to use on this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeTo(
f.stage_installdirs[stage_idx], obj_dir)
staged_install = f.stage_installdirs[use_stage_idx]
# Always do a clean build for the staged compiler.
f.addStep(
steps.RemoveDirectory(
name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
))
f.addStep(
steps.RemoveDirectory(
name='clean-%s-dir' % f.stage_installdirs[stage_idx],
dir=f.stage_installdirs[stage_idx],
haltOnFailure=False,
flunkOnFailure=False,
))
# Reconcile the cmake options for this stage.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
cmake_args.append(
WithProperties("-DCMAKE_CXX_COMPILER=%(builddir)s/" + staged_install +
"/bin/clang++"))
cmake_args.append(
WithProperties("-DCMAKE_C_COMPILER=%(builddir)s/" + staged_install +
"/bin/clang"))
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
# Create configuration files with cmake
f.addStep(
CmakeCommand(
name="cmake-configure-stage%s" % stage_num,
description=["stage%s cmake configure" % stage_num],
haltOnFailure=True,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
))
# Build clang by the staged compiler
f.addStep(
NinjaCommand(
name="build-stage%s-compiler" % stage_num,
jobs=jobs,
haltOnFailure=True,
description=["build stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
# Test just built compiler
f.addStep(
NinjaCommand(
name="test-stage%s-compiler" % stage_num,
targets=["check-all"],
jobs=jobs,
haltOnFailure=True,
description=["test stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
# Install just built compiler
f.addStep(
NinjaCommand(
name="install-stage%s-compiler" % stage_num,
targets=["install"],
jobs=jobs,
haltOnFailure=True,
description=["install stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
def _addSteps4SystemCompiler(f,
stage_idx=0,
clean=True,
jobs=None,
extra_configure_args=None,
env=None):
# Index is zero-based, so we want to use a human friendly number instead.
stage_num = stage_idx + 1
# Directories to use on this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeTo(
f.stage_installdirs[stage_idx], obj_dir)
# This stage could use incremental build.
# Clean stage1, only if requested.
f.addStep(
steps.RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=clean))
f.addStep(
steps.RemoveDirectory(name='clean-%s-dir' %
f.stage_installdirs[stage_idx],
dir=f.stage_installdirs[stage_idx],
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=clean))
# Reconcile the cmake options for this stage.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(
cmake_args,
[
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
# Do not expect warning free build by the system toolchain.
('-DLLVM_ENABLE_WERROR=', 'OFF'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
# Note: On this stage we do not care of warnings, as we build with
# a system toolchain and cannot control the environment.
# Warnings are likely, and we ignore them.
# Create configuration files with cmake
f.addStep(
CmakeCommand(
name="cmake-configure-stage%s" % stage_num,
description=["stage%s cmake configure" % stage_num],
haltOnFailure=True,
flunkOnWarnings=False,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
))
# Build clang by the system compiler
f.addStep(
NinjaCommand(
name="build-stage%s-compiler" % stage_num,
jobs=jobs,
haltOnFailure=True,
flunkOnWarnings=False,
description=["build stage%s compiler" % stage_num],
env=env,
workdir=obj_dir,
))
# Test stage1 compiler
f.addStep(
NinjaCommand(
name="test-stage%s-compiler" % stage_num,
targets=["check-all"], # or "check-llvm", "check-clang"
jobs=jobs,
haltOnFailure=True,
flunkOnWarnings=False,
description=["test stage%s compiler" % stage_num],
env=env,
workdir=obj_dir,
))
# Install stage1 compiler
f.addStep(
NinjaCommand(
name="install-stage%s-compiler" % stage_num,
targets=["install"],
jobs=jobs,
haltOnFailure=True,
description=["install stage%s compiler" % stage_num],
env=env,
workdir=obj_dir,
))
def getFuchsiaToolchainBuildFactory(
test=True,
env=None, # Environmental variables for all steps.
extra_configure_args=None,
**kwargs):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
"TERM": "dumb", # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
src_dir = "llvm.src"
obj_dir = "llvm.obj"
install_dir = "llvm.install"
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=[
"llvm", "clang", "clang-tools-extra", "compiler-rt", "libcxx",
"libcxxabi", "libunwind", "lld"
],
llvm_srcdir=src_dir,
obj_dir=obj_dir,
install_dir=install_dir,
**kwargs) # Pass through all the extra arguments.
# Get Fuchsia SDK.
sdk_dir = "fuchsia.sdk"
sdk_platform = {
"Linux": "linux-amd64",
"Darwin": "mac-amd64",
}[platform.system()]
sdk_version = "latest"
sdk_url = WithProperties(
"https://chrome-infra-packages.appspot.com/dl/"
"fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s",
sdk_platform=lambda _: sdk_platform,
sdk_version=lambda _: sdk_version)
f.addStep(
steps.RemoveDirectory(name="clean-sdk",
dir=sdk_dir,
haltOnFailure=True))
f.addStep(
ShellCommand(name="fetch-sdk",
command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url],
description=["download", "fuchsia sdk"],
workdir=sdk_dir))
f.addStep(
ShellCommand(name="extract-sdk",
command=["unzip", "-fo", "sdk.cipd"],
description=["extract", "fuchsia sdk"],
workdir=sdk_dir))
cleanBuildRequested = lambda step: step.build.getProperty("clean",
default=True)
# Clean up llvm build.
f.addStep(
steps.RemoveDirectory(name="clean-llvm.obj",
dir=obj_dir,
haltOnFailure=True,
doStepIf=cleanBuildRequested))
f.addStep(
steps.RemoveDirectory(name="clean-llvm.install",
dir=install_dir,
haltOnFailure=True,
doStepIf=cleanBuildRequested))
# Configure.
if extra_configure_args is None:
cmake_options = []
else:
cmake_options = extra_configure_args[:]
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_options, [
("-G", "Ninja"),
("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"),
("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"),
])
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_options, [
("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"),
("-DLLVM_ENABLE_LTO=", "OFF"),
])
cmake_options.append(
WithProperties("-DCMAKE_INSTALL_PREFIX=%(builddir)s/" + install_dir))
cmake_options.append(
WithProperties("-DFUCHSIA_SDK=%(builddir)s/" + sdk_dir))
CmakeCommand.applyRequiredOptions(cmake_options, [
("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"),
])
f.addStep(
CmakeCommand(name="cmake-configure",
options=cmake_options,
path='../' + src_dir + '/llvm',
haltOnFailure=True,
description=["configure"],
workdir=obj_dir,
env=merged_env,
doStepIf=FileDoesNotExist("CMakeCache.txt")))
# Build distribution.
f.addStep(
NinjaCommand(name="ninja-build",
targets=["stage2-distribution"],
haltOnFailure=True,
description=["build"],
workdir=obj_dir,
env=merged_env))
# Test llvm, clang and lld.
f.addStep(
NinjaCommand(
name="check",
targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")],
haltOnFailure=True,
description=["check"],
workdir=obj_dir,
env=merged_env,
doStepIf=test))
# Install distribution.
f.addStep(
NinjaCommand(name="install",
targets=["stage2-install-distribution"],
haltOnFailure=True,
description=["install"],
workdir=obj_dir,
env=merged_env))
return f
def Clean(**kwargs):
return steps.RemoveDirectory(
name = "clean",
description = "cleaning",
descriptionDone = "clean",
**kwargs)
def getLLDBCMakeBuildFactory(
clean=False,
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=None,
# Default values for VS devenv and build configuration
vs=None,
config='Release',
target_arch='x86',
extra_cmake_args=None,
test=False,
testTimeout=2400,
install=False):
############# PREPARING
# Global configurations
build_dir = 'build'
f = LLVMBuildFactory(depends_on_projects=["llvm", "clang", "lldb", "lld"],
obj_dir=build_dir)
env = {}
# Determine Slave Environment and Set MSVC environment.
if vs and vs != 'manual':
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractVSEnvironment))
env = Property('vs_env')
f.addGetSourcecodeSteps()
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
############# CLEANING
cleanBuildRequested = lambda step: clean or step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj"))
f.addStep(
steps.RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, f.obj_dir)
cmake_options = [
"-G",
"Ninja",
"-DCMAKE_BUILD_TYPE=" + config,
"-DCMAKE_INSTALL_PREFIX=../install",
"-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects),
]
if python_source_dir:
cmake_options.append("-DPYTHON_HOME=" + python_source_dir)
if extra_cmake_args:
cmake_options += extra_cmake_args
f.addStep(
CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
options=cmake_options,
path=rel_src_dir,
env=env,
workdir=build_dir))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=env))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=env))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
timeout=testTimeout,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=env))
return f
def getPollyBuildFactory(
clean=False,
install=False,
make='make',
jobs=None,
checkAll=False,
env=None,
extraCmakeArgs=None,
testsuite=False,extraTestsuiteCmakeArgs=None,
**kwargs):
if extraCmakeArgs is None:
extraCmakeArgs=[]
if extraTestsuiteCmakeArgs is None:
extraTestsuiteCmakeArgs = []
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
llvm_instdir = "llvm.inst"
testsuite_srcdir = "test-suite.src"
testsuite_builddir = "test-suite.build"
jobs_cmd = []
if jobs is not None:
jobs_cmd = ["-j{}".format(jobs)]
build_cmd = [make] + jobs_cmd
install_cmd = [make, 'install'] + jobs_cmd
check_all_cmd = [make, 'check-all'] + jobs_cmd
check_polly_cmd = [make, 'check-polly'] + jobs_cmd
cmake_install = []
if install:
cmake_install = ["-DCMAKE_INSTALL_PREFIX=../%s" % llvm_instdir]
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env:
merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything.
depends_on_projects = ['llvm','clang','polly']
if testsuite:
# XRay tests in test-suite require compiler-rt
depends_on_projects += ['compiler-rt']
cleanBuildRequestedByProperty = lambda step: step.build.getProperty("clean", False)
cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj"))
f = LLVMBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=llvm_objdir,
install_dir=llvm_instdir,
cleanBuildRequested=cleanBuildRequested,
**kwargs) # Pass through all the extra arguments.
f.addStep(steps.RemoveDirectory(name='clean-src-dir',
dir=f.monorepo_dir,
warnOnFailure=True,
doStepIf=cleanBuildRequestedByProperty))
# Get the source code.
f.addGetSourcecodeSteps(**kwargs)
# Clean build dir
f.addStep(steps.RemoveDirectory(name='clean-build-dir',
dir=llvm_objdir,
warnOnFailure=True,
doStepIf=cleanBuildRequested))
# Create configuration files with cmake
cmakeCommand = ["cmake", "../%s/llvm" % llvm_srcdir,
"-DCMAKE_COLOR_MAKEFILE=OFF",
"-DPOLLY_TEST_DISABLE_BAR=ON",
"-DPOLLY_ENABLE_GPGPU_CODEGEN=ON",
"-DCMAKE_BUILD_TYPE=Release",
"-DLLVM_POLLY_LINK_INTO_TOOLS=ON",
"-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects),
] + cmake_install + extraCmakeArgs
f.addStep(ShellCommand(name="cmake-configure",
command=cmakeCommand,
haltOnFailure=False,
description=["cmake configure"],
workdir=llvm_objdir,
env=merged_env))
# Build
f.addStep(WarningCountingShellCommand(name="build",
command=build_cmd,
haltOnFailure=True,
description=["build"],
workdir=llvm_objdir,
env=merged_env))
clangexe = "%(builddir)s/" + llvm_objdir + "/bin/clang"
clangxxexe = "%(builddir)s/" + llvm_objdir + "/bin/clang++"
litexe = "%(builddir)s/" + llvm_objdir + "/bin/llvm-lit"
sizeexe = "%(builddir)s/" + llvm_objdir + "/bin/llvm-size"
# Clean install dir
if install:
f.addStep(steps.RemoveDirectory(name='clean-install-dir',
dir=llvm_instdir,
haltOnFailure=False,
doStepIf=cleanBuildRequested))
f.addStep(ShellCommand(name="install",
command=install_cmd,
haltOnFailure=True,
description=["install"],
workdir=llvm_objdir,
env=merged_env))
# If installing, use the installed version of clang.
clangexe = "%(builddir)s/" + llvm_instdir + "/bin/clang"
clangxxexe = "%(builddir)s/" + llvm_instdir + "/bin/clang++"
sizeexe = "%(builddir)s/" + llvm_instdir + "/bin/llvm-size"
# Test
if checkAll:
f.addStep(LitTestCommand(name="check_all",
command=check_all_cmd,
haltOnFailure=False,
description=["check all"],
workdir=llvm_objdir,
env=merged_env))
else:
f.addStep(LitTestCommand(name="check_polly",
command=check_polly_cmd,
haltOnFailure=False,
description=["check polly"],
workdir=llvm_objdir,
env=merged_env))
if testsuite:
f.addStep(steps.RemoveDirectory(name='test-suite_clean-src-dir',
dir=testsuite_srcdir,
haltOnFailure=False,
warnOnFailure=True,
doStepIf=cleanBuildRequestedByProperty))
f.addGetSourcecodeForProject(
project='test-suite',
src_dir=testsuite_srcdir,
alwaysUseLatest=True)
f.addStep(steps.RemoveDirectory(name='test-suite_clean-build-dir',
dir=testsuite_builddir,
haltOnFailure=False,
warnOnFailure=True))
# -Wno-unused-command-line-argument is needed because linking will not uses the "-mllvm -polly" argument.
f.addStep(ShellCommand(name='test-suite_cmake-configure',
description=["Test-Suite: cmake"],
command=["cmake", '-B', testsuite_builddir, '-S', testsuite_srcdir,
"-DCMAKE_BUILD_TYPE=Release",
"-DTEST_SUITE_COLLECT_STATS=ON",
"-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly",
"-DTEST_SUITE_EXTRA_CXX_FLAGS=-mllvm -polly",
"-DTEST_SUITE_LIT_FLAGS=-vv;-o;report.json",
WithProperties("-DCMAKE_C_COMPILER=" + clangexe),
WithProperties("-DCMAKE_CXX_COMPILER=" + clangxxexe),
WithProperties("-DTEST_SUITE_LLVM_SIZE=" + sizeexe),
WithProperties("-DTEST_SUITE_LIT=" + litexe),
] + extraTestsuiteCmakeArgs,
haltOnFailure=True,
workdir='.',
env=merged_env))
f.addStep(WarningCountingShellCommand(name='test-suite_build',
description=["Test-Suite: build"],
# Continue building; programs that don't compile will fail with NOEXE.
command=[make, 'all', '-k0'] + jobs_cmd,
haltOnFailure=False,
flunkOnFailure=True,
workdir=testsuite_builddir,
env=merged_env))
f.addStep(LitTestCommand(name='test-suite_run',
description=['Test-Suite: run'],
command=[WithProperties(litexe), '-vv', '-o', 'report.json', '.'],
haltOnFailure=True,
workdir=testsuite_builddir,
logfiles={
'test.log' : 'test.log',
'report.json': 'report.json'
},
env=merged_env))
return f
def addCmakeSteps(f,
cleanBuildRequested,
obj_dir,
generator=None,
install_dir=None,
extra_configure_args=None,
env=None,
stage_name=None,
**kwargs):
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if obj_dir is None:
obj_dir = f.obj_dir
# This is an incremental build, unless otherwise has been requested.
# Remove obj and install dirs for a clean build.
# TODO: Some Windows workers do not handle RemoveDirectory command well.
# So, consider running "rmdir /S /Q <dir>" if the build runs on Windows.
f.addStep(
steps.RemoveDirectory(
name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
if install_dir:
install_dir_rel = LLVMBuildFactory.pathRelativeTo(install_dir, obj_dir)
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
])
f.addStep(
steps.RemoveDirectory(
name='clean-%s-dir' % install_dir,
dir=install_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
# Reconcile the cmake options for this build.
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_LIT_ARGS=', '-v -vv'),
])
# Create configuration files with cmake, unless this has been already done
# for an incremental build.
if stage_name:
step_name = "cmake-configure-%s" % stage_name
else:
stage_name = ""
step_name = "cmake-configure"
src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir)
# Make a local copy of the configure args, as we are going to modify that.
definitions = dict()
options = list()
for d in cmake_args:
if isinstance(d, str) and d.startswith("-D"):
k, v = d[2:].split('=', 1)
definitions[k] = v
else:
options.append(d)
f.addStep(
CmakeCommand(
name=step_name,
haltOnFailure=True,
description=["Cmake", "configure", stage_name],
generator=generator,
definitions=definitions,
options=options,
path=src_dir,
env=env or {},
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
def build_weekly():
create_src = steps.MakeDirectory(name="create src directory", dir="src")
clone_step = steps.GitHub(name="fetch PR source",
repourl=util.Property("repository"),
mode="full",
method="fresh",
submodules=True,
clobberOnFailure=True,
getDescription=True,
workdir="src")
rm_src_dir = steps.RemoveDirectory(dir=util.Interpolate(
os.path.join(_WEEKLY_SRC_BASE, "%(prop:branch)s",
"%(prop:commit-description)s")))
copy_src = steps.CopyDirectory(name="copy src to srcdir",
src="src",
dest=util.Interpolate(
os.path.join(
_WEEKLY_SRC_BASE, "%(prop:branch)s",
"%(prop:commit-description)s"), ))
set_merge_property = steps.SetProperty(
property=util.Interpolate("merge_%(prop:branch)s"),
value=True,
hideStepIf=True,
)
# load builders.json with definitions on how to build things
parent_path = os.path.dirname(__file__)
with open(os.path.join(parent_path, "builders.json"),
"r") as builders_file:
build_config = json.loads(builders_file.read())
# now we have all necessary merge properties together,
# we can actually kickoff builds for them
trigger_builds = custom_steps.BuildTrigger(
name="trigger all builders",
build_config=build_config,
schedulerNames=["trigger"],
runner="time",
set_properties={
"src_dir":
util.Interpolate(
os.path.join(_WEEKLY_SRC_BASE, "%(prop:branch)s",
"%(prop:commit-description)s")),
"got_revision":
util.Property("got_revision"),
},
updateSourceStamp=True,
waitForFinish=True)
factory = util.BuildFactory()
factory.addStep(create_src)
factory.addStep(clone_step)
factory.addStep(rm_src_dir)
factory.addStep(copy_src)
factory.addStep(set_merge_property)
factory.addStep(trigger_builds)
return factory
def __init__(self, *args, **kwargs):
util.BuildFactory.__init__(self, *args, **kwargs)
title = 'Liri OS'
releasever = '30'
self.addSteps([
ImagePropertiesStep(name='set properties'),
steps.ShellCommand(
name='update container',
haltOnFailure=True,
command=['dnf', 'update', '-y'],
),
steps.ShellCommand(
name='install tools',
haltOnFailure=True,
command=[
'dnf', 'install', '-y', 'git', 'spin-kickstarts',
'pykickstart', 'livecd-tools'
],
),
steps.Git(
name='checkout sources',
codebase=util.Property('codebase'),
repourl=util.Property('repository'),
branch=util.Property('branch'),
mode='incremental',
submodules=True,
shallow=True,
),
steps.ShellCommand(
name='ksflatten',
haltOnFailure=True,
command=[
'ksflatten',
Interpolate('--config=%(prop:product)s-livecd.ks'), '-o',
'livecd.ks'
],
),
steps.RemoveDirectory(
name='clean cache',
dir='/build/cache',
doStepIf=IsCacheDisabled,
),
steps.ShellCommand(
name='build image',
haltOnFailure=True,
timeout=60 * 60,
command=[
'livecd-creator', '--releasever=' + releasever,
'--config=livecd.ks',
Interpolate('--fslabel=%(prop:imgname)s'), '--title',
title,
Interpolate('--product=%(prop:product)s'),
'--cache=/build/cache'
],
),
steps.ShellCommand(
name='checksum',
haltOnFailure=True,
command=[
'bash', '-c',
Interpolate(
'sha256sum -b --tag %(prop:isofilename)s > /repo/images/nightly/%(prop:checksumfilename)s'
)
],
),
steps.ShellCommand(
name='move file',
command=[
'mv',
Interpolate('%(prop:isofilename)s'),
'/repo/images/nightly/'
],
),
steps.ShellCommand(
name='remove old images',
command=[
'bash', '-c',
'find /repo/images/nightly -type f -mtime +7 -exec rm {} \;'
],
)
])
def build_PR():
# @util.renderer
# def check_mergeable(props):
# mergeable = props.getProperty("github.mergeable", False)
# return mergeable
# check_mergeables = steps.Assert(
# check_mergeable,
# name="check if PR was mergeable",
# haltOnFailure=True
# )
create_src = steps.MakeDirectory(name="create src directory", dir="src")
clone_step = steps.GitHub(name="fetch PR source",
repourl=util.Property("repository"),
mode="full",
method="fresh",
submodules=True,
retryFetch=True,
clobberOnFailure=True,
workdir="src")
set_merge_property = steps.SetProperty(
name="set merge property",
property=util.Interpolate("merge_%(prop:github.base.ref)s"),
value=True,
hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS,
)
create_merge_branch = steps.ShellCommand(
name="create test_merge branch for further steps",
command=[
"git", "branch", "-f",
util.Interpolate("test_merge_%(prop:github.base.ref)s")
],
workdir="src")
rm_src_dir = steps.RemoveDirectory(
dir=util.Interpolate(
os.path.join(_BASEPATH, "pull", "%(prop:github.number)s",
"%(prop:github.base.ref)s")),
hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS,
)
copy_src = steps.CopyDirectory(
name="copy src to srcdir",
src="src",
dest=util.Interpolate(
os.path.join(_BASEPATH, "pull", "%(prop:github.number)s",
"%(prop:github.base.ref)s"), ),
hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS,
)
master_steps = sequences.mergeability_sequence("master", "maint",
_PULL_SRC_BASE)
next_steps = sequences.mergeability_sequence("next", "master",
_PULL_SRC_BASE)
python3_steps = sequences.mergeability_sequence("python3", "next",
_PULL_SRC_BASE)
# load builders.json with definitions on how to build things
parent_path = os.path.dirname(__file__)
with open(os.path.join(parent_path, "builders.json"),
"r") as builders_file:
build_config = json.loads(builders_file.read())
trigger_builds = custom_steps.BuildTrigger(
name="trigger the right builders",
build_config=build_config,
schedulerNames=["trigger"],
runner="pull",
set_properties={
"pr_base":
util.Property("github.base.ref"),
"src_dir":
util.Interpolate(
os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s"))
},
updateSourceStamp=False,
waitForFinish=True)
factory = util.BuildFactory()
factory.addStep(create_src)
factory.addStep(clone_step)
factory.addStep(set_merge_property)
factory.addStep(create_merge_branch)
factory.addStep(rm_src_dir)
factory.addStep(copy_src)
factory.addSteps(master_steps)
factory.addSteps(next_steps)
factory.addSteps(python3_steps)
factory.addStep(trigger_builds)
return factory
