def get_uchroot_workaround_steps():
workaround_steps = []
workaround_steps.append(ucompile('true', name='uchroot /proc bug workaround', hideStepIf=True,
haltOnFailure=False, flunkOnWarnings=False,
flunkOnFailure=False, warnOnWarnings=False,
warnOnFailure=False))
workaround_steps.append(cmd("sleep 1", hideStepIf=True))
return workaround_steps
def configure(c):
c['builders'].append(
builder("build-benchbuild", None, accepted_builders,
tags=['polyjit'],
factory=BuildFactory([
git('pjit-benchbuild', 'master', REPOS),
cmd("virtualenv", "-p", "python3", "_venv",
name="create virtualenv",
description="setup benchbuild virtual environment"),
cmd("_venv/bin/pip3", "install", ".",
name="install benchbuild",
description="install benchbuild into the venv",
env={'PYTHONPATH': 'benchbuild/bin',
'HOME': '.'}),
pylint("_venv/bin/pylint", "./benchbuild/",
warnOnFailure=True, flunkOnWarnings=False,
flunkOnFailure=False, haltOnFailure=False),
pylint("_venv/bin/pylint", "-E", "./benchbuild/",
warnOnFailure=True, flunkOnWarnings=False,
flunkOnFailure=False, haltOnFailure=False),
rmdir("_venv")])))
def get_uchroot_workaround_steps():
workaround_steps = []
workaround_steps.append(
ucompile('true',
name='uchroot /proc bug workaround',
hideStepIf=True,
haltOnFailure=False,
flunkOnWarnings=False,
flunkOnFailure=False,
warnOnWarnings=False,
warnOnFailure=False))
workaround_steps.append(cmd("sleep 1", hideStepIf=True))
return workaround_steps
def configure(c):
steps = [
define("LLVM_ROOT", ip("%(prop:builddir)s/llvm")),
define("UCHROOT_SRC_ROOT", "/mnt/llvm"),
define("CLANG_ROOT", ip("%(prop:LLVM_ROOT)s/tools/clang")),
define("POLLY_ROOT", ip("%(prop:LLVM_ROOT)s/tools/polly")),
define("COMPILERRT_ROOT", ip("%(prop:LLVM_ROOT)s/projects/compiler-rt")),
define("OPENMP_ROOT", ip("%(prop:LLVM_ROOT)s/projects/openmp")),
git('llvm', 'master', codebases, workdir=P("LLVM_ROOT")),
git('clang', 'master', codebases, workdir=P("CLANG_ROOT")),
git('polly', 'master', codebases, workdir=P("POLLY_ROOT")),
git('compiler-rt', 'master', codebases, workdir=P("COMPILERRT_ROOT")),
git('openmp', 'master', codebases, workdir=P("OPENMP_ROOT")),
ucmd('cmake', P("UCHROOT_SRC_ROOT"),
'-DCMAKE_BUILD_TYPE=Release',
'-DCMAKE_INSTALL_PREFIX=./_install',
'-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -DLLVM_ENABLE_STATS',
'-DBUILD_SHARED_LIBS=Off',
'-DPOLLY_BUILD_POLLI=Off',
'-DLLVM_POLLY_LINK_INTO_TOOLS=OFF',
'-DLLVM_TARGETS_TO_BUILD=X86',
'-DLLVM_BINUTILS_INCDIR=/usr/include',
'-DLLVM_ENABLE_PIC=On',
'-DLLVM_ENABLE_ASSERTIONS=On',
'-DLLVM_ENABLE_TERMINFO=Off',
'-DCLANG_DEFAULT_OPENMP_RUNTIME=libomp',
'-G', 'Ninja',
env={
"PATH": "/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin"
},
name="cmake",
description="cmake O3, Assertions, PIC, Static"),
ucompile("ninja", "install", haltOnFailure=True, name="build llvm"),
cmd("tar", "czf", "../llvm.tar.gz", "-C", "./_install", ".")
]
upload_llvm = hash_upload_to_master("llvm.tar.gz",
"../llvm.tar.gz", "public_html/llvm.tar.gz", URL)
steps.extend(upload_llvm)
c['builders'].append(builder("build-llvm", None, accepted_builders,
tags=['polyjit'], factory=BuildFactory(steps)))
def configure(c):
f = util.BuildFactory()
# TODO Check if this can be done without a dummy command
#f.addStep(GenerateGitCloneCommand())
f.addStep(GenerateGitCloneCommand(name="Dummy_1", command=['true'],
haltOnFailure=True, hideStepIf=True))
f.addStep(define('UCHROOT_SRC_ROOT', UCHROOT_SRC_ROOT))
f.addStep(define('UCHROOT_BUILD_DIR', UCHROOT_BUILD_DIR))
f.addStep(GenerateGitCheckoutCommand(
name="Get branch names",
command=['./tools/VaRA/utils/buildbot/bb-get-branches.sh'], workdir=ip(CHECKOUT_BASE_DIR),
haltOnFailure=True, hideStepIf=True))
# CMake
for step in get_uchroot_workaround_steps():
f.addStep(step)
f.addStep(ucompile('../tools/VaRA/utils/vara/builds/' + BUILD_SCRIPT,
env={'PATH': '/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin'},
name='cmake',
description=BUILD_SCRIPT,
workdir=UCHROOT_SRC_ROOT + '/build'))
f.addStep(GenerateMakeCleanCommand(name="Dummy_2", command=['true'],
haltOnFailure=True, hideStepIf=True))
# use mergecheck tool to make sure the 'upstream' remote is present
for repo in ['vara-llvm', 'vara-clang']:
f.addStep(steps.Compile(
command=['/scratch/pjtest/mergecheck/build/bin/mergecheck', 'rebase',
'--repo', '.' + REPOS[repo]['checkout_subdir'],
'--remote-url', REPOS[repo]['upstream_remote_url'],
'--remote-name', 'upstream',
'--upstream', 'refs/remotes/upstream/master',
'--branch', 'refs/remotes/upstream/master',
'-v'],
workdir=ip(CHECKOUT_BASE_DIR),
name='Add upstream remote to repository.', hideStepIf=True))
# Prepare project file list to filter out compiler warnings
f.addStep(cmd("../../tools/VaRA/utils/vara/getVaraSourceFiles.sh",
"--all", "--include-existing",
"--relative-to", ip(BUILD_DIR),
"--output", "buildbot-source-file-list.txt",
workdir=ip(BUILD_DIR), hideStepIf=True))
# Compile Step
f.addStep(GenerateBuildStepCommand(name="Dummy_3",
command=['cat', 'buildbot-source-file-list.txt'],
workdir=ip(BUILD_DIR),
haltOnFailure=True, hideStepIf=True))
# Regression Test step
for step in get_uchroot_workaround_steps():
f.addStep(step)
f.addStep(ucompile('ninja', 'check-vara', name='run VaRA regression tests',
workdir=UCHROOT_BUILD_DIR,
haltOnFailure=False, warnOnWarnings=True))
# Clang-Tidy
for step in get_uchroot_workaround_steps():
f.addStep(step)
f.addStep(ucompile('python3', 'tidy-vara.py', '-p', UCHROOT_BUILD_DIR, '-j', '8', '--gcc',
workdir='vara-llvm/tools/VaRA/test/', name='run Clang-Tidy',
haltOnFailure=False, warnOnWarnings=True,
env={'PATH': [UCHROOT_BUILD_DIR + "/bin", "${PATH}"]}, timeout=3600))
# ClangFormat
f.addStep(GenerateClangFormatStepCommand(name="Dummy_4",
command=['opt/clang-format-static/clang-format',
'-version'],
workdir=ip('%(prop:uchroot_image_path)s'),
haltOnFailure=True, hideStepIf=True))
# Mergecheck
for repo in ['vara-llvm', 'vara-clang', 'vara']:
f.addStep(define('mergecheck_repo', repo))
f.addStep(GenerateMergecheckCommand(name="Dummy_5", command=['git', 'symbolic-ref', 'HEAD'],
workdir=ip(REPOS[repo]['checkout_dir']),
haltOnFailure=True, hideStepIf=True))
c['builders'].append(builder(PROJECT_NAME, None, ACCEPTED_BUILDERS, tags=['vara'],
factory=f))
def configure(c):
f = util.BuildFactory()
# TODO Check if this can be done without a dummy command
#f.addStep(GenerateGitCloneCommand())
f.addStep(GenerateGitCloneCommand(name="Dummy_1", command=['true'],
haltOnFailure=True, hideStepIf=True))
f.addStep(define('UCHROOT_SRC_ROOT', UCHROOT_SRC_ROOT))
f.addStep(define('UCHROOT_BUILD_DIR', UCHROOT_BUILD_DIR))
f.addStep(GenerateGitCheckoutCommand(
name="Get branch names",
command=['./tools/VaRA/utils/buildbot/bb-get-branches.sh'], workdir=ip(CHECKOUT_BASE_DIR),
haltOnFailure=True, hideStepIf=True))
# CMake
for step in get_uchroot_workaround_steps():
f.addStep(step)
f.addStep(ucompile('../tools/VaRA/utils/vara/builds/' + BUILD_SCRIPT,
env={'PATH': '/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin'},
name='cmake',
description=BUILD_SCRIPT,
workdir=UCHROOT_SRC_ROOT + '/build'))
f.addStep(GenerateMakeCleanCommand(name="Dummy_2", command=['true'],
haltOnFailure=True, hideStepIf=True))
# use mergecheck tool to make sure the 'upstream' remote is present
for repo in ['vara-llvm', 'vara-clang']:
f.addStep(steps.Compile(
command=['/local/hdd/buildbot/mergecheck/build/bin/mergecheck', 'rebase',
'--repo', '.' + REPOS[repo]['checkout_subdir'],
'--remote-url', REPOS[repo]['upstream_remote_url'],
'--remote-name', 'upstream',
'--upstream', 'refs/remotes/upstream/master',
'--branch', 'refs/remotes/upstream/master',
'-v'],
workdir=ip(CHECKOUT_BASE_DIR),
name='Add upstream remote to repository.', hideStepIf=True))
# Prepare project file list to filter out compiler warnings
f.addStep(cmd("../../tools/VaRA/utils/vara/getVaraSourceFiles.sh",
"--vara", "--clang", "--llvm",
"--include-existing",
"--relative-to", ip(BUILD_DIR),
"--output", "buildbot-source-file-list.txt",
workdir=ip(BUILD_DIR), hideStepIf=True))
# Compile Step
f.addStep(GenerateBuildStepCommand(name="Dummy_3",
command=['cat', 'buildbot-source-file-list.txt'],
workdir=ip(BUILD_DIR),
haltOnFailure=True, hideStepIf=True))
# Regression Test step
for step in get_uchroot_workaround_steps():
f.addStep(step)
f.addStep(ucompile('ninja', 'check-vara', name='run VaRA regression tests',
workdir=UCHROOT_BUILD_DIR,
haltOnFailure=False, warnOnWarnings=True))
# Clang-Tidy
for step in get_uchroot_workaround_steps():
f.addStep(step)
f.addStep(ucompile('python3', 'tidy-vara.py', '-p', UCHROOT_BUILD_DIR, '-j', '8', '--gcc',
workdir='vara-llvm/tools/VaRA/test/', name='run Clang-Tidy',
haltOnFailure=False, warnOnWarnings=True,
timeout=3600))
# ClangFormat
f.addStep(GenerateClangFormatStepCommand(name="Dummy_4",
command=['opt/clang-format-static/clang-format',
'-version'],
workdir=ip('%(prop:uchroot_image_path)s'),
haltOnFailure=True, hideStepIf=True))
c['builders'].append(builder(PROJECT_NAME, None, ACCEPTED_BUILDERS, tags=['vara'],
factory=f))
def configure(c):
llvm_dl = hash_download_from_master("public_html/llvm.tar.gz",
"llvm.tar.gz", "llvm")
polyjit_dl = hash_download_from_master("public_html/polyjit.tar.gz",
"polyjit.tar.gz", "polyjit")
steps = [
# trigger(schedulerNames=['trigger-build-llvm', 'trigger-build-jit']),
define("scratch", ip("/scratch/pjtest/%(prop:buildnumber)s/"))
]
steps.extend(llvm_dl)
steps.extend(clean_unpack("llvm.tar.gz", "llvm"))
steps.extend(polyjit_dl)
steps.extend(clean_unpack("polyjit.tar.gz", "polyjit"))
steps.extend([
define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")),
git('benchbuild', 'master', codebases, workdir=P("BENCHBUILD_ROOT")),
])
steps.extend([
define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')),
define('llvm', ip('%(prop:scratch)s/llvm')),
define('polyjit', ip('%(prop:scratch)s/polyjit')),
mkdir(P("scratch")),
cmd('virtualenv', '-ppython3', ip('%(prop:scratch)s/env/')),
cmd(ip('%(prop:scratch)s/env/bin/pip3'), 'install', '--upgrade', '.',
workdir='build/benchbuild'),
cmd("rsync", "-var", "llvm", P("scratch")),
cmd("rsync", "-var", "polyjit", P("scratch")),
cmd(P('benchbuild'), 'bootstrap', '-s', env={
'BB_CONFIG_FILE': '/scratch/pjtest/.benchbuild.json',
'BB_TMP_DIR': '/scratch/pjtest/src/',
'BB_TEST_DIR': P("testinputs"),
'BB_GENTOO_HTTP_PROXY': 'debussy.fim.uni-passau.de:3128',
'BB_GENTOO_FTP_PROXY': 'debussy.fim.uni-passau.de:3128',
'BB_GENTOO_AUTOTEST_LOC': '/scratch/pjtest/gentoo-autotest',
'BB_SLURM_PARTITION': 'chimaira',
'BB_SLURM_NODE_DIR': '/local/hdd/pjtest/',
'BB_SLURM_ACCOUNT': 'cl',
'BB_SLURM_TIMELIMIT': '24:00:00',
'BB_ENV_COMPILER_PATH': ip('["%(prop:llvm)s/bin", "%(prop:polyjit)s/bin"]'),
'BB_ENV_COMPILER_LD_LIBRARY_PATH':
ip('["%(prop:llvm)s/lib", "%(prop:polyjit)s/lib"]'),
'BB_ENV_BINARY_PATH': ip('["%(prop:llvm)s/bin", "%(prop:polyjit)s/bin"]'),
'BB_ENV_BINARY_LD_LIBRARY_PATH':
ip('["%(prop:llvm)s/lib", "%(prop:polyjit)s/lib"]'),
'BB_ENV_LOOKUP_PATH':
ip('["%(prop:llvm)s/bin", "%(prop:polyjit)s/bin", "/scratch/pjtest/erlent/build"]'),
'BB_ENV_LOOKUP_LD_LIBRARY_PATH':
ip('["%(prop:llvm)s/lib", "%(prop:polyjit)s/lib"]'),
'BB_LLVM_DIR': ip('%(prop:scratch)s/llvm'),
'BB_LIKWID_PREFIX': '/usr',
'BB_PAPI_INCLUDE': '/usr/include',
'BB_PAPI_LIBRARY': '/usr/lib',
'BB_SRC_DIR': ip('%(prop:scratch)s/benchbuild'),
'BB_SLURM_LOGS': ip('%(prop:scratch)s/slurm.log'),
'BB_UNIONFS_ENABLE': 'false'
},
workdir=P('scratch')),
# This only works on infosun machines
cmd("ln", "-s", ip("/scratch/pjtest/benchbuild-src/"),
ip("%(prop:scratch)s/benchbuild")),
mkdir(ip("%(prop:scratch)s/results"))])
c['builders'].append(builder("build-slurm-set", None, accepted_builders,
tags=['polyjit'], factory=BuildFactory(steps)))
def configure(c):
sb_steps = [
define("SUPERBUILD_ROOT", ip("%(prop:builddir)s/polli-sb")),
define("POLYJIT_DEFAULT_BRANCH", "master"),
git('pjit-polli-sb', 'master', REPOS, workdir=P("SUPERBUILD_ROOT"),
mode="full", method="fresh"),
ICMD_P('%(prop:cmake_prefix)s/bin/cmake', P("SUPERBUILD_ROOT"),
'-DCMAKE_CXX_COMPILER=%(prop:cxx)s',
'-DCMAKE_C_COMPILER=%(prop:cc)s',
'-DCMAKE_BUILD_TYPE=Release',
'-DCMAKE_INSTALL_PREFIX=./_install',
'-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -DLLVM_ENABLE_STATS',
'-DBUILD_SHARED_LIBS=Off',
'-DPOLYJIT_PAPI_PREFIX=/scratch/pjtest/opt/papi-5.5.1/install',
'-DPOLYJIT_BRANCH_CLANG=%(prop:POLYJIT_DEFAULT_BRANCH)s',
'-DPOLYJIT_BRANCH_LLVM=%(prop:POLYJIT_DEFAULT_BRANCH)s',
'-DPOLYJIT_BRANCH_POLLI=%(prop:POLYJIT_DEFAULT_BRANCH)s',
'-DPOLYJIT_BRANCH_POLLY=%(prop:POLYJIT_DEFAULT_BRANCH)s',
'-G', 'Ninja',
name="cmake",
description="[uchroot] cmake: release build",
descriptionDone="[uchroot] configured."),
CMD_P("/usr/bin/ninja", "llvm-configure",
name="configure LLVM",
description="configuring LLVM",
descriptionDone="configured LLVM"),
CMD_P("/usr/bin/ninja", "polli-configure",
name="configure PolyJIT",
description="configuring PolyJIT",
descriptionDone="configured PolyJIT"),
CMD_P("/usr/bin/ninja",
name="build jit",
description="[uchroot] building PolyJIT",
descriptionDone="[uchroot] built PolyJIT"),
cmd("tar", "czf", "../polyjit_sb.tar.gz", "-C", "./_install", ".")
]
sb_steps.extend(hash_upload_to_master(
"polyjit_sb.tar.gz",
"../polyjit_sb.tar.gz",
"public_html/polyjit_sb.tar.gz", URL))
slurm_steps = [
define("scratch", ip("/scratch/pjtest/sb-%(prop:buildnumber)s/")),
*hash_download_from_master("public_html/polyjit_sb.tar.gz", "polyjit_sb.tar.gz", "polyjit"),
*clean_unpack("polyjit_sb.tar.gz", "llvm"),
define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")),
git('pjit-benchbuild', 'master', REPOS, workdir=P("BENCHBUILD_ROOT")),
define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')),
define('llvm', ip('%(prop:scratch)s/llvm')),
define('bb_src', ip('%(prop:scratch)s/benchbuild')),
icmd('virtualenv', '-ppython3', '%(prop:scratch)s/env'),
icmd('%(prop:scratch)s/env/bin/pip3', 'install', P("BENCHBUILD_ROOT")),
mkdir(P("scratch")),
cmd("rsync", "-var", "llvm", P("scratch")),
cmd(P('benchbuild'), 'bootstrap', '-s', logEnviron=True, env=BB_ENV, workdir=P('scratch')),
# This only works on infosun machines
icmd("ln", "-s", "/scratch/pjtest/benchbuild-src/", "%(prop:bb_src)s"),
mkdir(ip("%(prop:scratch)s/results"))
]
c['builders'].extend([
builder("polyjit-superbuild", None,
ACCEPTED_BUILDERS, tags=['polyjit'],
collapseRequests=True,
factory=BF(sb_steps)),
builder("polyjit-superbuild-slurm", None,
ACCEPTED_BUILDERS, tags=['polyjit'],
collapseRequests=True,
factory=BF(slurm_steps))
])
def icmd(*args, **kwargs):
ip_list = [ ip(le) if isinstance(le, str) else le for le in args]
return cmd(*ip_list, **kwargs)