def get_uchroot_workaround_steps(): workaround_steps = [] workaround_steps.append(ucompile('true', name='uchroot /proc bug workaround', hideStepIf=True, haltOnFailure=False, flunkOnWarnings=False, flunkOnFailure=False, warnOnWarnings=False, warnOnFailure=False)) workaround_steps.append(cmd("sleep 1", hideStepIf=True)) return workaround_steps
def configure(c): c['builders'].append( builder("build-benchbuild", None, accepted_builders, tags=['polyjit'], factory=BuildFactory([ git('pjit-benchbuild', 'master', REPOS), cmd("virtualenv", "-p", "python3", "_venv", name="create virtualenv", description="setup benchbuild virtual environment"), cmd("_venv/bin/pip3", "install", ".", name="install benchbuild", description="install benchbuild into the venv", env={'PYTHONPATH': 'benchbuild/bin', 'HOME': '.'}), pylint("_venv/bin/pylint", "./benchbuild/", warnOnFailure=True, flunkOnWarnings=False, flunkOnFailure=False, haltOnFailure=False), pylint("_venv/bin/pylint", "-E", "./benchbuild/", warnOnFailure=True, flunkOnWarnings=False, flunkOnFailure=False, haltOnFailure=False), rmdir("_venv")])))
def get_uchroot_workaround_steps(): workaround_steps = [] workaround_steps.append( ucompile('true', name='uchroot /proc bug workaround', hideStepIf=True, haltOnFailure=False, flunkOnWarnings=False, flunkOnFailure=False, warnOnWarnings=False, warnOnFailure=False)) workaround_steps.append(cmd("sleep 1", hideStepIf=True)) return workaround_steps
def configure(c): steps = [ define("LLVM_ROOT", ip("%(prop:builddir)s/llvm")), define("UCHROOT_SRC_ROOT", "/mnt/llvm"), define("CLANG_ROOT", ip("%(prop:LLVM_ROOT)s/tools/clang")), define("POLLY_ROOT", ip("%(prop:LLVM_ROOT)s/tools/polly")), define("COMPILERRT_ROOT", ip("%(prop:LLVM_ROOT)s/projects/compiler-rt")), define("OPENMP_ROOT", ip("%(prop:LLVM_ROOT)s/projects/openmp")), git('llvm', 'master', codebases, workdir=P("LLVM_ROOT")), git('clang', 'master', codebases, workdir=P("CLANG_ROOT")), git('polly', 'master', codebases, workdir=P("POLLY_ROOT")), git('compiler-rt', 'master', codebases, workdir=P("COMPILERRT_ROOT")), git('openmp', 'master', codebases, workdir=P("OPENMP_ROOT")), ucmd('cmake', P("UCHROOT_SRC_ROOT"), '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_INSTALL_PREFIX=./_install', '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -DLLVM_ENABLE_STATS', '-DBUILD_SHARED_LIBS=Off', '-DPOLLY_BUILD_POLLI=Off', '-DLLVM_POLLY_LINK_INTO_TOOLS=OFF', '-DLLVM_TARGETS_TO_BUILD=X86', '-DLLVM_BINUTILS_INCDIR=/usr/include', '-DLLVM_ENABLE_PIC=On', '-DLLVM_ENABLE_ASSERTIONS=On', '-DLLVM_ENABLE_TERMINFO=Off', '-DCLANG_DEFAULT_OPENMP_RUNTIME=libomp', '-G', 'Ninja', env={ "PATH": "/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin" }, name="cmake", description="cmake O3, Assertions, PIC, Static"), ucompile("ninja", "install", haltOnFailure=True, name="build llvm"), cmd("tar", "czf", "../llvm.tar.gz", "-C", "./_install", ".") ] upload_llvm = hash_upload_to_master("llvm.tar.gz", "../llvm.tar.gz", "public_html/llvm.tar.gz", URL) steps.extend(upload_llvm) c['builders'].append(builder("build-llvm", None, accepted_builders, tags=['polyjit'], factory=BuildFactory(steps)))
def configure(c): f = util.BuildFactory() # TODO Check if this can be done without a dummy command #f.addStep(GenerateGitCloneCommand()) f.addStep(GenerateGitCloneCommand(name="Dummy_1", command=['true'], haltOnFailure=True, hideStepIf=True)) f.addStep(define('UCHROOT_SRC_ROOT', UCHROOT_SRC_ROOT)) f.addStep(define('UCHROOT_BUILD_DIR', UCHROOT_BUILD_DIR)) f.addStep(GenerateGitCheckoutCommand( name="Get branch names", command=['./tools/VaRA/utils/buildbot/bb-get-branches.sh'], workdir=ip(CHECKOUT_BASE_DIR), haltOnFailure=True, hideStepIf=True)) # CMake for step in get_uchroot_workaround_steps(): f.addStep(step) f.addStep(ucompile('../tools/VaRA/utils/vara/builds/' + BUILD_SCRIPT, env={'PATH': '/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin'}, name='cmake', description=BUILD_SCRIPT, workdir=UCHROOT_SRC_ROOT + '/build')) f.addStep(GenerateMakeCleanCommand(name="Dummy_2", command=['true'], haltOnFailure=True, hideStepIf=True)) # use mergecheck tool to make sure the 'upstream' remote is present for repo in ['vara-llvm', 'vara-clang']: f.addStep(steps.Compile( command=['/scratch/pjtest/mergecheck/build/bin/mergecheck', 'rebase', '--repo', '.' + REPOS[repo]['checkout_subdir'], '--remote-url', REPOS[repo]['upstream_remote_url'], '--remote-name', 'upstream', '--upstream', 'refs/remotes/upstream/master', '--branch', 'refs/remotes/upstream/master', '-v'], workdir=ip(CHECKOUT_BASE_DIR), name='Add upstream remote to repository.', hideStepIf=True)) # Prepare project file list to filter out compiler warnings f.addStep(cmd("../../tools/VaRA/utils/vara/getVaraSourceFiles.sh", "--all", "--include-existing", "--relative-to", ip(BUILD_DIR), "--output", "buildbot-source-file-list.txt", workdir=ip(BUILD_DIR), hideStepIf=True)) # Compile Step f.addStep(GenerateBuildStepCommand(name="Dummy_3", command=['cat', 'buildbot-source-file-list.txt'], workdir=ip(BUILD_DIR), haltOnFailure=True, hideStepIf=True)) # Regression Test step for step in get_uchroot_workaround_steps(): f.addStep(step) f.addStep(ucompile('ninja', 'check-vara', name='run VaRA regression tests', workdir=UCHROOT_BUILD_DIR, haltOnFailure=False, warnOnWarnings=True)) # Clang-Tidy for step in get_uchroot_workaround_steps(): f.addStep(step) f.addStep(ucompile('python3', 'tidy-vara.py', '-p', UCHROOT_BUILD_DIR, '-j', '8', '--gcc', workdir='vara-llvm/tools/VaRA/test/', name='run Clang-Tidy', haltOnFailure=False, warnOnWarnings=True, env={'PATH': [UCHROOT_BUILD_DIR + "/bin", "${PATH}"]}, timeout=3600)) # ClangFormat f.addStep(GenerateClangFormatStepCommand(name="Dummy_4", command=['opt/clang-format-static/clang-format', '-version'], workdir=ip('%(prop:uchroot_image_path)s'), haltOnFailure=True, hideStepIf=True)) # Mergecheck for repo in ['vara-llvm', 'vara-clang', 'vara']: f.addStep(define('mergecheck_repo', repo)) f.addStep(GenerateMergecheckCommand(name="Dummy_5", command=['git', 'symbolic-ref', 'HEAD'], workdir=ip(REPOS[repo]['checkout_dir']), haltOnFailure=True, hideStepIf=True)) c['builders'].append(builder(PROJECT_NAME, None, ACCEPTED_BUILDERS, tags=['vara'], factory=f))
def configure(c): f = util.BuildFactory() # TODO Check if this can be done without a dummy command #f.addStep(GenerateGitCloneCommand()) f.addStep(GenerateGitCloneCommand(name="Dummy_1", command=['true'], haltOnFailure=True, hideStepIf=True)) f.addStep(define('UCHROOT_SRC_ROOT', UCHROOT_SRC_ROOT)) f.addStep(define('UCHROOT_BUILD_DIR', UCHROOT_BUILD_DIR)) f.addStep(GenerateGitCheckoutCommand( name="Get branch names", command=['./tools/VaRA/utils/buildbot/bb-get-branches.sh'], workdir=ip(CHECKOUT_BASE_DIR), haltOnFailure=True, hideStepIf=True)) # CMake for step in get_uchroot_workaround_steps(): f.addStep(step) f.addStep(ucompile('../tools/VaRA/utils/vara/builds/' + BUILD_SCRIPT, env={'PATH': '/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin'}, name='cmake', description=BUILD_SCRIPT, workdir=UCHROOT_SRC_ROOT + '/build')) f.addStep(GenerateMakeCleanCommand(name="Dummy_2", command=['true'], haltOnFailure=True, hideStepIf=True)) # use mergecheck tool to make sure the 'upstream' remote is present for repo in ['vara-llvm', 'vara-clang']: f.addStep(steps.Compile( command=['/local/hdd/buildbot/mergecheck/build/bin/mergecheck', 'rebase', '--repo', '.' + REPOS[repo]['checkout_subdir'], '--remote-url', REPOS[repo]['upstream_remote_url'], '--remote-name', 'upstream', '--upstream', 'refs/remotes/upstream/master', '--branch', 'refs/remotes/upstream/master', '-v'], workdir=ip(CHECKOUT_BASE_DIR), name='Add upstream remote to repository.', hideStepIf=True)) # Prepare project file list to filter out compiler warnings f.addStep(cmd("../../tools/VaRA/utils/vara/getVaraSourceFiles.sh", "--vara", "--clang", "--llvm", "--include-existing", "--relative-to", ip(BUILD_DIR), "--output", "buildbot-source-file-list.txt", workdir=ip(BUILD_DIR), hideStepIf=True)) # Compile Step f.addStep(GenerateBuildStepCommand(name="Dummy_3", command=['cat', 'buildbot-source-file-list.txt'], workdir=ip(BUILD_DIR), haltOnFailure=True, hideStepIf=True)) # Regression Test step for step in get_uchroot_workaround_steps(): f.addStep(step) f.addStep(ucompile('ninja', 'check-vara', name='run VaRA regression tests', workdir=UCHROOT_BUILD_DIR, haltOnFailure=False, warnOnWarnings=True)) # Clang-Tidy for step in get_uchroot_workaround_steps(): f.addStep(step) f.addStep(ucompile('python3', 'tidy-vara.py', '-p', UCHROOT_BUILD_DIR, '-j', '8', '--gcc', workdir='vara-llvm/tools/VaRA/test/', name='run Clang-Tidy', haltOnFailure=False, warnOnWarnings=True, timeout=3600)) # ClangFormat f.addStep(GenerateClangFormatStepCommand(name="Dummy_4", command=['opt/clang-format-static/clang-format', '-version'], workdir=ip('%(prop:uchroot_image_path)s'), haltOnFailure=True, hideStepIf=True)) c['builders'].append(builder(PROJECT_NAME, None, ACCEPTED_BUILDERS, tags=['vara'], factory=f))
def configure(c): llvm_dl = hash_download_from_master("public_html/llvm.tar.gz", "llvm.tar.gz", "llvm") polyjit_dl = hash_download_from_master("public_html/polyjit.tar.gz", "polyjit.tar.gz", "polyjit") steps = [ # trigger(schedulerNames=['trigger-build-llvm', 'trigger-build-jit']), define("scratch", ip("/scratch/pjtest/%(prop:buildnumber)s/")) ] steps.extend(llvm_dl) steps.extend(clean_unpack("llvm.tar.gz", "llvm")) steps.extend(polyjit_dl) steps.extend(clean_unpack("polyjit.tar.gz", "polyjit")) steps.extend([ define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")), git('benchbuild', 'master', codebases, workdir=P("BENCHBUILD_ROOT")), ]) steps.extend([ define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')), define('llvm', ip('%(prop:scratch)s/llvm')), define('polyjit', ip('%(prop:scratch)s/polyjit')), mkdir(P("scratch")), cmd('virtualenv', '-ppython3', ip('%(prop:scratch)s/env/')), cmd(ip('%(prop:scratch)s/env/bin/pip3'), 'install', '--upgrade', '.', workdir='build/benchbuild'), cmd("rsync", "-var", "llvm", P("scratch")), cmd("rsync", "-var", "polyjit", P("scratch")), cmd(P('benchbuild'), 'bootstrap', '-s', env={ 'BB_CONFIG_FILE': '/scratch/pjtest/.benchbuild.json', 'BB_TMP_DIR': '/scratch/pjtest/src/', 'BB_TEST_DIR': P("testinputs"), 'BB_GENTOO_HTTP_PROXY': 'debussy.fim.uni-passau.de:3128', 'BB_GENTOO_FTP_PROXY': 'debussy.fim.uni-passau.de:3128', 'BB_GENTOO_AUTOTEST_LOC': '/scratch/pjtest/gentoo-autotest', 'BB_SLURM_PARTITION': 'chimaira', 'BB_SLURM_NODE_DIR': '/local/hdd/pjtest/', 'BB_SLURM_ACCOUNT': 'cl', 'BB_SLURM_TIMELIMIT': '24:00:00', 'BB_ENV_COMPILER_PATH': ip('["%(prop:llvm)s/bin", "%(prop:polyjit)s/bin"]'), 'BB_ENV_COMPILER_LD_LIBRARY_PATH': ip('["%(prop:llvm)s/lib", "%(prop:polyjit)s/lib"]'), 'BB_ENV_BINARY_PATH': ip('["%(prop:llvm)s/bin", "%(prop:polyjit)s/bin"]'), 'BB_ENV_BINARY_LD_LIBRARY_PATH': ip('["%(prop:llvm)s/lib", "%(prop:polyjit)s/lib"]'), 'BB_ENV_LOOKUP_PATH': ip('["%(prop:llvm)s/bin", "%(prop:polyjit)s/bin", "/scratch/pjtest/erlent/build"]'), 'BB_ENV_LOOKUP_LD_LIBRARY_PATH': ip('["%(prop:llvm)s/lib", "%(prop:polyjit)s/lib"]'), 'BB_LLVM_DIR': ip('%(prop:scratch)s/llvm'), 'BB_LIKWID_PREFIX': '/usr', 'BB_PAPI_INCLUDE': '/usr/include', 'BB_PAPI_LIBRARY': '/usr/lib', 'BB_SRC_DIR': ip('%(prop:scratch)s/benchbuild'), 'BB_SLURM_LOGS': ip('%(prop:scratch)s/slurm.log'), 'BB_UNIONFS_ENABLE': 'false' }, workdir=P('scratch')), # This only works on infosun machines cmd("ln", "-s", ip("/scratch/pjtest/benchbuild-src/"), ip("%(prop:scratch)s/benchbuild")), mkdir(ip("%(prop:scratch)s/results"))]) c['builders'].append(builder("build-slurm-set", None, accepted_builders, tags=['polyjit'], factory=BuildFactory(steps)))
def configure(c): sb_steps = [ define("SUPERBUILD_ROOT", ip("%(prop:builddir)s/polli-sb")), define("POLYJIT_DEFAULT_BRANCH", "master"), git('pjit-polli-sb', 'master', REPOS, workdir=P("SUPERBUILD_ROOT"), mode="full", method="fresh"), ICMD_P('%(prop:cmake_prefix)s/bin/cmake', P("SUPERBUILD_ROOT"), '-DCMAKE_CXX_COMPILER=%(prop:cxx)s', '-DCMAKE_C_COMPILER=%(prop:cc)s', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_INSTALL_PREFIX=./_install', '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -DLLVM_ENABLE_STATS', '-DBUILD_SHARED_LIBS=Off', '-DPOLYJIT_PAPI_PREFIX=/scratch/pjtest/opt/papi-5.5.1/install', '-DPOLYJIT_BRANCH_CLANG=%(prop:POLYJIT_DEFAULT_BRANCH)s', '-DPOLYJIT_BRANCH_LLVM=%(prop:POLYJIT_DEFAULT_BRANCH)s', '-DPOLYJIT_BRANCH_POLLI=%(prop:POLYJIT_DEFAULT_BRANCH)s', '-DPOLYJIT_BRANCH_POLLY=%(prop:POLYJIT_DEFAULT_BRANCH)s', '-G', 'Ninja', name="cmake", description="[uchroot] cmake: release build", descriptionDone="[uchroot] configured."), CMD_P("/usr/bin/ninja", "llvm-configure", name="configure LLVM", description="configuring LLVM", descriptionDone="configured LLVM"), CMD_P("/usr/bin/ninja", "polli-configure", name="configure PolyJIT", description="configuring PolyJIT", descriptionDone="configured PolyJIT"), CMD_P("/usr/bin/ninja", name="build jit", description="[uchroot] building PolyJIT", descriptionDone="[uchroot] built PolyJIT"), cmd("tar", "czf", "../polyjit_sb.tar.gz", "-C", "./_install", ".") ] sb_steps.extend(hash_upload_to_master( "polyjit_sb.tar.gz", "../polyjit_sb.tar.gz", "public_html/polyjit_sb.tar.gz", URL)) slurm_steps = [ define("scratch", ip("/scratch/pjtest/sb-%(prop:buildnumber)s/")), *hash_download_from_master("public_html/polyjit_sb.tar.gz", "polyjit_sb.tar.gz", "polyjit"), *clean_unpack("polyjit_sb.tar.gz", "llvm"), define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")), git('pjit-benchbuild', 'master', REPOS, workdir=P("BENCHBUILD_ROOT")), define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')), define('llvm', ip('%(prop:scratch)s/llvm')), define('bb_src', ip('%(prop:scratch)s/benchbuild')), icmd('virtualenv', '-ppython3', '%(prop:scratch)s/env'), icmd('%(prop:scratch)s/env/bin/pip3', 'install', P("BENCHBUILD_ROOT")), mkdir(P("scratch")), cmd("rsync", "-var", "llvm", P("scratch")), cmd(P('benchbuild'), 'bootstrap', '-s', logEnviron=True, env=BB_ENV, workdir=P('scratch')), # This only works on infosun machines icmd("ln", "-s", "/scratch/pjtest/benchbuild-src/", "%(prop:bb_src)s"), mkdir(ip("%(prop:scratch)s/results")) ] c['builders'].extend([ builder("polyjit-superbuild", None, ACCEPTED_BUILDERS, tags=['polyjit'], collapseRequests=True, factory=BF(sb_steps)), builder("polyjit-superbuild-slurm", None, ACCEPTED_BUILDERS, tags=['polyjit'], collapseRequests=True, factory=BF(slurm_steps)) ])
def icmd(*args, **kwargs): ip_list = [ ip(le) if isinstance(le, str) else le for le in args] return cmd(*ip_list, **kwargs)