def run(self): command = yield self.makeRemoteShellCommand() yield self.runCommand(command) result = command.results() if result == util.SUCCESS: checkout_feature_br = self.get_feature_branch(self.observer.getStdout()) if checkout_feature_br: self.build.addStepsAfterCurrentStep([ define('FEATURE', checkout_feature_br), define('branch', checkout_feature_br), steps.ShellCommand( name='Checking out feature branch \"' + str(checkout_feature_br) + '\"', command=['./tools/VaRA/utils/buildbot/bb-checkout-branches.sh', checkout_feature_br], workdir=ip(CHECKOUT_BASE_DIR)), ]) else: force_feature = None if self.hasProperty('options'): options = self.getProperty('options') force_feature = options['force_feature'] self.build.addStepsAfterCurrentStep([ define('FEATURE', force_feature), define('branch', ''), steps.ShellCommand( name=ip('Checking out feature branch \"%(prop:FEATURE)s\"'), command=['./tools/VaRA/utils/buildbot/bb-checkout-branches.sh', force_feature], workdir=ip(CHECKOUT_BASE_DIR)), ]) defer.returnValue(result)
def run(self): command = yield self.makeRemoteShellCommand() yield self.runCommand(command) force_complete_rebuild = None if self.hasProperty('options'): options = self.getProperty('options') force_complete_rebuild = options['force_complete_rebuild'] buildsteps = [] for repo in REPOS: buildsteps.append(define(str(repo).upper() +'_ROOT', ip(REPOS[repo]['checkout_dir']))) if force_complete_rebuild: buildsteps.append(define('FORCE_COMPLETE_REBUILD', 'true')) buildsteps.append(steps.ShellCommand(name='Delete old build directory', command=['rm', '-rf', 'build'], workdir=ip(CHECKOUT_BASE_DIR))) for repo in REPOS: if 'repository_clone_url' in codebases[repo].keys(): url = codebases[repo]['repository_clone_url'] else: url = codebases[repo]['repository'] branch = REPOS[repo]['default_branch'] buildsteps.append(steps.Git(repourl=url, branch=branch, codebase=repo, name="checkout: {0}".format(url), description="checkout: {0}@{1}".format(url, branch), timeout=1200, progress=True, submodules=True, workdir=P(str(repo).upper()+'_ROOT'), mode='full', method='clobber')) else: self.build.addStepsAfterCurrentStep([define('FORCE_COMPLETE_REBUILD', 'false')]) for repo in REPOS: if 'repository_clone_url' in codebases[repo].keys(): url = codebases[repo]['repository_clone_url'] else: url = codebases[repo]['repository'] branch = REPOS[repo]['default_branch'] buildsteps.append(steps.Git(repourl=url, branch=branch, codebase=repo, name="checkout: {0}".format(url), description="checkout: {0}@{1}".format(url, branch), timeout=1200, progress=True, submodules=True, workdir=P(str(repo).upper()+'_ROOT'))) buildsteps.append(steps.ShellCommand(name='Create build directory', command=['mkdir', '-p', 'build'], workdir=ip(CHECKOUT_BASE_DIR), hideStepIf=True)) self.build.addStepsAfterCurrentStep(buildsteps) defer.returnValue(command.results())
def run(self): command = yield self.makeRemoteShellCommand() yield self.runCommand(command) result = command.results() if result == util.SUCCESS: mergecheck_repo = self.getProperty('mergecheck_repo') current_branch = self.observer.getStdout().strip() default_branch = REPOS[mergecheck_repo]['default_branch'] repo_subdir = REPOS[mergecheck_repo]['checkout_subdir'] if default_branch == current_branch.replace('refs/heads/', ''): # This repository has no feature branch, so nothing has to be merged. defer.returnValue(result) self.build.addStepsAfterCurrentStep([ steps.Compile( command=['/scratch/pjtest/mergecheck/build/bin/mergecheck', 'rebase', '--repo', '.' + repo_subdir, '--upstream', 'refs/remotes/origin/' + default_branch, '--branch', current_branch, '-v', '--print-conflicts', ], workdir=ip(CHECKOUT_BASE_DIR), name='Mergecheck \"' + mergecheck_repo + '\"', warnOnWarnings=False, warningPattern=r'^CONFLICT \((content|add\/add)\).*'), ]) defer.returnValue(result)
def run(self): command = yield self.makeRemoteShellCommand() yield self.runCommand(command) result = command.results() if result == util.SUCCESS: checkout_feature_br = self.get_feature_branch( self.observer.getStdout()) if checkout_feature_br: self.build.addStepsAfterCurrentStep([ define('FEATURE', checkout_feature_br), define('branch', checkout_feature_br), steps.ShellCommand( name='Checking out feature branch \"' + str(checkout_feature_br) + '\"', command=[ './tools/VaRA/utils/buildbot/bb-checkout-branches.sh', checkout_feature_br ], workdir=ip(CHECKOUT_BASE_DIR)), ]) else: force_feature = None if self.hasProperty('options'): options = self.getProperty('options') force_feature = options['force_feature'] self.build.addStepsAfterCurrentStep([ define('FEATURE', force_feature), define('branch', ''), steps.ShellCommand( name=ip( 'Checking out feature branch \"%(prop:FEATURE)s\"' ), command=[ './tools/VaRA/utils/buildbot/bb-checkout-branches.sh', force_feature ], workdir=ip(CHECKOUT_BASE_DIR)), ]) defer.returnValue(result)
def configure(c): steps = [ define("LLVM_ROOT", ip("%(prop:builddir)s/llvm")), define("UCHROOT_SRC_ROOT", "/mnt/llvm"), define("CLANG_ROOT", ip("%(prop:LLVM_ROOT)s/tools/clang")), define("POLLY_ROOT", ip("%(prop:LLVM_ROOT)s/tools/polly")), define("COMPILERRT_ROOT", ip("%(prop:LLVM_ROOT)s/projects/compiler-rt")), define("OPENMP_ROOT", ip("%(prop:LLVM_ROOT)s/projects/openmp")), git('llvm', 'master', codebases, workdir=P("LLVM_ROOT")), git('clang', 'master', codebases, workdir=P("CLANG_ROOT")), git('polly', 'master', codebases, workdir=P("POLLY_ROOT")), git('compiler-rt', 'master', codebases, workdir=P("COMPILERRT_ROOT")), git('openmp', 'master', codebases, workdir=P("OPENMP_ROOT")), ucmd('cmake', P("UCHROOT_SRC_ROOT"), '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_INSTALL_PREFIX=./_install', '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -DLLVM_ENABLE_STATS', '-DBUILD_SHARED_LIBS=Off', '-DPOLLY_BUILD_POLLI=Off', '-DLLVM_POLLY_LINK_INTO_TOOLS=OFF', '-DLLVM_TARGETS_TO_BUILD=X86', '-DLLVM_BINUTILS_INCDIR=/usr/include', '-DLLVM_ENABLE_PIC=On', '-DLLVM_ENABLE_ASSERTIONS=On', '-DLLVM_ENABLE_TERMINFO=Off', '-DCLANG_DEFAULT_OPENMP_RUNTIME=libomp', '-G', 'Ninja', env={ "PATH": "/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin" }, name="cmake", description="cmake O3, Assertions, PIC, Static"), ucompile("ninja", "install", haltOnFailure=True, name="build llvm"), cmd("tar", "czf", "../llvm.tar.gz", "-C", "./_install", ".") ] upload_llvm = hash_upload_to_master("llvm.tar.gz", "../llvm.tar.gz", "public_html/llvm.tar.gz", URL) steps.extend(upload_llvm) c['builders'].append(builder("build-llvm", None, accepted_builders, tags=['polyjit'], factory=BuildFactory(steps)))
def run(self): command = yield self.makeRemoteShellCommand() yield self.runCommand(command) result = command.results() if result == util.SUCCESS: mergecheck_repo = self.getProperty('mergecheck_repo') current_branch = self.observer.getStdout().strip() #default_branch = REPOS[mergecheck_repo]['default_branch'] repo_subdir = REPOS[mergecheck_repo]['checkout_subdir'] upstream_merge_base = '' upstream_remote_url = '' if ('upstream_merge_base' not in REPOS[mergecheck_repo] or 'upstream_remote_url' not in REPOS[mergecheck_repo]): # This repository has no remote to compare against, so no mergecheck has to be done. defer.returnValue(result) else: upstream_merge_base = REPOS[mergecheck_repo][ 'upstream_merge_base'] upstream_remote_url = REPOS[mergecheck_repo][ 'upstream_remote_url'] self.build.addStepsAfterCurrentStep([ steps.Compile( command=[ '/local/hdd/buildbot/mergecheck/build/bin/mergecheck', 'rebase', '--repo', '.' + repo_subdir, '--remote-url', upstream_remote_url, '--remote-name', 'upstream', '--onto', 'refs/remotes/upstream/master', '--upstream', upstream_merge_base, '--branch', current_branch, '-v', '--print-conflicts', ], workdir=ip(CHECKOUT_BASE_DIR), name='Mergecheck \"' + mergecheck_repo + '\"', warnOnWarnings=False, warningPattern=r'^CONFLICT \((content|add\/add)\).*'), ]) defer.returnValue(result)
def run(self): command = yield self.makeRemoteShellCommand() yield self.runCommand(command) result = command.results() if result == util.SUCCESS: mergecheck_repo = self.getProperty('mergecheck_repo') current_branch = self.observer.getStdout().strip() #default_branch = REPOS[mergecheck_repo]['default_branch'] repo_subdir = REPOS[mergecheck_repo]['checkout_subdir'] upstream_merge_base = '' upstream_remote_url = '' if ('upstream_merge_base' not in REPOS[mergecheck_repo] or 'upstream_remote_url' not in REPOS[mergecheck_repo]): # This repository has no remote to compare against, so no mergecheck has to be done. defer.returnValue(result) else: upstream_merge_base = REPOS[mergecheck_repo]['upstream_merge_base'] upstream_remote_url = REPOS[mergecheck_repo]['upstream_remote_url'] self.build.addStepsAfterCurrentStep([ steps.Compile( command=['/scratch/pjtest/mergecheck/build/bin/mergecheck', 'rebase', '--repo', '.' + repo_subdir, '--remote-url', upstream_remote_url, '--remote-name', 'upstream', '--onto', 'refs/remotes/upstream/master', '--upstream', upstream_merge_base, '--branch', current_branch, '-v', '--print-conflicts', ], workdir=ip(CHECKOUT_BASE_DIR), name='Mergecheck \"' + mergecheck_repo + '\"', warnOnWarnings=False, warningPattern=r'^CONFLICT \((content|add\/add)\).*'), ]) defer.returnValue(result)
def run(self): command = yield self.makeRemoteShellCommand() yield self.runCommand(command) result = command.results() if result == util.SUCCESS: mergecheck_repo = self.getProperty('mergecheck_repo') current_branch = self.observer.getStdout().strip() default_branch = REPOS[mergecheck_repo]['default_branch'] repo_subdir = REPOS[mergecheck_repo]['checkout_subdir'] if default_branch == current_branch.replace('refs/heads/', ''): # This repository has no feature branch, so nothing has to be merged. defer.returnValue(result) self.build.addStepsAfterCurrentStep([ steps.Compile( command=[ '/local/hdd/buildbot/mergecheck/build/bin/mergecheck', 'rebase', '--repo', '.' + repo_subdir, '--upstream', 'refs/remotes/origin/' + default_branch, '--branch', current_branch, '-v', '--print-conflicts', ], workdir=ip(CHECKOUT_BASE_DIR), name='Mergecheck \"' + mergecheck_repo + '\"', warnOnWarnings=False, warningPattern=r'^CONFLICT \((content|add\/add)\).*'), ]) defer.returnValue(result)
def configure(c): f = util.BuildFactory() # TODO Check if this can be done without a dummy command #f.addStep(GenerateGitCloneCommand()) f.addStep(GenerateGitCloneCommand(name="Dummy_1", command=['true'], haltOnFailure=True, hideStepIf=True)) f.addStep(define('UCHROOT_SRC_ROOT', UCHROOT_SRC_ROOT)) f.addStep(define('UCHROOT_BUILD_DIR', UCHROOT_BUILD_DIR)) f.addStep(GenerateGitCheckoutCommand( name="Get branch names", command=['./tools/VaRA/utils/buildbot/bb-get-branches.sh'], workdir=ip(CHECKOUT_BASE_DIR), haltOnFailure=True, hideStepIf=True)) # CMake for step in get_uchroot_workaround_steps(): f.addStep(step) f.addStep(ucompile('../tools/VaRA/utils/vara/builds/' + BUILD_SCRIPT, env={'PATH': '/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin'}, name='cmake', description=BUILD_SCRIPT, workdir=UCHROOT_SRC_ROOT + '/build')) f.addStep(GenerateMakeCleanCommand(name="Dummy_2", command=['true'], haltOnFailure=True, hideStepIf=True)) # use mergecheck tool to make sure the 'upstream' remote is present for repo in ['vara-llvm', 'vara-clang']: f.addStep(steps.Compile( command=['/scratch/pjtest/mergecheck/build/bin/mergecheck', 'rebase', '--repo', '.' + REPOS[repo]['checkout_subdir'], '--remote-url', REPOS[repo]['upstream_remote_url'], '--remote-name', 'upstream', '--upstream', 'refs/remotes/upstream/master', '--branch', 'refs/remotes/upstream/master', '-v'], workdir=ip(CHECKOUT_BASE_DIR), name='Add upstream remote to repository.', hideStepIf=True)) # Prepare project file list to filter out compiler warnings f.addStep(cmd("../../tools/VaRA/utils/vara/getVaraSourceFiles.sh", "--all", "--include-existing", "--relative-to", ip(BUILD_DIR), "--output", "buildbot-source-file-list.txt", workdir=ip(BUILD_DIR), hideStepIf=True)) # Compile Step f.addStep(GenerateBuildStepCommand(name="Dummy_3", command=['cat', 'buildbot-source-file-list.txt'], workdir=ip(BUILD_DIR), haltOnFailure=True, hideStepIf=True)) # Regression Test step for step in get_uchroot_workaround_steps(): f.addStep(step) f.addStep(ucompile('ninja', 'check-vara', name='run VaRA regression tests', workdir=UCHROOT_BUILD_DIR, haltOnFailure=False, warnOnWarnings=True)) # Clang-Tidy for step in get_uchroot_workaround_steps(): f.addStep(step) f.addStep(ucompile('python3', 'tidy-vara.py', '-p', UCHROOT_BUILD_DIR, '-j', '8', '--gcc', workdir='vara-llvm/tools/VaRA/test/', name='run Clang-Tidy', haltOnFailure=False, warnOnWarnings=True, env={'PATH': [UCHROOT_BUILD_DIR + "/bin", "${PATH}"]}, timeout=3600)) # ClangFormat f.addStep(GenerateClangFormatStepCommand(name="Dummy_4", command=['opt/clang-format-static/clang-format', '-version'], workdir=ip('%(prop:uchroot_image_path)s'), haltOnFailure=True, hideStepIf=True)) # Mergecheck for repo in ['vara-llvm', 'vara-clang', 'vara']: f.addStep(define('mergecheck_repo', repo)) f.addStep(GenerateMergecheckCommand(name="Dummy_5", command=['git', 'symbolic-ref', 'HEAD'], workdir=ip(REPOS[repo]['checkout_dir']), haltOnFailure=True, hideStepIf=True)) c['builders'].append(builder(PROJECT_NAME, None, ACCEPTED_BUILDERS, tags=['vara'], factory=f))
def configure(c): f = util.BuildFactory() # TODO Check if this can be done without a dummy command #f.addStep(GenerateGitCloneCommand()) f.addStep(GenerateGitCloneCommand(name="Dummy_1", command=['true'], haltOnFailure=True, hideStepIf=True)) f.addStep(define('UCHROOT_SRC_ROOT', UCHROOT_SRC_ROOT)) f.addStep(define('UCHROOT_BUILD_DIR', UCHROOT_BUILD_DIR)) f.addStep(GenerateGitCheckoutCommand( name="Get branch names", command=['./tools/VaRA/utils/buildbot/bb-get-branches.sh'], workdir=ip(CHECKOUT_BASE_DIR), haltOnFailure=True, hideStepIf=True)) # CMake for step in get_uchroot_workaround_steps(): f.addStep(step) f.addStep(ucompile('../tools/VaRA/utils/vara/builds/' + BUILD_SCRIPT, env={'PATH': '/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin'}, name='cmake', description=BUILD_SCRIPT, workdir=UCHROOT_SRC_ROOT + '/build')) f.addStep(GenerateMakeCleanCommand(name="Dummy_2", command=['true'], haltOnFailure=True, hideStepIf=True)) # use mergecheck tool to make sure the 'upstream' remote is present for repo in ['vara-llvm', 'vara-clang']: f.addStep(steps.Compile( command=['/local/hdd/buildbot/mergecheck/build/bin/mergecheck', 'rebase', '--repo', '.' + REPOS[repo]['checkout_subdir'], '--remote-url', REPOS[repo]['upstream_remote_url'], '--remote-name', 'upstream', '--upstream', 'refs/remotes/upstream/master', '--branch', 'refs/remotes/upstream/master', '-v'], workdir=ip(CHECKOUT_BASE_DIR), name='Add upstream remote to repository.', hideStepIf=True)) # Prepare project file list to filter out compiler warnings f.addStep(cmd("../../tools/VaRA/utils/vara/getVaraSourceFiles.sh", "--vara", "--clang", "--llvm", "--include-existing", "--relative-to", ip(BUILD_DIR), "--output", "buildbot-source-file-list.txt", workdir=ip(BUILD_DIR), hideStepIf=True)) # Compile Step f.addStep(GenerateBuildStepCommand(name="Dummy_3", command=['cat', 'buildbot-source-file-list.txt'], workdir=ip(BUILD_DIR), haltOnFailure=True, hideStepIf=True)) # Regression Test step for step in get_uchroot_workaround_steps(): f.addStep(step) f.addStep(ucompile('ninja', 'check-vara', name='run VaRA regression tests', workdir=UCHROOT_BUILD_DIR, haltOnFailure=False, warnOnWarnings=True)) # Clang-Tidy for step in get_uchroot_workaround_steps(): f.addStep(step) f.addStep(ucompile('python3', 'tidy-vara.py', '-p', UCHROOT_BUILD_DIR, '-j', '8', '--gcc', workdir='vara-llvm/tools/VaRA/test/', name='run Clang-Tidy', haltOnFailure=False, warnOnWarnings=True, timeout=3600)) # ClangFormat f.addStep(GenerateClangFormatStepCommand(name="Dummy_4", command=['opt/clang-format-static/clang-format', '-version'], workdir=ip('%(prop:uchroot_image_path)s'), haltOnFailure=True, hideStepIf=True)) c['builders'].append(builder(PROJECT_NAME, None, ACCEPTED_BUILDERS, tags=['vara'], factory=f))
def run(self): command = yield self.makeRemoteShellCommand() yield self.runCommand(command) force_complete_rebuild = None if self.hasProperty('options'): options = self.getProperty('options') force_complete_rebuild = options['force_complete_rebuild'] buildsteps = [] for repo in REPOS: buildsteps.append( define( str(repo).upper() + '_ROOT', ip(REPOS[repo]['checkout_dir']))) if force_complete_rebuild: buildsteps.append(define('FORCE_COMPLETE_REBUILD', 'true')) buildsteps.append( steps.ShellCommand(name='Delete old build directory', command=['rm', '-rf', 'build'], workdir=ip(CHECKOUT_BASE_DIR))) for repo in REPOS: if 'repository_clone_url' in codebases[repo].keys(): url = codebases[repo]['repository_clone_url'] else: url = codebases[repo]['repository'] branch = REPOS[repo]['default_branch'] buildsteps.append( steps.Git(repourl=url, branch=branch, codebase=repo, name="checkout: {0}".format(url), description="checkout: {0}@{1}".format( url, branch), timeout=1200, progress=True, submodules=True, workdir=P(str(repo).upper() + '_ROOT'), mode='full', method='clobber')) else: self.build.addStepsAfterCurrentStep( [define('FORCE_COMPLETE_REBUILD', 'false')]) for repo in REPOS: if 'repository_clone_url' in codebases[repo].keys(): url = codebases[repo]['repository_clone_url'] else: url = codebases[repo]['repository'] branch = REPOS[repo]['default_branch'] buildsteps.append( steps.Git(repourl=url, branch=branch, codebase=repo, name="checkout: {0}".format(url), description="checkout: {0}@{1}".format( url, branch), timeout=1200, progress=True, submodules=True, workdir=P(str(repo).upper() + '_ROOT'))) buildsteps.append( steps.ShellCommand(name='Create build directory', command=['mkdir', '-p', 'build'], workdir=ip(CHECKOUT_BASE_DIR), hideStepIf=True)) self.build.addStepsAfterCurrentStep(buildsteps) defer.returnValue(command.results())
def configure(c): llvm_dl = hash_download_from_master("public_html/llvm.tar.gz", "llvm.tar.gz", "llvm") polyjit_dl = hash_download_from_master("public_html/polyjit.tar.gz", "polyjit.tar.gz", "polyjit") steps = [ # trigger(schedulerNames=['trigger-build-llvm', 'trigger-build-jit']), define("scratch", ip("/scratch/pjtest/%(prop:buildnumber)s/")) ] steps.extend(llvm_dl) steps.extend(clean_unpack("llvm.tar.gz", "llvm")) steps.extend(polyjit_dl) steps.extend(clean_unpack("polyjit.tar.gz", "polyjit")) steps.extend([ define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")), git('benchbuild', 'master', codebases, workdir=P("BENCHBUILD_ROOT")), ]) steps.extend([ define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')), define('llvm', ip('%(prop:scratch)s/llvm')), define('polyjit', ip('%(prop:scratch)s/polyjit')), mkdir(P("scratch")), cmd('virtualenv', '-ppython3', ip('%(prop:scratch)s/env/')), cmd(ip('%(prop:scratch)s/env/bin/pip3'), 'install', '--upgrade', '.', workdir='build/benchbuild'), cmd("rsync", "-var", "llvm", P("scratch")), cmd("rsync", "-var", "polyjit", P("scratch")), cmd(P('benchbuild'), 'bootstrap', '-s', env={ 'BB_CONFIG_FILE': '/scratch/pjtest/.benchbuild.json', 'BB_TMP_DIR': '/scratch/pjtest/src/', 'BB_TEST_DIR': P("testinputs"), 'BB_GENTOO_HTTP_PROXY': 'debussy.fim.uni-passau.de:3128', 'BB_GENTOO_FTP_PROXY': 'debussy.fim.uni-passau.de:3128', 'BB_GENTOO_AUTOTEST_LOC': '/scratch/pjtest/gentoo-autotest', 'BB_SLURM_PARTITION': 'chimaira', 'BB_SLURM_NODE_DIR': '/local/hdd/pjtest/', 'BB_SLURM_ACCOUNT': 'cl', 'BB_SLURM_TIMELIMIT': '24:00:00', 'BB_ENV_COMPILER_PATH': ip('["%(prop:llvm)s/bin", "%(prop:polyjit)s/bin"]'), 'BB_ENV_COMPILER_LD_LIBRARY_PATH': ip('["%(prop:llvm)s/lib", "%(prop:polyjit)s/lib"]'), 'BB_ENV_BINARY_PATH': ip('["%(prop:llvm)s/bin", "%(prop:polyjit)s/bin"]'), 'BB_ENV_BINARY_LD_LIBRARY_PATH': ip('["%(prop:llvm)s/lib", "%(prop:polyjit)s/lib"]'), 'BB_ENV_LOOKUP_PATH': ip('["%(prop:llvm)s/bin", "%(prop:polyjit)s/bin", "/scratch/pjtest/erlent/build"]'), 'BB_ENV_LOOKUP_LD_LIBRARY_PATH': ip('["%(prop:llvm)s/lib", "%(prop:polyjit)s/lib"]'), 'BB_LLVM_DIR': ip('%(prop:scratch)s/llvm'), 'BB_LIKWID_PREFIX': '/usr', 'BB_PAPI_INCLUDE': '/usr/include', 'BB_PAPI_LIBRARY': '/usr/lib', 'BB_SRC_DIR': ip('%(prop:scratch)s/benchbuild'), 'BB_SLURM_LOGS': ip('%(prop:scratch)s/slurm.log'), 'BB_UNIONFS_ENABLE': 'false' }, workdir=P('scratch')), # This only works on infosun machines cmd("ln", "-s", ip("/scratch/pjtest/benchbuild-src/"), ip("%(prop:scratch)s/benchbuild")), mkdir(ip("%(prop:scratch)s/results"))]) c['builders'].append(builder("build-slurm-set", None, accepted_builders, tags=['polyjit'], factory=BuildFactory(steps)))
def configure(c): sb_steps = [ define("SUPERBUILD_ROOT", ip("%(prop:builddir)s/polli-sb")), define("POLYJIT_DEFAULT_BRANCH", "master"), git('pjit-polli-sb', 'master', REPOS, workdir=P("SUPERBUILD_ROOT"), mode="full", method="fresh"), ICMD_P('%(prop:cmake_prefix)s/bin/cmake', P("SUPERBUILD_ROOT"), '-DCMAKE_CXX_COMPILER=%(prop:cxx)s', '-DCMAKE_C_COMPILER=%(prop:cc)s', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_INSTALL_PREFIX=./_install', '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -DLLVM_ENABLE_STATS', '-DBUILD_SHARED_LIBS=Off', '-DPOLYJIT_PAPI_PREFIX=/scratch/pjtest/opt/papi-5.5.1/install', '-DPOLYJIT_BRANCH_CLANG=%(prop:POLYJIT_DEFAULT_BRANCH)s', '-DPOLYJIT_BRANCH_LLVM=%(prop:POLYJIT_DEFAULT_BRANCH)s', '-DPOLYJIT_BRANCH_POLLI=%(prop:POLYJIT_DEFAULT_BRANCH)s', '-DPOLYJIT_BRANCH_POLLY=%(prop:POLYJIT_DEFAULT_BRANCH)s', '-G', 'Ninja', name="cmake", description="[uchroot] cmake: release build", descriptionDone="[uchroot] configured."), CMD_P("/usr/bin/ninja", "llvm-configure", name="configure LLVM", description="configuring LLVM", descriptionDone="configured LLVM"), CMD_P("/usr/bin/ninja", "polli-configure", name="configure PolyJIT", description="configuring PolyJIT", descriptionDone="configured PolyJIT"), CMD_P("/usr/bin/ninja", name="build jit", description="[uchroot] building PolyJIT", descriptionDone="[uchroot] built PolyJIT"), cmd("tar", "czf", "../polyjit_sb.tar.gz", "-C", "./_install", ".") ] sb_steps.extend(hash_upload_to_master( "polyjit_sb.tar.gz", "../polyjit_sb.tar.gz", "public_html/polyjit_sb.tar.gz", URL)) slurm_steps = [ define("scratch", ip("/scratch/pjtest/sb-%(prop:buildnumber)s/")), *hash_download_from_master("public_html/polyjit_sb.tar.gz", "polyjit_sb.tar.gz", "polyjit"), *clean_unpack("polyjit_sb.tar.gz", "llvm"), define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")), git('pjit-benchbuild', 'master', REPOS, workdir=P("BENCHBUILD_ROOT")), define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')), define('llvm', ip('%(prop:scratch)s/llvm')), define('bb_src', ip('%(prop:scratch)s/benchbuild')), icmd('virtualenv', '-ppython3', '%(prop:scratch)s/env'), icmd('%(prop:scratch)s/env/bin/pip3', 'install', P("BENCHBUILD_ROOT")), mkdir(P("scratch")), cmd("rsync", "-var", "llvm", P("scratch")), cmd(P('benchbuild'), 'bootstrap', '-s', logEnviron=True, env=BB_ENV, workdir=P('scratch')), # This only works on infosun machines icmd("ln", "-s", "/scratch/pjtest/benchbuild-src/", "%(prop:bb_src)s"), mkdir(ip("%(prop:scratch)s/results")) ] c['builders'].extend([ builder("polyjit-superbuild", None, ACCEPTED_BUILDERS, tags=['polyjit'], collapseRequests=True, factory=BF(sb_steps)), builder("polyjit-superbuild-slurm", None, ACCEPTED_BUILDERS, tags=['polyjit'], collapseRequests=True, factory=BF(slurm_steps)) ])
def icmd(*args, **kwargs): ip_list = [ ip(le) if isinstance(le, str) else le for le in args] return cmd(*ip_list, **kwargs)
BF = util.BuildFactory ACCEPTED_BUILDERS = \ slaves.get_hostlist(slaves.infosun, predicate=lambda host: host["host"] in {'debussy', 'ligeti'}) BB_ENV = { 'BB_CONFIG_FILE': '/scratch/pjtest/.benchbuild.json', 'BB_TMP_DIR': '/scratch/pjtest/src/', 'BB_TEST_DIR': P("testinputs"), 'BB_GENTOO_AUTOTEST_LOC': '/scratch/pjtest/gentoo-autotest', 'BB_SLURM_PARTITION': 'anywhere', 'BB_SLURM_NODE_DIR': '/local/hdd/buildbot-polyjit/', 'BB_SLURM_ACCOUNT': 'anywhere', 'BB_SLURM_TIMELIMIT': '24:00:00', 'BB_CONTAINER_MOUNTS': ip('["%(prop:llvm)s", "%(prop:bb_src)s"]'), 'BB_CONTAINER_PREFIXES': '["/opt/benchbuild", ' '"/", ' '"/usr", ' '"/usr/local"]', 'BB_ENV_PATH': ip('["/scratch/pjtest/opt/papi-5.5.1/install/bin", ' '"%(prop:llvm)s/bin", ' '"%(prop:llvm_prefix)s/bin", ' '"/scratch/pjtest/erlent/build"]'), 'BB_ENV_LD_LIBRARY_PATH': ip('["/scratch/pjtest/opt/papi-5.5.1/install/lib", ' '"%(prop:llvm)s/lib", ' '"%(prop:llvm_libs)s", ' '"/scratch/pjtest/erlent/build"]'),