Example #1
0
def get_uchroot_workaround_steps():
    workaround_steps = []
    workaround_steps.append(ucompile('true', name='uchroot /proc bug workaround', hideStepIf=True,
                                     haltOnFailure=False, flunkOnWarnings=False,
                                     flunkOnFailure=False, warnOnWarnings=False,
                                     warnOnFailure=False))
    workaround_steps.append(cmd("sleep 1", hideStepIf=True))
    return workaround_steps
Example #2
0
def configure(c):
    c['builders'].append(
        builder("build-benchbuild", None, accepted_builders,
                tags=['polyjit'],
                factory=BuildFactory([
                    git('pjit-benchbuild', 'master', REPOS),
                    cmd("virtualenv", "-p", "python3", "_venv",
                        name="create virtualenv",
                        description="setup benchbuild virtual environment"),
                    cmd("_venv/bin/pip3", "install", ".",
                        name="install benchbuild",
                        description="install benchbuild into the venv",
                        env={'PYTHONPATH': 'benchbuild/bin',
                             'HOME': '.'}),
                    pylint("_venv/bin/pylint", "./benchbuild/",
                           warnOnFailure=True, flunkOnWarnings=False,
                           flunkOnFailure=False, haltOnFailure=False),
                    pylint("_venv/bin/pylint", "-E", "./benchbuild/",
                           warnOnFailure=True, flunkOnWarnings=False,
                           flunkOnFailure=False, haltOnFailure=False),
                    rmdir("_venv")])))
Example #3
0
def get_uchroot_workaround_steps():
    workaround_steps = []
    workaround_steps.append(
        ucompile('true',
                 name='uchroot /proc bug workaround',
                 hideStepIf=True,
                 haltOnFailure=False,
                 flunkOnWarnings=False,
                 flunkOnFailure=False,
                 warnOnWarnings=False,
                 warnOnFailure=False))
    workaround_steps.append(cmd("sleep 1", hideStepIf=True))
    return workaround_steps
Example #4
0
def configure(c):
    steps = [
        define("LLVM_ROOT", ip("%(prop:builddir)s/llvm")),
        define("UCHROOT_SRC_ROOT", "/mnt/llvm"),
        define("CLANG_ROOT", ip("%(prop:LLVM_ROOT)s/tools/clang")),
        define("POLLY_ROOT", ip("%(prop:LLVM_ROOT)s/tools/polly")),
        define("COMPILERRT_ROOT", ip("%(prop:LLVM_ROOT)s/projects/compiler-rt")),
        define("OPENMP_ROOT", ip("%(prop:LLVM_ROOT)s/projects/openmp")),

        git('llvm', 'master', codebases, workdir=P("LLVM_ROOT")),
        git('clang', 'master', codebases, workdir=P("CLANG_ROOT")),
        git('polly', 'master', codebases, workdir=P("POLLY_ROOT")),
        git('compiler-rt', 'master', codebases, workdir=P("COMPILERRT_ROOT")),
        git('openmp', 'master', codebases, workdir=P("OPENMP_ROOT")),
        ucmd('cmake', P("UCHROOT_SRC_ROOT"),
             '-DCMAKE_BUILD_TYPE=Release',
             '-DCMAKE_INSTALL_PREFIX=./_install',
             '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -DLLVM_ENABLE_STATS',
             '-DBUILD_SHARED_LIBS=Off',
             '-DPOLLY_BUILD_POLLI=Off',
             '-DLLVM_POLLY_LINK_INTO_TOOLS=OFF',
             '-DLLVM_TARGETS_TO_BUILD=X86',
             '-DLLVM_BINUTILS_INCDIR=/usr/include',
             '-DLLVM_ENABLE_PIC=On',
             '-DLLVM_ENABLE_ASSERTIONS=On',
             '-DLLVM_ENABLE_TERMINFO=Off',
             '-DCLANG_DEFAULT_OPENMP_RUNTIME=libomp',
             '-G', 'Ninja',
             env={
                 "PATH": "/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin"
             },
             name="cmake",
             description="cmake O3, Assertions, PIC, Static"),
        ucompile("ninja", "install", haltOnFailure=True, name="build llvm"),
        cmd("tar", "czf", "../llvm.tar.gz", "-C", "./_install", ".")
    ]
    upload_llvm = hash_upload_to_master("llvm.tar.gz",
        "../llvm.tar.gz", "public_html/llvm.tar.gz", URL)
    steps.extend(upload_llvm)

    c['builders'].append(builder("build-llvm", None, accepted_builders,
                         tags=['polyjit'], factory=BuildFactory(steps)))
Example #5
0
def configure(c):
    f = util.BuildFactory()

    # TODO Check if this can be done without a dummy command
    #f.addStep(GenerateGitCloneCommand())
    f.addStep(GenerateGitCloneCommand(name="Dummy_1", command=['true'],
                                      haltOnFailure=True, hideStepIf=True))

    f.addStep(define('UCHROOT_SRC_ROOT', UCHROOT_SRC_ROOT))
    f.addStep(define('UCHROOT_BUILD_DIR', UCHROOT_BUILD_DIR))

    f.addStep(GenerateGitCheckoutCommand(
        name="Get branch names",
        command=['./tools/VaRA/utils/buildbot/bb-get-branches.sh'], workdir=ip(CHECKOUT_BASE_DIR),
        haltOnFailure=True, hideStepIf=True))

    # CMake
    for step in get_uchroot_workaround_steps():
        f.addStep(step)
    f.addStep(ucompile('../tools/VaRA/utils/vara/builds/' + BUILD_SCRIPT,
                       env={'PATH': '/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin'},
                       name='cmake',
                       description=BUILD_SCRIPT,
                       workdir=UCHROOT_SRC_ROOT + '/build'))

    f.addStep(GenerateMakeCleanCommand(name="Dummy_2", command=['true'],
                                       haltOnFailure=True, hideStepIf=True))

    # use mergecheck tool to make sure the 'upstream' remote is present
    for repo in ['vara-llvm', 'vara-clang']:
        f.addStep(steps.Compile(
            command=['/scratch/pjtest/mergecheck/build/bin/mergecheck', 'rebase',
                     '--repo', '.' + REPOS[repo]['checkout_subdir'],
                     '--remote-url', REPOS[repo]['upstream_remote_url'],
                     '--remote-name', 'upstream',
                     '--upstream', 'refs/remotes/upstream/master',
                     '--branch', 'refs/remotes/upstream/master',
                     '-v'],
            workdir=ip(CHECKOUT_BASE_DIR),
            name='Add upstream remote to repository.', hideStepIf=True))

    # Prepare project file list to filter out compiler warnings
    f.addStep(cmd("../../tools/VaRA/utils/vara/getVaraSourceFiles.sh",
                  "--all", "--include-existing",
                  "--relative-to", ip(BUILD_DIR),
                  "--output", "buildbot-source-file-list.txt",
                  workdir=ip(BUILD_DIR), hideStepIf=True))

    # Compile Step
    f.addStep(GenerateBuildStepCommand(name="Dummy_3",
                                       command=['cat', 'buildbot-source-file-list.txt'],
                                       workdir=ip(BUILD_DIR),
                                       haltOnFailure=True, hideStepIf=True))

    # Regression Test step
    for step in get_uchroot_workaround_steps():
        f.addStep(step)
    f.addStep(ucompile('ninja', 'check-vara', name='run VaRA regression tests',
                       workdir=UCHROOT_BUILD_DIR,
                       haltOnFailure=False, warnOnWarnings=True))

    # Clang-Tidy
    for step in get_uchroot_workaround_steps():
        f.addStep(step)
    f.addStep(ucompile('python3', 'tidy-vara.py', '-p', UCHROOT_BUILD_DIR, '-j', '8', '--gcc',
                       workdir='vara-llvm/tools/VaRA/test/', name='run Clang-Tidy',
                       haltOnFailure=False, warnOnWarnings=True,
                       env={'PATH': [UCHROOT_BUILD_DIR + "/bin", "${PATH}"]}, timeout=3600))

    # ClangFormat
    f.addStep(GenerateClangFormatStepCommand(name="Dummy_4",
                                             command=['opt/clang-format-static/clang-format',
                                                      '-version'],
                                             workdir=ip('%(prop:uchroot_image_path)s'),
                                             haltOnFailure=True, hideStepIf=True))

    # Mergecheck
    for repo in ['vara-llvm', 'vara-clang', 'vara']:
        f.addStep(define('mergecheck_repo', repo))
        f.addStep(GenerateMergecheckCommand(name="Dummy_5", command=['git', 'symbolic-ref', 'HEAD'],
                                            workdir=ip(REPOS[repo]['checkout_dir']),
                                            haltOnFailure=True, hideStepIf=True))

    c['builders'].append(builder(PROJECT_NAME, None, ACCEPTED_BUILDERS, tags=['vara'],
                                 factory=f))
Example #6
0
def configure(c):
    f = util.BuildFactory()

    # TODO Check if this can be done without a dummy command
    #f.addStep(GenerateGitCloneCommand())
    f.addStep(GenerateGitCloneCommand(name="Dummy_1", command=['true'],
                                      haltOnFailure=True, hideStepIf=True))

    f.addStep(define('UCHROOT_SRC_ROOT', UCHROOT_SRC_ROOT))
    f.addStep(define('UCHROOT_BUILD_DIR', UCHROOT_BUILD_DIR))

    f.addStep(GenerateGitCheckoutCommand(
        name="Get branch names",
        command=['./tools/VaRA/utils/buildbot/bb-get-branches.sh'], workdir=ip(CHECKOUT_BASE_DIR),
        haltOnFailure=True, hideStepIf=True))

    # CMake
    for step in get_uchroot_workaround_steps():
        f.addStep(step)
    f.addStep(ucompile('../tools/VaRA/utils/vara/builds/' + BUILD_SCRIPT,
                       env={'PATH': '/opt/cmake/bin:/usr/local/bin:/usr/bin:/bin'},
                       name='cmake',
                       description=BUILD_SCRIPT,
                       workdir=UCHROOT_SRC_ROOT + '/build'))

    f.addStep(GenerateMakeCleanCommand(name="Dummy_2", command=['true'],
                                       haltOnFailure=True, hideStepIf=True))

    # use mergecheck tool to make sure the 'upstream' remote is present
    for repo in ['vara-llvm', 'vara-clang']:
        f.addStep(steps.Compile(
            command=['/local/hdd/buildbot/mergecheck/build/bin/mergecheck', 'rebase',
                     '--repo', '.' + REPOS[repo]['checkout_subdir'],
                     '--remote-url', REPOS[repo]['upstream_remote_url'],
                     '--remote-name', 'upstream',
                     '--upstream', 'refs/remotes/upstream/master',
                     '--branch', 'refs/remotes/upstream/master',
                     '-v'],
            workdir=ip(CHECKOUT_BASE_DIR),
            name='Add upstream remote to repository.', hideStepIf=True))

    # Prepare project file list to filter out compiler warnings
    f.addStep(cmd("../../tools/VaRA/utils/vara/getVaraSourceFiles.sh",
                  "--vara", "--clang", "--llvm",
                  "--include-existing",
                  "--relative-to", ip(BUILD_DIR),
                  "--output", "buildbot-source-file-list.txt",
                  workdir=ip(BUILD_DIR), hideStepIf=True))

    # Compile Step
    f.addStep(GenerateBuildStepCommand(name="Dummy_3",
                                       command=['cat', 'buildbot-source-file-list.txt'],
                                       workdir=ip(BUILD_DIR),
                                       haltOnFailure=True, hideStepIf=True))

    # Regression Test step
    for step in get_uchroot_workaround_steps():
        f.addStep(step)
    f.addStep(ucompile('ninja', 'check-vara', name='run VaRA regression tests',
                       workdir=UCHROOT_BUILD_DIR,
                       haltOnFailure=False, warnOnWarnings=True))

    # Clang-Tidy
    for step in get_uchroot_workaround_steps():
        f.addStep(step)
    f.addStep(ucompile('python3', 'tidy-vara.py', '-p', UCHROOT_BUILD_DIR, '-j', '8', '--gcc',
                       workdir='vara-llvm/tools/VaRA/test/', name='run Clang-Tidy',
                       haltOnFailure=False, warnOnWarnings=True,
                       timeout=3600))

    # ClangFormat
    f.addStep(GenerateClangFormatStepCommand(name="Dummy_4",
                                             command=['opt/clang-format-static/clang-format',
                                                      '-version'],
                                             workdir=ip('%(prop:uchroot_image_path)s'),
                                             haltOnFailure=True, hideStepIf=True))

    c['builders'].append(builder(PROJECT_NAME, None, ACCEPTED_BUILDERS, tags=['vara'],
                                 factory=f))
Example #7
0
def configure(c):
    llvm_dl = hash_download_from_master("public_html/llvm.tar.gz",
                                        "llvm.tar.gz", "llvm")
    polyjit_dl = hash_download_from_master("public_html/polyjit.tar.gz",
                                           "polyjit.tar.gz", "polyjit")
    steps = [
#        trigger(schedulerNames=['trigger-build-llvm', 'trigger-build-jit']),
        define("scratch", ip("/scratch/pjtest/%(prop:buildnumber)s/"))
    ]
    steps.extend(llvm_dl)
    steps.extend(clean_unpack("llvm.tar.gz", "llvm"))
    steps.extend(polyjit_dl)
    steps.extend(clean_unpack("polyjit.tar.gz", "polyjit"))
    steps.extend([
        define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")),
        git('benchbuild', 'master', codebases, workdir=P("BENCHBUILD_ROOT")),

    ])

    steps.extend([
        define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')),
        define('llvm', ip('%(prop:scratch)s/llvm')),
        define('polyjit', ip('%(prop:scratch)s/polyjit')),

        mkdir(P("scratch")),
        cmd('virtualenv', '-ppython3', ip('%(prop:scratch)s/env/')),
        cmd(ip('%(prop:scratch)s/env/bin/pip3'), 'install', '--upgrade', '.',
            workdir='build/benchbuild'),
        cmd("rsync", "-var", "llvm", P("scratch")),
        cmd("rsync", "-var", "polyjit", P("scratch")),
        cmd(P('benchbuild'), 'bootstrap', '-s', env={
            'BB_CONFIG_FILE': '/scratch/pjtest/.benchbuild.json',
            'BB_TMP_DIR': '/scratch/pjtest/src/',
            'BB_TEST_DIR': P("testinputs"),
            'BB_GENTOO_HTTP_PROXY': 'debussy.fim.uni-passau.de:3128',
            'BB_GENTOO_FTP_PROXY': 'debussy.fim.uni-passau.de:3128',
            'BB_GENTOO_AUTOTEST_LOC': '/scratch/pjtest/gentoo-autotest',
            'BB_SLURM_PARTITION': 'chimaira',
            'BB_SLURM_NODE_DIR': '/local/hdd/pjtest/',
            'BB_SLURM_ACCOUNT': 'cl',
            'BB_SLURM_TIMELIMIT': '24:00:00',
            'BB_ENV_COMPILER_PATH': ip('["%(prop:llvm)s/bin", "%(prop:polyjit)s/bin"]'),
            'BB_ENV_COMPILER_LD_LIBRARY_PATH':
                ip('["%(prop:llvm)s/lib", "%(prop:polyjit)s/lib"]'),
            'BB_ENV_BINARY_PATH': ip('["%(prop:llvm)s/bin", "%(prop:polyjit)s/bin"]'),
            'BB_ENV_BINARY_LD_LIBRARY_PATH':
                ip('["%(prop:llvm)s/lib", "%(prop:polyjit)s/lib"]'),
            'BB_ENV_LOOKUP_PATH':
                ip('["%(prop:llvm)s/bin", "%(prop:polyjit)s/bin", "/scratch/pjtest/erlent/build"]'),
            'BB_ENV_LOOKUP_LD_LIBRARY_PATH':
                ip('["%(prop:llvm)s/lib", "%(prop:polyjit)s/lib"]'),
            'BB_LLVM_DIR': ip('%(prop:scratch)s/llvm'),
            'BB_LIKWID_PREFIX': '/usr',
            'BB_PAPI_INCLUDE': '/usr/include',
            'BB_PAPI_LIBRARY': '/usr/lib',
            'BB_SRC_DIR': ip('%(prop:scratch)s/benchbuild'),
            'BB_SLURM_LOGS': ip('%(prop:scratch)s/slurm.log'),
            'BB_UNIONFS_ENABLE': 'false'
            },
            workdir=P('scratch')),
        # This only works on infosun machines
        cmd("ln", "-s", ip("/scratch/pjtest/benchbuild-src/"),
            ip("%(prop:scratch)s/benchbuild")),
        mkdir(ip("%(prop:scratch)s/results"))])

    c['builders'].append(builder("build-slurm-set", None, accepted_builders,
                         tags=['polyjit'], factory=BuildFactory(steps)))
Example #8
0
def configure(c):
    sb_steps = [
        define("SUPERBUILD_ROOT", ip("%(prop:builddir)s/polli-sb")),
        define("POLYJIT_DEFAULT_BRANCH", "master"),

        git('pjit-polli-sb', 'master', REPOS, workdir=P("SUPERBUILD_ROOT"),
            mode="full", method="fresh"),
        ICMD_P('%(prop:cmake_prefix)s/bin/cmake', P("SUPERBUILD_ROOT"),
               '-DCMAKE_CXX_COMPILER=%(prop:cxx)s',
               '-DCMAKE_C_COMPILER=%(prop:cc)s',
               '-DCMAKE_BUILD_TYPE=Release',
               '-DCMAKE_INSTALL_PREFIX=./_install',
               '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -DLLVM_ENABLE_STATS',
               '-DBUILD_SHARED_LIBS=Off',
               '-DPOLYJIT_PAPI_PREFIX=/scratch/pjtest/opt/papi-5.5.1/install',
               '-DPOLYJIT_BRANCH_CLANG=%(prop:POLYJIT_DEFAULT_BRANCH)s',
               '-DPOLYJIT_BRANCH_LLVM=%(prop:POLYJIT_DEFAULT_BRANCH)s',
               '-DPOLYJIT_BRANCH_POLLI=%(prop:POLYJIT_DEFAULT_BRANCH)s',
               '-DPOLYJIT_BRANCH_POLLY=%(prop:POLYJIT_DEFAULT_BRANCH)s',
               '-G', 'Ninja',
               name="cmake",
               description="[uchroot] cmake: release build",
               descriptionDone="[uchroot] configured."),
        CMD_P("/usr/bin/ninja", "llvm-configure",
              name="configure LLVM",
              description="configuring LLVM",
              descriptionDone="configured LLVM"),
        CMD_P("/usr/bin/ninja", "polli-configure",
              name="configure PolyJIT",
              description="configuring PolyJIT",
              descriptionDone="configured PolyJIT"),
        CMD_P("/usr/bin/ninja",
              name="build jit",
              description="[uchroot] building PolyJIT",
              descriptionDone="[uchroot] built PolyJIT"),
        cmd("tar", "czf", "../polyjit_sb.tar.gz", "-C", "./_install", ".")
    ]

    sb_steps.extend(hash_upload_to_master(
        "polyjit_sb.tar.gz",
        "../polyjit_sb.tar.gz",
        "public_html/polyjit_sb.tar.gz", URL))

    slurm_steps = [
        define("scratch", ip("/scratch/pjtest/sb-%(prop:buildnumber)s/")),
        *hash_download_from_master("public_html/polyjit_sb.tar.gz", "polyjit_sb.tar.gz", "polyjit"),
        *clean_unpack("polyjit_sb.tar.gz", "llvm"),
        define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")),
        git('pjit-benchbuild', 'master', REPOS, workdir=P("BENCHBUILD_ROOT")),
        define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')),
        define('llvm', ip('%(prop:scratch)s/llvm')),
        define('bb_src', ip('%(prop:scratch)s/benchbuild')),
        icmd('virtualenv', '-ppython3', '%(prop:scratch)s/env'),
        icmd('%(prop:scratch)s/env/bin/pip3', 'install', P("BENCHBUILD_ROOT")),
        mkdir(P("scratch")),
        cmd("rsync", "-var", "llvm", P("scratch")),
        cmd(P('benchbuild'), 'bootstrap', '-s', logEnviron=True, env=BB_ENV, workdir=P('scratch')),
        # This only works on infosun machines
        icmd("ln", "-s", "/scratch/pjtest/benchbuild-src/", "%(prop:bb_src)s"),
        mkdir(ip("%(prop:scratch)s/results"))
    ]

    c['builders'].extend([
        builder("polyjit-superbuild", None,
                ACCEPTED_BUILDERS, tags=['polyjit'],
                collapseRequests=True,
                factory=BF(sb_steps)),
        builder("polyjit-superbuild-slurm", None,
                ACCEPTED_BUILDERS, tags=['polyjit'],
                collapseRequests=True,
                factory=BF(slurm_steps))
    ])
Example #9
0
def icmd(*args, **kwargs):
    ip_list = [ ip(le) if isinstance(le, str) else le for le in args]
    return cmd(*ip_list, **kwargs)