Example #1
0
def _addSteps4SystemCompiler(
           f,
           stage_idx = 0,
           clean = True,
           jobs  = None,
           extra_configure_args = None,
           env   = None):

    # Index is zero-based, so we want to use a human friendly  number instead.
    stage_num = stage_idx + 1

    # Directories to use on this stage.
    obj_dir = f.stage_objdirs[stage_idx]
    src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
    install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir)

    # This stage could use incremental build.
    # Clean stage1, only if requested.
    f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
              dir=obj_dir,
              haltOnFailure=False,
              flunkOnFailure=False,
              doStepIf=clean
              ))
    f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx],
              dir=f.stage_installdirs[stage_idx],
              haltOnFailure=False,
              flunkOnFailure=False,
              doStepIf=clean
              ))

    # Reconcile the cmake options for this stage.

    # Make a local copy of the configure args, as we are going to modify that.
    if extra_configure_args:
        cmake_args = extra_configure_args[:]
    else:
        cmake_args = list()

    # Set proper defaults.
    CmakeCommand.applyDefaultOptions(cmake_args, [
        ('-DCMAKE_BUILD_TYPE=',        'Release'),
        ('-DCLANG_BUILD_EXAMPLES=',    'OFF'),
        ('-DLLVM_BUILD_TESTS=',        'ON'),
        ('-DLLVM_ENABLE_ASSERTIONS=',  'OFF'),
        ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
        # Do not expect warning free build by the system toolchain.
        ('-DLLVM_ENABLE_WERROR=',      'OFF'),
        ('-DCMAKE_COLOR_MAKEFILE=',    'OFF'),
        ])

    # Some options are required for this stage no matter what.
    CmakeCommand.applyRequiredOptions(cmake_args, [
        ('-G',                      'Unix Makefiles'),
        ('-DCMAKE_INSTALL_PREFIX=', install_dir),
        ])

    # Note: On this stage we do not care of warnings, as we build with
    # a system toolchain and cannot control the environment.
    # Warnings are likely, and we ignore them.

    # Create configuration files with cmake
    f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num,
                           description=["stage%s cmake configure" % stage_num],
                           haltOnFailure=True,
                           flunkOnWarnings=False,
                           options=cmake_args,
                           path=src_dir,
                           env=env,
                           workdir=obj_dir,
                           doStepIf=FileDoesNotExist("CMakeCache.txt")))

    # Build clang by the system compiler
    f.addStep(ShellCommand(name="build-stage%s-compiler"% stage_num,
                           command=['make', WithProperties("-j%s" % jobs), "-k"],
                           haltOnFailure=True,
                           flunkOnWarnings=False,
                           description=["build stage%s compiler" % stage_num],
                           env=env,
                           workdir=obj_dir))

    # Test stage1 compiler
    f.addStep(ShellCommand(name="test-stage%s-compiler"% stage_num,
                           command=["make", WithProperties("-j%s" % jobs), "check-all"], # or "check-llvm", "check-clang"
                           haltOnFailure=True,
                           flunkOnWarnings=False,
                           description=["test stage%s compiler" % stage_num],
                           env=env,
                           workdir=obj_dir))

    # Install stage1 compiler
    f.addStep(ShellCommand(name="install-stage%s-compiler"% stage_num,
                           command=["make", WithProperties("-j%s" % jobs), "install"],
                           haltOnFailure=True,
                           flunkOnWarnings=False,
                           description=["install stage%s compiler" % stage_num],
                           env=env,
                           workdir=obj_dir))
Example #2
0
def getToolchainBuildFactory(
        clean=False,
        test=True,
        env=None, # Environmental variables for all steps.
        extra_configure_args=None):
    # Prepare environmental variables. Set here all env we want everywhere.
    merged_env = {
        "TERM" : "dumb", # Make sure Clang doesn't use color escape sequences.
    }
    if env is not None:
        # Overwrite pre-set items with the given ones, so user can set anything.
        merged_env.update(env)

    src_dir = "llvm.src"
    obj_dir = "llvm.obj"
    install_dir = "llvm.install"

    f = LLVMBuildFactory(
            depends_on_projects=[
                "llvm",
                "clang",
                "clang-tools-extra",
                "compiler-rt",
                "libcxx",
                "libcxxabi",
                "libunwind",
                "lld"
            ])

    # Get Fuchsia SDK.
    sdk_dir = "fuchsia.sdk"
    sdk_platform = {
        "Linux": "linux-amd64",
        "Darwin": "mac-amd64",
    }[platform.system()]
    sdk_version = "latest"

    sdk_url = WithProperties(
        "https://chrome-infra-packages.appspot.com/dl/"
        "fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s",
        sdk_platform=lambda _: sdk_platform,
        sdk_version=lambda _: sdk_version)

    f.addStep(RemoveDirectory(name="clean-sdk",
                              dir=sdk_dir,
                              haltOnFailure=True))

    f.addStep(ShellCommand(name="fetch-sdk",
                           command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url],
                           description=["download", "fuchsia sdk"],
                           workdir=sdk_dir))

    f.addStep(ShellCommand(name="extract-sdk",
                           command=["unzip", "-fo", "sdk.cipd"],
                           description=["extract", "fuchsia sdk"],
                           workdir=sdk_dir))

    cleanCheckoutRequested = lambda step: step.build.getProperty("clean", default=False) or clean

    # Clean up llvm sources.
    f.addStep(RemoveDirectory(name="clean-llvm.src",
                              dir=src_dir,
                              haltOnFailure=True,
                              doStepIf=cleanCheckoutRequested))

    # Get sources.
    for project in f.depends_on_projects:
        _, baseURL = svn_repos[project]
        f.addStep(SVN(name="svn-%s" % project,
                      workdir=src_dir + "/" + project,
                      baseURL=baseURL))

    cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean

    # Clean up llvm build.
    f.addStep(RemoveDirectory(name="clean-llvm.obj",
                              dir=obj_dir,
                              haltOnFailure=True,
                              doStepIf=cleanBuildRequested))

    # Configure.
    if extra_configure_args is None:
        cmake_options = []
    else:
        cmake_options = extra_configure_args[:]

    # Some options are required for this stage no matter what.
    CmakeCommand.applyRequiredOptions(cmake_options, [
        ("-G",                      "Ninja"),
        ("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"),
        ("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"),
        ])

    # Set proper defaults.
    CmakeCommand.applyDefaultOptions(cmake_options, [
        ("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"),
        ("-DLLVM_ENABLE_LTO=",           "OFF"),
        ])

    cmake_options.append(
        WithProperties(
            "-DCMAKE_INSTALL_PREFIX=%(workdir)s/" + install_dir
        ))
    cmake_options.append(
        WithProperties(
            "-DFUCHSIA_SDK=%(workdir)s/" + sdk_dir
        ))

    CmakeCommand.applyRequiredOptions(cmake_options, [
        ("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"),
        ])

    f.addStep(CmakeCommand(name="cmake-configure",
                           options=cmake_options,
                           path='../' + src_dir + '/llvm',
                           haltOnFailure=True,
                           description=["configure"],
                           workdir=obj_dir,
                           env=merged_env,
                           doStepIf=FileDoesNotExist("CMakeCache.txt")))

    # Build distribution.
    f.addStep(NinjaCommand(name="ninja-build",
                           targets=["stage2-distribution"],
                           haltOnFailure=True,
                           description=["build"],
                           workdir=obj_dir,
                           env=merged_env))

    # Test llvm, clang and lld.
    f.addStep(NinjaCommand(name="check",
                           targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")],
                           haltOnFailure=True,
                           description=["check"],
                           workdir=obj_dir,
                           env=merged_env,
                           doStepIf=test))

    # Install distribution.
    f.addStep(NinjaCommand(name="install",
                           targets=["stage2-install-distribution"],
                           haltOnFailure=True,
                           description=["install"],
                           workdir=obj_dir,
                           env=merged_env))

    return f
Example #3
0
def _getClangCMakeBuildFactory(
        clean=True,
        test=True,
        cmake='cmake',
        jobs=None,

        # VS tools environment variable if using MSVC. For example,
        # %VS120COMNTOOLS% selects the 2013 toolchain.
        vs=None,
        vs_target_arch='x86',

        # Multi-stage compilation
        useTwoStage=False,
        testStage1=True,
        stage1_config='Release',
        stage2_config='Release',

        # Test-suite
        runTestSuite=False,
        nt_flags=None,
        testsuite_flags=None,
        submitURL=None,
        testerName=None,

        # Environmental variables for all steps.
        env=None,
        extra_cmake_args=None,

        # Extra repositories
        checkout_clang_tools_extra=True,
        checkout_compiler_rt=True,
        checkout_lld=True,
        checkout_libcxx=False,
        checkout_test_suite=False,
        checkout_flang=False,

        # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
        stage1_upload_directory=None,

        # Use a lower compression level to generate the build-cache package faster
        # default is 6 according to documentation
        xz_compression_factor=6,
        use_pixz_compression=False,

        # Triggers
        trigger_after_stage1=None):

    ############# PREPARING
    if nt_flags is None:
        nt_flags = []
    if testsuite_flags is None:
        testsuite_flags = []
    if env is None:
        env = {}
    if extra_cmake_args is None:
        extra_cmake_args = []
    if trigger_after_stage1 is None:
        trigger_after_stage1 = []

    clean_build_requested = lambda step: \
        step.build.getProperty( \
            "clean", \
            default=step.build.getProperty("clean_obj") \
        ) or clean

    # We *must* checkout at least Clang+LLVM
    depends_on_projects = ['llvm', 'clang']
    if checkout_clang_tools_extra:
        depends_on_projects.append('clang-tools-extra')
    if checkout_compiler_rt:
        depends_on_projects.append('compiler-rt')
    if checkout_lld:
        depends_on_projects.append('lld')
    if checkout_libcxx:
        depends_on_projects.append('libcxx')
        depends_on_projects.append('libcxxabi')
        depends_on_projects.append('libunwind')
    if checkout_flang:
        depends_on_projects.append('flang')
        depends_on_projects.append('mlir')

    f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
                         llvm_srcdir='llvm')

    # Checkout the latest code for LNT
    # and the test-suite separately. Le's do this first,
    # so we wouldn't poison got_revision property.
    if runTestSuite or checkout_test_suite:
        f.addGetSourcecodeForProject(project='lnt',
                                     src_dir='test/lnt',
                                     alwaysUseLatest=True)
        f.addGetSourcecodeForProject(project='test-suite',
                                     src_dir='test/test-suite',
                                     alwaysUseLatest=True)

    # Then get the LLVM source code revision this particular build is for.
    f.addGetSourcecodeSteps()

    # If jobs not defined, Ninja will choose a suitable value
    jobs_cmd = []
    lit_args = "'-v"
    if jobs is not None:
        jobs_cmd = ["-j" + str(jobs)]
        lit_args += " -j" + str(jobs) + "'"
    else:
        lit_args += "'"
    ninja_cmd = ['ninja'] + jobs_cmd
    ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
    ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd

    # Global configurations
    stage1_build = 'stage1'
    stage1_install = 'stage1.install'
    stage2_build = 'stage2'
    stage2_install = 'stage2.install'

    # Set up VS environment, if appropriate.
    if vs and vs != "manual":
        f.addStep(
            SetProperty(command=builders_util.getVisualStudioEnvironment(
                vs, vs_target_arch),
                        extract_fn=builders_util.extractSlaveEnvironment))
        assert not env, "Can't have custom builder env vars with VS"
        env = Property('slave_env')

    ############# CLEANING
    f.addStep(
        ShellCommand(name='clean stage 1',
                     command=['rm', '-rf', stage1_build],
                     warnOnFailure=True,
                     haltOnFailure=False,
                     flunkOnFailure=False,
                     description='cleaning stage 1',
                     descriptionDone='clean',
                     workdir='.',
                     doStepIf=clean_build_requested))

    ############# STAGE 1
    CmakeCommand.applyRequiredOptions(extra_cmake_args, [
        ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
    ])
    rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage1_build)

    f.addStep(
        ShellCommand(
            name='cmake stage 1',
            command=[
                cmake, "-G", "Ninja", rel_src_dir,
                "-DCMAKE_BUILD_TYPE=" + stage1_config,
                "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
                "-DCMAKE_INSTALL_PREFIX=../" + stage1_install
            ] + extra_cmake_args,
            haltOnFailure=True,
            description='cmake stage 1',
            workdir=stage1_build,
            doStepIf=FileDoesNotExist("build.ninja"),
            env=env))

    f.addStep(
        WarningCountingShellCommand(name='build stage 1',
                                    command=ninja_cmd,
                                    haltOnFailure=True,
                                    description='ninja all',
                                    workdir=stage1_build,
                                    env=env))

    if test and testStage1:
        haltOnStage1Check = not useTwoStage and not runTestSuite
        f.addStep(
            LitTestCommand(name='ninja check 1',
                           command=ninja_check_cmd,
                           haltOnFailure=haltOnStage1Check,
                           description=["checking stage 1"],
                           descriptionDone=["stage 1 checked"],
                           workdir=stage1_build,
                           env=env))

    if useTwoStage or runTestSuite or stage1_upload_directory:
        f.addStep(
            ShellCommand(name='clean stage 1 install',
                         command=['rm', '-rf', stage1_install],
                         warnOnFailure=True,
                         haltOnFailure=False,
                         flunkOnFailure=False,
                         description='cleaning stage 1 install',
                         descriptionDone='clean',
                         workdir='.'))
        f.addStep(
            ShellCommand(name='install stage 1',
                         command=ninja_install_cmd,
                         description='ninja install',
                         workdir=stage1_build,
                         env=env))

    if stage1_upload_directory:
        addGCSUploadSteps(f,
                          'stage 1',
                          stage1_install,
                          stage1_upload_directory,
                          env,
                          gcs_url_property='stage1_package_gcs_url',
                          use_pixz_compression=use_pixz_compression,
                          xz_compression_factor=xz_compression_factor)

    # Compute the cmake define flag to set the C and C++ compiler to clang. Use
    # clang-cl if we used MSVC for stage1.
    if not vs:
        cc = 'clang'
        cxx = 'clang++'
    else:
        cc = 'clang-cl.exe'
        cxx = 'clang-cl.exe'

    ############# STAGE 2
    if useTwoStage:
        # We always cleanly build the stage 2. If the compiler has been
        # changed on the stage 1, we cannot trust any of the intermediate file
        # from the old compiler. And if the stage 1 compiler is the same, we
        # should not build in the first place.
        f.addStep(
            ShellCommand(name='clean stage 2',
                         command=['rm', '-rf', stage2_build],
                         warnOnFailure=True,
                         description='cleaning stage 2',
                         descriptionDone='clean',
                         workdir='.'))

        # Set the compiler using the CC and CXX environment variables to work around
        # backslash string escaping bugs somewhere between buildbot and cmake. The
        # env.exe helper is required to run the tests, so hopefully it's already on
        # PATH.
        rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir,
                                                      stage2_build)
        cmake_cmd2 = [
            cmake, "-G", "Ninja", rel_src_dir,
            WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + stage1_install +
                           "/bin/" + cc),
            WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" +
                           stage1_install + "/bin/" + cxx),
            "-DCMAKE_BUILD_TYPE=" + stage2_config,
            "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
            "-DCMAKE_INSTALL_PREFIX=../" + stage2_install
        ] + extra_cmake_args

        f.addStep(
            ShellCommand(name='cmake stage 2',
                         command=cmake_cmd2,
                         haltOnFailure=True,
                         description='cmake stage 2',
                         workdir=stage2_build,
                         env=env))

        f.addStep(
            WarningCountingShellCommand(name='build stage 2',
                                        command=ninja_cmd,
                                        haltOnFailure=True,
                                        description='ninja all',
                                        workdir=stage2_build,
                                        env=env))

        if test:
            f.addStep(
                LitTestCommand(name='ninja check 2',
                               command=ninja_check_cmd,
                               haltOnFailure=not runTestSuite,
                               description=["checking stage 2"],
                               descriptionDone=["stage 2 checked"],
                               workdir=stage2_build,
                               env=env))

    ############# TEST SUITE
    ## Test-Suite (stage 2 if built, stage 1 otherwise)
    if runTestSuite:
        compiler_path = stage1_install
        if useTwoStage:
            compiler_path = stage2_install
            f.addStep(
                ShellCommand(name='clean stage 2 install',
                             command=['rm', '-rf', stage2_install],
                             warnOnFailure=True,
                             description='cleaning stage 2 install',
                             descriptionDone='clean',
                             workdir='.'))
            f.addStep(
                ShellCommand(name='install stage 2',
                             command=ninja_install_cmd,
                             description='ninja install 2',
                             workdir=stage2_build,
                             env=env))

        # Get generated python, lnt
        python = WithProperties('%(workdir)s/test/sandbox/bin/python')
        lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
        lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')

        # Paths
        sandbox = WithProperties('%(workdir)s/test/sandbox')
        test_suite_dir = WithProperties('%(workdir)s/test/test-suite')

        # Get latest built Clang (stage1 or stage2)
        cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
        cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)

        # LNT Command line (don't pass -jN. Users need to pass both --threads
        # and --build-threads in nt_flags/test_suite_flags to get the same effect)
        use_runtest_testsuite = len(nt_flags) == 0
        if not use_runtest_testsuite:
            test_suite_cmd = [
                python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
                sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx',
                cxx
            ]
            # Append any option provided by the user
            test_suite_cmd.extend(nt_flags)
        else:
            lit = WithProperties('%(workdir)s/' + stage1_build +
                                 '/bin/llvm-lit')
            test_suite_cmd = [
                python, lnt, 'runtest', 'test-suite', '--no-timestamp',
                '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc',
                cc, '--cxx', cxx, '--use-lit', lit
            ]
            # Append any option provided by the user
            test_suite_cmd.extend(testsuite_flags)

        # Only submit if a URL has been specified
        if submitURL is not None:
            if not isinstance(submitURL, list):
                submitURL = [submitURL]
            for url in submitURL:
                test_suite_cmd.extend(['--submit', url])
            # lnt runtest test-suite doesn't understand --no-machdep-info:
            if testerName and not use_runtest_testsuite:
                test_suite_cmd.extend(['--no-machdep-info', testerName])
        # CC and CXX are needed as env for build-tools
        test_suite_env = copy.deepcopy(env)
        test_suite_env['CC'] = cc
        test_suite_env['CXX'] = cxx

        # Steps to prepare, build and run LNT
        f.addStep(
            ShellCommand(name='clean sandbox',
                         command=['rm', '-rf', 'sandbox'],
                         haltOnFailure=True,
                         description='removing sandbox directory',
                         workdir='test',
                         env=env))
        f.addStep(
            ShellCommand(name='recreate sandbox',
                         command=['virtualenv', 'sandbox'],
                         haltOnFailure=True,
                         description='recreating sandbox',
                         workdir='test',
                         env=env))
        f.addStep(
            ShellCommand(name='setup lit',
                         command=[python, lnt_setup, 'develop'],
                         haltOnFailure=True,
                         description='setting up LNT in sandbox',
                         workdir='test/sandbox',
                         env=env))
        f.addStep(
            LitTestCommand(name='test-suite',
                           command=test_suite_cmd,
                           haltOnFailure=True,
                           description=['running the test suite'],
                           workdir='test/sandbox',
                           logfiles={
                               'configure.log': 'build/configure.log',
                               'build-tools.log': 'build/build-tools.log',
                               'test.log': 'build/test.log',
                               'report.json': 'build/report.json'
                           },
                           env=test_suite_env))

    return f
Example #4
0
def _addSteps4StagedCompiler(
           f,
           stage_idx = 1,
           use_stage_idx = -1,
           withLTOSupport = False,
           jobs = None,
           extra_configure_args = None,
           env = None):

    if use_stage_idx < 0:
        use_stage_idx = stage_idx - 1

    # Index is zero-based, so we want to use a human friendly  number instead.
    stage_num = stage_idx + 1

    # Directories to use on this stage.
    obj_dir = f.stage_objdirs[stage_idx]
    src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
    install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir)
    staged_install = f.stage_installdirs[use_stage_idx]

    # Always do a clean build for the staged compiler.
    f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
              dir=obj_dir,
              haltOnFailure=False,
              flunkOnFailure=False,
              ))

    f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx],
              dir=f.stage_installdirs[stage_idx],
              haltOnFailure=False,
              flunkOnFailure=False,
              ))

    # Reconcile the cmake options for this stage.

    # Make a local copy of the configure args, as we are going to modify that.
    if extra_configure_args:
        cmake_args = extra_configure_args[:]
    else:
        cmake_args = list()

    # Set proper defaults.
    CmakeCommand.applyDefaultOptions(cmake_args, [
        ('-DCMAKE_BUILD_TYPE=',        'Release'),
        ('-DCLANG_BUILD_EXAMPLES=',    'OFF'),
        ('-DLLVM_BUILD_TESTS=',        'ON'),
        ('-DLLVM_ENABLE_ASSERTIONS=',  'ON'),
        ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
        ])
    if withLTOSupport:
        CmakeCommand.applyDefaultOptions(cmake_args, [
            # LTO Plugin dependency:
            ('-DLLVM_BINUTILS_INCDIR=',    '/opt/binutils/include'),
            ])

    # Some options are required for this stage no matter what.
    CmakeCommand.applyRequiredOptions(cmake_args, [
        ('-G',                      'Ninja'),
        ('-DCMAKE_INSTALL_PREFIX=', install_dir),
        ])

    cmake_args.append(
        WithProperties(
            "-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++"
        ))
    cmake_args.append(
        WithProperties(
            "-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang"
        ))

    # Create configuration files with cmake
    f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num,
                           description=["stage%s cmake configure" % stage_num],
                           haltOnFailure=True,
                           options=cmake_args,
                           path=src_dir,
                           env=env,
                           workdir=obj_dir,
                           doStepIf=FileDoesNotExist("CMakeCache.txt")
                           ))
    if withLTOSupport:
        # Build LTO plugin if requested.
        f.addStep(NinjaCommand(name="build-stage%s-LLVMgold.so" % stage_num,
                               targets=['lib/LLVMgold.so'],
                               haltOnFailure=True,
                               description=["stage%s build LLVMgold.so" % stage_num],
                               env=env,
                               workdir=obj_dir,
                               ))

    # Build clang by the staged compiler
    f.addStep(NinjaCommand(name="build-stage%s-compiler" % stage_num,
                           haltOnFailure=True,
                           description=["build stage%s compiler" % stage_num],
                           timeout=10800, # LTO could take time.
                           env=env,
                           workdir=obj_dir,
                           ))

    # Test just built compiler
    f.addStep(NinjaCommand(name="test-stage%s-compiler"% stage_num,
                           targets=["check-all"],
                           haltOnFailure=True,
                           description=["test stage%s compiler" % stage_num],
                           timeout=10800, # LTO could take time.
                           env=env,
                           workdir=obj_dir,
                           ))

    # Install just built compiler
    f.addStep(NinjaCommand(name="install-stage%s-compiler"% stage_num,
                           targets=["install"],
                           haltOnFailure=True,
                           description=["install stage%s compiler" % stage_num],
                           timeout=10800, # LTO could take time.
                           env=env,
                           workdir=obj_dir,
                           ))
Example #5
0
def getCmakeBuildFactory(depends_on_projects=None,
                         llvm_srcdir=None,
                         obj_dir=None,
                         install_dir=None,
                         clean=False,
                         extra_configure_args=None,
                         env=None,
                         **kwargs):

    # Set defaults
    if not depends_on_projects:
        depends_on_projects = ['llvm', 'clang']

    if extra_configure_args is None:
        extra_configure_args = []

    # Prepare environmental variables. Set here all env we want everywhere.
    merged_env = {
        'TERM': 'dumb'  # Be cautious and disable color output from all tools.
    }
    if env is not None:
        # Overwrite pre-set items with the given ones, so user can set anything.
        merged_env.update(env)

    if not obj_dir:
        obj_dir = "build"

    if install_dir:
        install_dir_rel = LLVMBuildFactory.pathRelativeToBuild(
            install_dir, obj_dir)
        CmakeCommand.applyRequiredOptions(extra_configure_args, [
            ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
        ])

    cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean

    f = LLVMBuildFactory(
        depends_on_projects=depends_on_projects,
        llvm_srcdir=llvm_srcdir or "llvm.src",
        obj_dir=obj_dir,
        install_dir=install_dir,
        cleanBuildRequested=cleanBuildRequested,
        **kwargs  # Pass through all the extra arguments.
    )

    # Directories to use on this stage.
    src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)

    # Do a clean checkout if requested.
    f.addStep(
        RemoveDirectory(
            name='clean-src-dir',
            dir=f.llvm_srcdir,
            haltOnFailure=False,
            flunkOnFailure=False,
            doStepIf=cleanBuildRequested,
        ))

    # Get the source code.
    f.addSVNSteps()

    # This is an incremental build, unless otherwise has been requested.
    # Remove obj and install dirs for a clean build.
    f.addStep(
        RemoveDirectory(
            name='clean-%s-dir' % f.obj_dir,
            dir=f.obj_dir,
            haltOnFailure=False,
            flunkOnFailure=False,
            doStepIf=cleanBuildRequested,
        ))

    if f.install_dir:
        f.addStep(
            RemoveDirectory(
                name='clean-%s-dir' % f.install_dir,
                dir=f.install_dir,
                haltOnFailure=False,
                flunkOnFailure=False,
                doStepIf=cleanBuildRequested,
            ))

    # Reconcile the cmake options for this build.

    # Make a local copy of the configure args, as we are going to modify that.
    if extra_configure_args:
        cmake_args = extra_configure_args[:]
    else:
        cmake_args = list()

    # Set proper defaults.
    CmakeCommand.applyDefaultOptions(cmake_args, [
        ('-DCMAKE_BUILD_TYPE=', 'Release'),
        ('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
        ('-DLLVM_BUILD_TESTS=', 'ON'),
        ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
        ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
    ])

    # Create configuration files with cmake, unless this has been already done
    # for an incremental build.
    f.addStep(
        CmakeCommand(
            name="cmake-configure",
            description=["cmake configure"],
            haltOnFailure=True,
            options=cmake_args,
            path=src_dir,
            env=env,
            workdir=obj_dir,
            doStepIf=FileDoesNotExist("CMakeCache.txt"),
            **kwargs  # Pass through all the extra arguments.
        ))

    return f
Example #6
0
def getSanitizerBuildFactoryII(
        clean=False,
        sanity_check=True,
        sanitizers=[
            'sanitizer', 'asan', 'lsan', 'msan', 'tsan', 'ubsan', 'dfsan'
        ],
        common_cmake_options=None,  # FIXME: For backward compatibility. Will be removed.
        extra_configure_args=[],
        prefixCommand=["nice", "-n", "10"],  # For backward compatibility.
        env=None,
        jobs="%(jobs)s",
        timeout=1200):

    llvm_srcdir = "llvm.src"
    llvm_objdir = "llvm.obj"

    # Prepare environmental variables. Set here all env we want everywhere.
    merged_env = {
        'TERM': 'dumb',  # Make sure Clang doesn't use color escape sequences.
    }
    if env:
        # Overwrite pre-set items with the given ones, so user can set anything.
        merged_env.update(env)

    f = BuildFactory()

    # Clean directory, if requested.
    cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
    f.addStep(
        RemoveDirectory(name='clean ' + llvm_objdir,
                        dir=llvm_objdir,
                        haltOnFailure=False,
                        flunkOnFailure=False,
                        doStepIf=cleanBuildRequested))

    # Get llvm, clang, ompiler-rt, libcxx, libcxxabi, libunwind
    f.addStep(
        SVN(name='svn-llvm',
            mode='update',
            description='svn-llvm',
            descriptionDone='svn-llvm',
            baseURL='http://llvm.org/svn/llvm-project/llvm/',
            defaultBranch='trunk',
            workdir=llvm_srcdir))

    f.addStep(
        SVN(name='svn-clang',
            mode='update',
            description='svn-clang',
            descriptionDone='svn-clang',
            baseURL='http://llvm.org/svn/llvm-project/cfe/',
            defaultBranch='trunk',
            workdir='%s/tools/clang' % llvm_srcdir))

    f.addStep(
        SVN(name='svn-compiler-rt',
            mode='update',
            description='svn-compiler-rt',
            descriptionDone='svn--compiler-rt',
            baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
            defaultBranch='trunk',
            workdir='%s/projects/compiler-rt' % llvm_srcdir))

    f.addStep(
        SVN(name='svn-libcxx',
            mode='update',
            description='svn-libcxx',
            descriptionDone='svn-libcxx',
            baseURL='http://llvm.org/svn/llvm-project/libcxx/',
            defaultBranch='trunk',
            workdir='%s/projects/libcxx' % llvm_srcdir))

    f.addStep(
        SVN(name='svn-libcxxabi',
            mode='update',
            description='svn-libcxxabi',
            descriptionDone='svn-libcxxabi',
            baseURL='http://llvm.org/svn/llvm-project/libcxxabi/',
            defaultBranch='trunk',
            workdir='%s/projects/libcxxabi' % llvm_srcdir))

    f.addStep(
        SVN(name='svn-libunwind',
            mode='update',
            description='svn-libunwind',
            descriptionDone='svn-libunwind',
            baseURL='http://llvm.org/svn/llvm-project/libunwind/',
            defaultBranch='trunk',
            workdir='%s/projects/libunwind' % llvm_srcdir))

    # Run annotated command for sanitizer.
    if sanity_check:
        f.addStep(
            AnnotatedCommand(
                name="lint check",
                description="lint check",
                timeout=timeout,
                haltOnFailure=False,  #True,
                warnOnWarnings=True,
                command=["./check_lint.sh"],
                workdir="%s/projects/compiler-rt/lib/sanitizer_common/scripts"
                % llvm_srcdir,
                env=merged_env))

    # Always build with ninja.
    cmakeCommand = ["cmake", "-G", "Ninja"]

    # Reconsile configure args with the defaults we want.
    if not any(
            a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args):
        cmakeCommand.append('-DCMAKE_BUILD_TYPE=Release')
    if not any(
            a.startswith('-DLLVM_ENABLE_WERROR=')
            for a in extra_configure_args):
        cmakeCommand.append('-DLLVM_ENABLE_WERROR=OFF')
    if not any(
            a.startswith('-DLLVM_ENABLE_ASSERTIONS=')
            for a in extra_configure_args):
        cmakeCommand.append('-DLLVM_ENABLE_ASSERTIONS=ON')
    if not any(
            a.startswith('-DCMAKE_C_COMPILER') for a in extra_configure_args):
        cmakeCommand.append('-DCMAKE_C_COMPILER=clang')
    if not any(
            a.startswith('-DCMAKE_CXX_COMPILER')
            for a in extra_configure_args):
        cmakeCommand.append('-DCMAKE_CXX_COMPILER=clang++')
    if not any(
            a.startswith('-DCMAKE_CXX_FLAGS') for a in extra_configure_args):
        cmakeCommand.append('-DCMAKE_CXX_FLAGS=\"-std=c++11 -stdlib=libc++\"')
    if not any(
            a.startswith('-DCMAKE_EXE_LINKER_FLAGS')
            for a in extra_configure_args):
        cmakeCommand.append('-DCMAKE_EXE_LINKER_FLAGS=-lcxxrt')
    if not any(
            a.startswith('-DLIBCXXABI_USE_LLVM_UNWINDER=')
            for a in extra_configure_args):
        cmakeCommand.append('-DLIBCXXABI_USE_LLVM_UNWINDER=ON')
    if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args):
        cmakeCommand.append('-DLLVM_LIT_ARGS=\"-v\"')

    cmakeCommand += extra_configure_args + ["../%s" % llvm_srcdir]

    # Note: ShellCommand does not pass the params with special symbols right.
    # The " ".join is a workaround for this bug.
    f.addStep(
        ShellCommand(
            name="cmake-configure",
            description=["cmake configure"],
            haltOnFailure=False,  #True,
            warnOnWarnings=True,
            command=WithProperties(" ".join(cmakeCommand)),
            env=merged_env,
            workdir=llvm_objdir,
            doStepIf=FileDoesNotExist("./%s/CMakeCache.txt" % llvm_objdir)))

    # Build everything.
    f.addStep(
        NinjaCommand(
            name='build',
            haltOnFailure=False,  #True,
            warnOnWarnings=True,
            description=['building', 'with', 'ninja'],
            descriptionDone=['built', 'with', 'ninja'],
            workdir=llvm_objdir,
            env=merged_env))

    # Run tests for each of the requested sanitizers.
    if sanitizers:
        for s in sanitizers:
            f.addStep(
                NinjaCommand(
                    name='test %s' % s,
                    targets=['check-%s' % s],
                    haltOnFailure=False,  #True,
                    description=['testing', '%s' % s],
                    descriptionDone=['test', '%s' % s],
                    workdir=llvm_objdir,
                    env=merged_env))

    return f
Example #7
0
def getLLDWinBuildFactory(
           clean = True,

           # Default values for VS devenv and build configuration
           vs = None,          # What to run to configure Visual Studio utils.
           target_arch = None, # Native.

           extra_configure_args = None,
           env   = None):

    # Set defaults
    if vs is None:
        vs = r"""%VS140COMNTOOLS%"""   # Visual Studio 2015.
    if extra_configure_args is None:
        extra_configure_args = []
    if env is None:
        env = {}

    f = LLVMBuildFactory(
            depends_on_projects=['llvm', 'lld'],
            llvm_srcdir="llvm.src",
            llvm_objdir="llvm.obj")

    # Get LLVM and Lld
    f.addSVNSteps()

    # Clean directory, if requested.
    cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
    f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir,
              dir=f.llvm_objdir,
              haltOnFailure=False,
              flunkOnFailure=False,
              doStepIf=cleanBuildRequested
              ))

    # If set up environment step is requested, do this now.
    if vs:
        f.addStep(SetProperty(
            command=getVisualStudioEnvironment(vs, target_arch),
            extract_fn=extractSlaveEnvironment))
        assert not env, "Can't have custom builder env vars with VS"
        env = Property('slave_env')

    # Always build with ninja.
    cmake_options = ["-G", "Ninja"]
    # Reconsile configure args with the defaults we want.
    if not any(a.startswith('-DCMAKE_BUILD_TYPE=')   for a in extra_configure_args):
        cmake_options.append('-DCMAKE_BUILD_TYPE=Release')
    if not any(a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args):
        cmake_options.append('-DLLVM_ENABLE_WERROR=ON')
    if not any(a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args):
        cmake_options.append('-DLLVM_ENABLE_ASSERTIONS=ON')
    if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args):
        cmake_options.append('-DLLVM_LIT_ARGS=\"-v\"')

    cmake_options += extra_configure_args

    # Note: ShellCommand does not pass the params with special symbols right.
    # The " ".join is a workaround for this bug.
    f.addStep(CmakeCommand(name="cmake-configure",
                           description=["cmake configure"],
                           haltOnFailure=True,
                           warnOnWarnings=True,
                           options=cmake_options,
                           path="../%s" % f.llvm_srcdir,
                           env=env,
                           workdir=f.llvm_objdir,
                           doStepIf=FileDoesNotExist(
                                        "./%s/CMakeCache.txt" % f.llvm_objdir)))

    # Build Lld.
    f.addStep(NinjaCommand(name='build lld',
                           haltOnFailure=True,
                           warnOnWarnings=True,
                           description='build lld',
                           workdir=f.llvm_objdir,
                           env=env))

    # Test Lld
    f.addStep(NinjaCommand(name='test lld',
                           targets=['lld-test'],
                           haltOnFailure=True,
                           warnOnWarnings=True,
                           description='test lld',
                           workdir=f.llvm_objdir,
                           env=env))

    return f
Example #8
0
def getLLDBuildFactory(
           clean = True,
           jobs  = None,
           extra_configure_args = None,
           env   = None):

    # Set defaults
    if jobs is None:
        jobs = "%(jobs)s"
    if extra_configure_args is None:
        extra_configure_args = []

    # Prepare environmental variables. Set here all env we want everywhere.
    merged_env = {
                   'CC'   : "clang",
                   'CXX'  : "clang++",
                   'TERM' : 'dumb'     # Be cautious and disable color output from all tools.
                 }
    if env is not None:
        # Overwrite pre-set items with the given ones, so user can set anything.
        merged_env.update(env)

    f = LLVMBuildFactory(
            depends_on_projects=['llvm', 'lld'],
            llvm_srcdir="llvm.src",
            llvm_objdir="llvm.obj")

    # Get LLVM and Lld
    f.addSVNSteps()

    # Clean directory, if requested.
    cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
    f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir,
              dir=f.llvm_objdir,
              haltOnFailure=False,
              flunkOnFailure=False,
              doStepIf=cleanBuildRequested
              ))

    # Create configuration files with cmake
    f.addStep(CmakeCommand(name="cmake-configure",
                           description=["cmake configure"],
                           haltOnFailure=True,
                           options=extra_configure_args,
                           path="../%s" % f.llvm_srcdir,
                           env=merged_env,
                           workdir=f.llvm_objdir,
                           doStepIf=FileDoesNotExist(
                                        "./%s/CMakeCache.txt" % f.llvm_objdir)))

    # Build Lld
    f.addStep(ShellCommand(name="build_Lld",
                           command=['nice', '-n', '10',
                                    'make', WithProperties("-j%s" % jobs)],
                           haltOnFailure=True,
                           description=["build lld"],
                           env=merged_env,
                           workdir=f.llvm_objdir))

    # Test Lld
    f.addStep(ShellCommand(name="test_lld",
                           command=["make", "lld-test"],
                           haltOnFailure=True,
                           description=["test lld"],
                           env=merged_env,
                           workdir=f.llvm_objdir))

    return f
Example #9
0
def _getClangCMakeBuildFactory(
        clean=True,
        test=True,
        cmake='cmake',
        jobs=None,

        # VS tools environment variable if using MSVC. For example,
        # %VS120COMNTOOLS% selects the 2013 toolchain.
        vs=None,
        vs_target_arch='x86',

        # Multi-stage compilation
        useTwoStage=False,
        testStage1=True,
        stage1_config='Release',
        stage2_config='Release',

        # Test-suite
        runTestSuite=False,
        nt_flags=[],
        testsuite_flags=[],
        submitURL=None,
        testerName=None,

        # Environmental variables for all steps.
        env={},
        extra_cmake_args=[],

        # Extra repositories
        checkout_clang_tools_extra=True,
        checkout_compiler_rt=True,
        checkout_lld=True,
        checkout_libcxx=False,
        checkout_test_suite=False,

        # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
        stage1_upload_directory=None,

        # Triggers
        trigger_after_stage1=[]):

    ############# PREPARING
    f = buildbot.process.factory.BuildFactory()

    clean_build_requested = lambda step: clean or step.build.getProperty(
        "clean")

    addSVNUpdateSteps(f,
                      checkout_clang_tools_extra=checkout_clang_tools_extra,
                      checkout_compiler_rt=checkout_compiler_rt,
                      checkout_lld=checkout_lld,
                      checkout_test_suite=runTestSuite or checkout_test_suite,
                      checkout_libcxx=checkout_libcxx)

    # If jobs not defined, Ninja will choose a suitable value
    jobs_cmd = []
    lit_args = "'-v"
    if jobs is not None:
        jobs_cmd = ["-j" + str(jobs)]
        lit_args += " -j" + str(jobs) + "'"
    else:
        lit_args += "'"
    ninja_cmd = ['ninja'] + jobs_cmd
    ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
    ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd

    # Global configurations
    stage1_build = 'stage1'
    stage1_install = 'stage1.install'
    stage2_build = 'stage2'
    stage2_install = 'stage2.install'

    # Set up VS environment, if appropriate.
    if vs:
        f.addStep(
            SetProperty(command=builders_util.getVisualStudioEnvironment(
                vs, vs_target_arch),
                        extract_fn=builders_util.extractSlaveEnvironment))
        assert not env, "Can't have custom builder env vars with VS"
        env = Property('slave_env')

    ############# CLEANING
    f.addStep(
        ShellCommand(name='clean stage 1',
                     command=['rm', '-rf', stage1_build],
                     warnOnFailure=True,
                     haltOnFailure=False,
                     flunkOnFailure=False,
                     description='cleaning stage 1',
                     descriptionDone='clean',
                     workdir='.',
                     doStepIf=clean_build_requested))

    ############# STAGE 1
    f.addStep(
        ShellCommand(
            name='cmake stage 1',
            command=[
                cmake, "-G", "Ninja", "../llvm",
                "-DCMAKE_BUILD_TYPE=" + stage1_config,
                "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
                "-DCMAKE_INSTALL_PREFIX=../" + stage1_install
            ] + extra_cmake_args,
            haltOnFailure=True,
            description='cmake stage 1',
            workdir=stage1_build,
            doStepIf=FileDoesNotExist("build.ninja"),
            env=env))

    f.addStep(
        WarningCountingShellCommand(name='build stage 1',
                                    command=ninja_cmd,
                                    haltOnFailure=True,
                                    description='ninja all',
                                    workdir=stage1_build,
                                    env=env))

    if test and testStage1:
        haltOnStage1Check = not useTwoStage and not runTestSuite
        f.addStep(
            lit_test_command.LitTestCommand(
                name='ninja check 1',
                command=ninja_check_cmd,
                haltOnFailure=haltOnStage1Check,
                description=["checking stage 1"],
                descriptionDone=["stage 1 checked"],
                workdir=stage1_build,
                env=env))

    if useTwoStage or runTestSuite or stage1_upload_directory:
        f.addStep(
            ShellCommand(name='clean stage 1 install',
                         command=['rm', '-rf', stage1_install],
                         warnOnFailure=True,
                         haltOnFailure=False,
                         flunkOnFailure=False,
                         description='cleaning stage 1 install',
                         descriptionDone='clean',
                         workdir='.'))
        f.addStep(
            ShellCommand(name='install stage 1',
                         command=ninja_install_cmd,
                         description='ninja install',
                         workdir=stage1_build,
                         env=env))

    if stage1_upload_directory:
        addGCSUploadSteps(f,
                          'stage 1',
                          stage1_install,
                          stage1_upload_directory,
                          env,
                          gcs_url_property='stage1_package_gcs_url')

    # Compute the cmake define flag to set the C and C++ compiler to clang. Use
    # clang-cl if we used MSVC for stage1.
    if not vs:
        cc = 'clang'
        cxx = 'clang++'
    else:
        cc = 'clang-cl.exe'
        cxx = 'clang-cl.exe'

    ############# STAGE 2
    if useTwoStage:
        # We always cleanly build the stage 2. If the compiler has been
        # changed on the stage 1, we cannot trust any of the intermediate file
        # from the old compiler. And if the stage 1 compiler is the same, we
        # should not build in the first place.
        f.addStep(
            ShellCommand(name='clean stage 2',
                         command=['rm', '-rf', stage2_build],
                         warnOnFailure=True,
                         description='cleaning stage 2',
                         descriptionDone='clean',
                         workdir='.'))

        # Set the compiler using the CC and CXX environment variables to work around
        # backslash string escaping bugs somewhere between buildbot and cmake. The
        # env.exe helper is required to run the tests, so hopefully it's already on
        # PATH.
        cmake_cmd2 = [
            'env',
            WithProperties('CC=%(workdir)s/' + stage1_install + '/bin/' + cc),
            WithProperties('CXX=%(workdir)s/' + stage1_install + '/bin/' +
                           cxx), cmake, "-G", "Ninja", "../llvm",
            "-DCMAKE_BUILD_TYPE=" + stage2_config,
            "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
            "-DCMAKE_INSTALL_PREFIX=../" + stage2_install
        ] + extra_cmake_args

        f.addStep(
            ShellCommand(name='cmake stage 2',
                         command=cmake_cmd2,
                         haltOnFailure=True,
                         description='cmake stage 2',
                         workdir=stage2_build,
                         env=env))

        f.addStep(
            WarningCountingShellCommand(name='build stage 2',
                                        command=ninja_cmd,
                                        haltOnFailure=True,
                                        description='ninja all',
                                        workdir=stage2_build,
                                        env=env))

        if test:
            f.addStep(
                lit_test_command.LitTestCommand(
                    name='ninja check 2',
                    command=ninja_check_cmd,
                    haltOnFailure=not runTestSuite,
                    description=["checking stage 2"],
                    descriptionDone=["stage 2 checked"],
                    workdir=stage2_build,
                    env=env))

    ############# TEST SUITE
    ## Test-Suite (stage 2 if built, stage 1 otherwise)
    if runTestSuite:
        compiler_path = stage1_install
        if useTwoStage:
            compiler_path = stage2_install
            f.addStep(
                ShellCommand(name='clean stage 2 install',
                             command=['rm', '-rf', stage2_install],
                             warnOnFailure=True,
                             description='cleaning stage 2 install',
                             descriptionDone='clean',
                             workdir='.'))
            f.addStep(
                ShellCommand(name='install stage 2',
                             command=ninja_install_cmd,
                             description='ninja install 2',
                             workdir=stage2_build,
                             env=env))

        # Get generated python, lnt
        python = WithProperties('%(workdir)s/test/sandbox/bin/python')
        lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
        lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')

        # Paths
        sandbox = WithProperties('%(workdir)s/test/sandbox')
        test_suite_dir = WithProperties('%(workdir)s/test/test-suite')

        # Get latest built Clang (stage1 or stage2)
        cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
        cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)

        # LNT Command line (don't pass -jN. Users need to pass both --threads
        # and --build-threads in nt_flags/test_suite_flags to get the same effect)
        use_runtest_testsuite = len(nt_flags) == 0
        if not use_runtest_testsuite:
            test_suite_cmd = [
                python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
                sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx',
                cxx
            ]
            # Append any option provided by the user
            test_suite_cmd.extend(nt_flags)
        else:
            lit = WithProperties('%(workdir)s/' + stage1_build +
                                 '/bin/llvm-lit')
            test_suite_cmd = [
                python, lnt, 'runtest', 'test-suite', '--no-timestamp',
                '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc',
                cc, '--cxx', cxx, '--use-lit', lit
            ]
            # Append any option provided by the user
            test_suite_cmd.extend(testsuite_flags)

        # Only submit if a URL has been specified
        if submitURL is not None:
            if not isinstance(submitURL, list):
                submitURL = [submitURL]
            for url in submitURL:
                test_suite_cmd.extend(['--submit', url])
            # lnt runtest test-suite doesn't understand --no-machdep-info:
            if testerName and not use_runtest_testsuite:
                test_suite_cmd.extend(['--no-machdep-info', testerName])
        # CC and CXX are needed as env for build-tools
        test_suite_env = copy.deepcopy(env)
        test_suite_env['CC'] = cc
        test_suite_env['CXX'] = cxx

        # Steps to prepare, build and run LNT
        f.addStep(
            ShellCommand(name='clean sandbox',
                         command=['rm', '-rf', 'sandbox'],
                         haltOnFailure=True,
                         description='removing sandbox directory',
                         workdir='test',
                         env=env))
        f.addStep(
            ShellCommand(name='recreate sandbox',
                         command=['virtualenv', 'sandbox'],
                         haltOnFailure=True,
                         description='recreating sandbox',
                         workdir='test',
                         env=env))
        f.addStep(
            ShellCommand(name='setup lit',
                         command=[python, lnt_setup, 'develop'],
                         haltOnFailure=True,
                         description='setting up LNT in sandbox',
                         workdir='test/sandbox',
                         env=env))
        f.addStep(
            commands.LitTestCommand.LitTestCommand(
                name='test-suite',
                command=test_suite_cmd,
                haltOnFailure=True,
                description=['running the test suite'],
                workdir='test/sandbox',
                logfiles={
                    'configure.log': 'build/configure.log',
                    'build-tools.log': 'build/build-tools.log',
                    'test.log': 'build/test.log',
                    'report.json': 'build/report.json'
                },
                env=test_suite_env))

    return f