def getClangWithLTOBuildFactory( depends_on_projects=None, clean=False, jobs=None, extra_configure_args=None, extra_configure_args_lto_stage=None, compare_last_2_stages=True, lto=None, # The string gets passed to -flto flag as is. Like -flto=thin. env=None, **kwargs): # Set defaults if depends_on_projects: depends_on_projects = list(depends_on_projects) else: # By default we link with LLD. depends_on_projects = ['llvm', 'clang', 'lld'] if lto is None: lto = 'ON' if extra_configure_args is None: extra_configure_args = [] else: extra_configure_args = list(extra_configure_args) if extra_configure_args_lto_stage is None: extra_configure_args_lto_stage = [] else: extra_configure_args_lto_stage = list(extra_configure_args_lto_stage) # Make sure CMAKE_INSTALL_PREFIX and -G are not specified # in the extra_configure_args. We set them internally as needed. # TODO: assert extra_configure_args. install_prefix_specified = (any( a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) or any( a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args_lto_stage)) assert not install_prefix_specified, "Please do not explicitly specify the install prefix for multi-stage build." # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) f = UnifiedTreeBuilder.getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects=depends_on_projects, stage_objdirs=[ "build/stage1", "build/stage2", "build/stage3", "build/stage4", ], stage_installdirs=[ "install/stage1", "install/stage2", "install/stage3", "install/stage4", ], staged_compiler_idx=1, **kwargs) cleanBuildRequested = lambda step: clean or step.build.getProperty( "clean", default=step.build.getProperty("clean_obj")) # Get the source code. f.addGetSourcecodeSteps() # Build with the system compiler first _addSteps4SystemCompiler(f, stage_idx=0, clean=cleanBuildRequested, jobs=jobs, extra_configure_args=extra_configure_args, env=merged_env) # Then build the compiler we would use for the bootstrap. _addSteps4StagedCompiler(f, stage_idx=1, jobs=jobs, extra_configure_args=extra_configure_args, env=merged_env) # Build all the remaining stages with exactly the same configuration. CmakeCommand.applyRequiredOptions(extra_configure_args, [ ('-DLLVM_ENABLE_LTO=', lto), ]) # If we build LLD, we would link with LLD. # Otherwise we link with the system linker. if 'lld' in depends_on_projects: CmakeCommand.applyRequiredOptions(extra_configure_args, [ ('-DLLVM_ENABLE_LLD=', 'ON'), ]) # The rest are test stages, which depend on the staged compiler we are ultimately after. s = f.staged_compiler_idx + 1 staged_install = f.stage_installdirs[f.staged_compiler_idx] for i in range(s, len(f.stage_objdirs[s:]) + s): configure_args = extra_configure_args[:] + extra_configure_args_lto_stage[:] configure_args.append( WithProperties("-DCMAKE_AR=%(builddir)s/" + staged_install + "/bin/llvm-ar")) configure_args.append( WithProperties("-DCMAKE_RANLIB=%(builddir)s/" + staged_install + "/bin/llvm-ranlib")) _addSteps4StagedCompiler(f, stage_idx=i, use_stage_idx=f.staged_compiler_idx, jobs=jobs, extra_configure_args=configure_args, env=merged_env) if compare_last_2_stages: # Compare the compilers built on the last 2 stages if requested. diff_command = [ "diff", "-q", f.stage_installdirs[-2] + "/bin/clang", f.stage_installdirs[-1] + "/bin/clang", ] f.addStep( ShellCommand(name="compare-compilers", description=[ "compare", "stage%d" % (len(f.stage_installdirs) - 1), "and", "stage%d" % len(f.stage_installdirs), "compilers", ], haltOnFailure=False, command=WithProperties(" ".join(diff_command)), workdir=".", env=merged_env)) # Only if the compare-compilers step has failed. def _prevStepFailed(step): steps = step.build.executedSteps prev_step = steps[-2] return (prev_step.results == FAILURE) dir1 = f.stage_objdirs[-2] dir2 = f.stage_objdirs[-1] inc_pattern = "-type f -not -name *.inc -printf '%f\n'" find_cmd = "find %s %s" % (dir1, dir2) diff_cmd = "diff -ru %s %s -x '*.tmp*' -X -" % (dir1, dir2) # Note: Use a string here as we want the command executed by a shell. diff_tablegen_inc_files_command = "%s %s | %s" % ( find_cmd, inc_pattern, diff_cmd) f.addStep( ShellCommand( name="compare-tablegen-inc-files", description=[ "compare", "stage%d" % (len(f.stage_installdirs) - 1), "and", "stage%d" % len(f.stage_installdirs), "Tablegen inc files", ], command=diff_tablegen_inc_files_command, workdir=".", env=merged_env, doStepIf=_prevStepFailed, )) return f
def getABITestsuitBuildFactory( clean=True, depends_on_projects=None, extra_configure_args=None, # Extra CMake args for all stages. jobs=None, # Restrict a degree of parallelism if needed. env=None, # Environmental variables for all steps. **kwargs): # Prepare environmental variables. Set here all env we want for all steps. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) if depends_on_projects: depends_on_projects = list(depends_on_projects) else: depends_on_projects = [ 'llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld' ] if extra_configure_args is None: cmake_args = list() else: cmake_args = list(extra_configure_args) # Some options are required for this build no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ]) cleanBuildRequested = lambda step: step.build.getProperty( "clean", default=step.build.getProperty("clean_obj")) or clean f = UnifiedTreeBuilder.getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects=depends_on_projects, llvm_srcdir="llvm", obj_dir="build", cleanBuildRequested=cleanBuildRequested, env=merged_env, **kwargs) # Pass through all the extra arguments. # First of all, we shall checkout the latest test-suite. f.addGetSourcecodeForProject(project='test-suite', src_dir='test-suite', alwaysUseLatest=True, **kwargs) # Then get the LLVM source code revision this particular build is for. f.addGetSourcecodeSteps(**kwargs) UnifiedTreeBuilder.addCmakeSteps(f, cleanBuildRequested=cleanBuildRequested, obj_dir=f.obj_dir, extra_configure_args=cmake_args, env=env, **kwargs) f.addStep( NinjaCommand( name="build-unified-tree", haltOnFailure=True, description=["Build", "unified", "tree"], env=merged_env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) # Run the ABI test. abi_test_env = { 'PYTHONPATH': WithProperties("%(workdir)s/" + f.llvm_srcdir + "/utils/lit:${PYTHONPATH}"), 'PATH': WithProperties("%(workdir)s/" + f.obj_dir + "/bin:${PATH}"), } merged_env.update(abi_test_env) abi_test_cmd = ["python", "linux-x86.py", "clang", "test", "-v"] if jobs: abi_test_cmd.append("-j" + str(jobs)) f.addStep( LitTestCommand(name='abi-test-suite', command=abi_test_cmd, description=["running", "ABI", "test-suite"], descriptionDone=["ABI", "test-suite", "completed"], workdir='test-suite/ABI-Testsuite', env=merged_env)) return f