def getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects = None, llvm_srcdir = None, obj_dir = None, install_dir = None, cleanBuildRequested = None, env = None, **kwargs): def cleanBuildRequestedByProperty(step): return step.build.getProperty("clean") if cleanBuildRequested is None: # We want a clean checkout only if requested by the property. cleanBuildRequested = cleanBuildRequestedByProperty f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, cleanBuildRequested=cleanBuildRequested, **kwargs) # Pass through all the extra arguments. # Remove the source code for a clean checkout if requested by property. # TODO: Some Windows slaves do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep(RemoveDirectory(name='clean-src-dir', dir=f.monorepo_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequestedByProperty, )) return f
def getAnnotatedBuildFactory(script, clean=False, depends_on_projects=None, env=None, timeout=1200): """ Returns a new build factory that uses AnnotatedCommand, which allows the build to be run by version-controlled scripts that do not require a buildmaster restart to update. """ f = LLVMBuildFactory(depends_on_projects=depends_on_projects, llvm_srcdir='llvm.src') if clean: f.addStep(SetProperty(property='clean', command='echo 1')) # We normally use the clean property to indicate that we want a # clean build, but AnnotatedCommand uses the clobber property # instead. Therefore, set clobber if clean is set to a truthy # value. This will cause AnnotatedCommand to set # BUILDBOT_CLOBBER=1 in the environment, which is how we # communicate to the script that we need a clean build. f.addStep( SetProperty( property='clobber', command='echo 1', doStepIf=lambda step: step.build.getProperty('clean', False))) merged_env = { 'TERM': 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set # anything. merged_env.update(env) scripts_dir = "annotated" f.addStep( SVN(name='update-annotate-scripts', mode='update', svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/' 'zorg/buildbot/builders/annotated', workdir=scripts_dir, alwaysUseLatest=True)) # Explicitly use '/' as separator, because it works on *nix and Windows. script_path = "../%s/%s" % (scripts_dir, script) f.addStep( AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, command=WithProperties( "python %(script)s --jobs=%(jobs:-)s", script=lambda _: script_path), env=merged_env)) return f
def getAnnotatedBuildFactory( script, clean=False, depends_on_projects=None, env=None, timeout=1200): """ Returns a new build factory that uses AnnotatedCommand, which allows the build to be run by version-controlled scripts that do not require a buildmaster restart to update. """ f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir='llvm.src') if clean: f.addStep(SetProperty(property='clean', command='echo 1')) # We normally use the clean property to indicate that we want a # clean build, but AnnotatedCommand uses the clobber property # instead. Therefore, set clobber if clean is set to a truthy # value. This will cause AnnotatedCommand to set # BUILDBOT_CLOBBER=1 in the environment, which is how we # communicate to the script that we need a clean build. f.addStep(SetProperty( property='clobber', command='echo 1', doStepIf=lambda step: step.build.getProperty('clean', False))) merged_env = { 'TERM': 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set # anything. merged_env.update(env) scripts_dir = "annotated" f.addStep(SVN(name='update-annotate-scripts', mode='update', svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/' 'zorg/buildbot/builders/annotated', workdir=scripts_dir, alwaysUseLatest=True)) # Explicitly use '/' as separator, because it works on *nix and Windows. script_path = "../%s/%s" % (scripts_dir, script) f.addStep(AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, command=WithProperties( "python %(script)s --jobs=%(jobs:-)s", script=lambda _: script_path), env=merged_env)) return f
def getSanitizerBuildFactory(clean=False, depends_on_projects=None, env=None, timeout=1200, is_legacy_mode=False): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { "TERM": "dumb", # Make sure Clang doesn't use color escape sequences. } # Clobber bot if we need a clean build. if clean: merged_env["BUILDBOT_CLOBBER"] = "1" # Overwrite pre-set items with the given ones, so user can set anything. if env is not None: merged_env.update(env) if depends_on_projects is None: depends_on_projects = [ "llvm", "clang", "compiler-rt", "libcxx", "libcxxabi", "libunwind", "lld" ] # Explicitly use '/' as separator, because it works on *nix and Windows. sanitizer_script_dir = "sanitizer_buildbot/sanitizers" sanitizer_script = "../%s/zorg/buildbot/builders/sanitizers/%s" % ( sanitizer_script_dir, "buildbot_selector.py") f = LLVMBuildFactory(is_legacy_mode=is_legacy_mode, clean=clean, depends_on_projects=depends_on_projects, llvm_srcdir=sanitizer_script_dir) # Determine the build directory. f.addStep( SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".", env=merged_env)) # Get sanitizer buildbot scripts. f.addGetSourcecodeForProject(name='update-annotate-scripts', project='zorg', src_dir=sanitizer_script_dir, alwaysUseLatest=True) # Run annotated command for sanitizer. f.addStep( AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, command="python " + sanitizer_script, env=merged_env)) return f
def getLLVMBuildFactoryAndSVNSteps( depends_on_projects = None, llvm_srcdir = None, obj_dir = None, install_dir = None, cleanBuildRequested = None, env = None, **kwargs): def cleanBuildRequestedByProperty(step): return step.build.getProperty("clean") # Set defaults if not depends_on_projects: depends_on_projects=['llvm', 'clang'] if cleanBuildRequested is None: # We want a clean checkout only if requested by the property. cleanBuildRequested = cleanBuildRequestedByProperty f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir or "llvm", obj_dir=obj_dir or "build", install_dir=install_dir, cleanBuildRequested=cleanBuildRequested, **kwargs) # Pass through all the extra arguments. # Do a clean checkout if requested by a build property. # TODO: Some Windows slaves do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep(RemoveDirectory(name='clean-src-dir', dir=f.llvm_srcdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequestedByProperty, )) # Get the source code. f.addSVNSteps() return f
def getLLVMBuildFactoryAndSVNSteps(depends_on_projects=None, llvm_srcdir=None, obj_dir=None, install_dir=None, cleanBuildRequested=None, env=None, **kwargs): def cleanBuildRequestedByProperty(step): return step.build.getProperty("clean") # Set defaults if not depends_on_projects: depends_on_projects = ['llvm', 'clang'] if cleanBuildRequested is None: # We want a clean checkout only if requested by the property. cleanBuildRequested = cleanBuildRequestedByProperty f = LLVMBuildFactory(depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir or "llvm", obj_dir=obj_dir or "build", install_dir=install_dir, cleanBuildRequested=cleanBuildRequested, **kwargs) # Pass through all the extra arguments. # Do a clean checkout if requested by a build property. # TODO: Some Windows slaves do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep( RemoveDirectory( name='clean-src-dir', dir=f.llvm_srcdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequestedByProperty, )) # Get the source code. f.addSVNSteps() return f
def getCmakeWithMSVCBuildFactory( clean = True, # False for incremental builds. depends_on_projects = None, # List of projects to listen. cmake_cache = None, # Path to a cmake cache file. extra_configure_args = None, # Extra CMake args if any. llvm_srcdir = None, # Source code root directory. obj_dir = None, # Build tree root directory. install_dir = None, # Directory to install the results to. checks = None, # List of checks to test the build. checks_on_target = None, # [(<name>,[<command tokens>])] array of # check name and command to run on target. jobs = None, # Restrict a degree of parallelism. env = None, # Environmental variables for all steps. # VS tools environment variable if using MSVC. # For example, "autodetect" to auto detect, %VS140COMNTOOLS% to select # the VS 2015 toolchain, or empty string if environment is already set. vs=None, **kwargs): if vs is None: # We autodetect Visual Studio, unless otherwise is requested. vs = "autodetect" if install_dir is None: install_dir = 'install' # Prepare environmental variables. Set here all env we want for all steps. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: assert not vs, "Cannot have custom builder env vars with VS setup." # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() if depends_on_projects is None: depends_on_projects = [ 'llvm', 'compiler-rt', 'clang', 'clang-tools-extra', 'libunwind', 'libcxx', 'libcxxabi', 'lld', ] if checks is None: # Check only host-side tools. Target-side tests should run on a target. checks=[ "check-llvm", "check-clang", "check-lld", ] source_remove_requested = lambda step: step.build.getProperty("clean") clean_build_requested = lambda step: \ clean or \ step.build.getProperty("clean", \ default=step.build.getProperty("clean_obj") \ ) f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, cleanBuildRequested=clean_build_requested, **kwargs) # Pass through all the extra arguments. # Remove the source code tree if requested. # NOTE: Somehow RemoveDirectory buildbot command often fails on Windows, # as somthing keeps a lock. We use rm command instead realying on a shell # to support that. f.addStep(ShellCommand(name='clean-%s-dir' % f.monorepo_dir, command=['rm','-rf', f.monorepo_dir], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='Remove the source code', workdir='.', env=merged_env, doStepIf=source_remove_requested)) # Get the source code. f.addGetSourcecodeSteps(**kwargs) # Remove the build directory if requested. f.addStep(ShellCommand(name='clean-%s-dir' % f.obj_dir, command=['rm','-rf', f.obj_dir], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='Remove build directory', workdir='.', env=merged_env, doStepIf=clean_build_requested)) if vs: # Configure MSVC environment if requested. f.addStep(SetProperty( command=builders_util.getVisualStudioEnvironment(vs, None), extract_fn=builders_util.extractSlaveEnvironment)) merged_env = Property('slave_env') # Since this is a build of a cross toolchain, we build only the host-side # tools first by the host system compiler. Libraries will be cross-compiled. cmake_args.append(InterpolateToPosixPath( '-DLLVM_AR=%(workdir)s/' + f.obj_dir + '/bin/llvm-ar.exe')), CmakeCommand.applyDefaultOptions(cmake_args, [ ('-G', 'Ninja'), ('-DLLVM_ENABLE_PROJECTS=', 'llvm;clang;clang-tools-extra;lld'), ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCMAKE_CXX_FLAGS=', '-D__OPTIMIZE__'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_LIT_ARGS=', '-v -vv'), ]) if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeTo( install_dir, f.obj_dir) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel), ]) # Remove the build directory if requested. f.addStep(ShellCommand(name='clean-%s-dir' % install_dir, command=['rm','-rf', install_dir], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='Remove install directory', workdir='.', env=merged_env, doStepIf=clean_build_requested)) src_dir_rel = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, f.obj_dir) # Add given cmake cache at the very end. if cmake_cache: cmake_args.append('-C%s' % cmake_cache) f.addStep(CmakeCommand(name="cmake-configure", haltOnFailure=True, description=["Cmake", "configure"], options=cmake_args, path=src_dir_rel, workdir=f.obj_dir, env=merged_env, **kwargs # Pass through all the extra arguments. )) f.addStep(NinjaCommand(name="build-%s" % f.monorepo_dir, haltOnFailure=True, description=["Build", f.monorepo_dir], workdir=f.obj_dir, env=merged_env, **kwargs # Pass through all the extra arguments. )) # Test the components if requested one check at a time. for check in checks: f.addStep( LitTestCommand( haltOnFailure=False, # We want to test as much as we could. name='test-%s' % check, command=["ninja", WithProperties(check)], description=[ "Testing", "just", "built", "components", "for", check], descriptionDone=[ "Test", "just", "built", "components", "for", check, "completed"], env=merged_env, **kwargs # Pass through all the extra arguments. )) # Run commands on a target if requested. if checks_on_target: for check, cmd in checks_on_target: f.addStep( LitTestCommand( haltOnFailure=False, # We want to test as much as we could. name='test-%s' % check, command=cmd, description=[ "Testing", "just", "built", "components", "for", check], descriptionDone=[ "Test", "just", "built", "components", "for", check, "completed"], env=merged_env, **kwargs # Pass through all the extra arguments. )) # Install just built components if install_dir: f.addStep(NinjaCommand(name="install-all", haltOnFailure=True, targets=["install"], description=[ "Install", "just", "built", "components"], workdir=f.obj_dir, env=merged_env, **kwargs # Pass through all the extra arguments. )) return f
def getCmakeBuildFactory(depends_on_projects=None, llvm_srcdir=None, obj_dir=None, install_dir=None, clean=False, extra_configure_args=None, env=None, **kwargs): # Set defaults if not depends_on_projects: depends_on_projects = ['llvm', 'clang'] if extra_configure_args is None: extra_configure_args = [] # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) if not obj_dir: obj_dir = "build" if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeToBuild( install_dir, obj_dir) CmakeCommand.applyRequiredOptions(extra_configure_args, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel), ]) cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir or "llvm.src", obj_dir=obj_dir, install_dir=install_dir, cleanBuildRequested=cleanBuildRequested, **kwargs # Pass through all the extra arguments. ) # Directories to use on this stage. src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) # Do a clean checkout if requested. f.addStep( RemoveDirectory( name='clean-src-dir', dir=f.llvm_srcdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Get the source code. f.addSVNSteps() # This is an incremental build, unless otherwise has been requested. # Remove obj and install dirs for a clean build. f.addStep( RemoveDirectory( name='clean-%s-dir' % f.obj_dir, dir=f.obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) if f.install_dir: f.addStep( RemoveDirectory( name='clean-%s-dir' % f.install_dir, dir=f.install_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Reconcile the cmake options for this build. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ]) # Create configuration files with cmake, unless this has been already done # for an incremental build. f.addStep( CmakeCommand( name="cmake-configure", description=["cmake configure"], haltOnFailure=True, options=cmake_args, path=src_dir, env=env, workdir=obj_dir, doStepIf=FileDoesNotExist("CMakeCache.txt"), **kwargs # Pass through all the extra arguments. )) return f
def getCmakeWithNinjaMultistageBuildFactory( depends_on_projects = None, llvm_srcdir = None, obj_dir = None, checks = None, install_dir = None, clean = False, extra_configure_args = None, env = None, stages=2, stage_names=None, **kwargs): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() assert stages > 1, "It should be at least 2 stages in a multistage build." if stage_names is None: stage_names = list() for i in range(1, stages + 1): stage_names.append("stage%s" % i) else: assert len(stage_names) == stages, "Please specify names for none or all of the requested stages." if obj_dir is None: obj_dir = "build" if install_dir is None: install_dir = "install" if checks is None: checks = ['check-all'] stage_objdirs = list() stage_installdirs = list() for s in stage_names: stage_objdirs.append("%s/%s" % (obj_dir, s)) stage_installdirs.append("%s/%s" % (install_dir, s)) f = getLLVMBuildFactoryAndSVNSteps( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, env=env, stage_objdirs=stage_objdirs, stage_installdirs=stage_installdirs, stage_names=stage_names, **kwargs) # Pass through all the extra arguments. # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ]) if 'clang' in depends_on_projects: CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ]) # Some options are required for this build no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ]) # The stage 1 is special, though. We use the system compiler and # do incremental build, unless a clean one has been requested. cmake_args_stage1 = cmake_args[:] CmakeCommand.applyDefaultOptions(cmake_args_stage1, [ # Do not expect warning free build by the system toolchain. ('-DLLVM_ENABLE_WERROR=', 'OFF'), ]) cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean addCmakeSteps( f, cleanBuildRequested=cleanBuildRequested, obj_dir=stage_objdirs[0], install_dir=stage_installdirs[0], extra_configure_args=cmake_args_stage1, env=env, stage_name=stage_names[0], **kwargs) addNinjaSteps( f, obj_dir=stage_objdirs[0], checks=checks, install_dir=stage_installdirs[0], env=env, stage_name=stage_names[0], **kwargs) # Build the rest stage by stage, using just built compiler to compile # the next stage. CmakeCommand.applyDefaultOptions(cmake_args, [ # We should be warnings free when use just built compiler. ('-DLLVM_ENABLE_WERROR=', 'ON'), ]) # If we build LLD, we would link with LLD. # Otherwise we link with a system linker. if 'lld' in f.depends_on_projects: CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DLLVM_ENABLE_LLD=', 'ON'), ]) for stage_idx in range(1, stages): # Directories to use in this stage. obj_dir = f.stage_objdirs[stage_idx] src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir) staged_install = f.stage_installdirs[stage_idx - 1] # Configure the compiler to use in this stage. cmake_args_stageN = cmake_args[:] CmakeCommand.applyRequiredOptions(cmake_args_stageN, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) cmake_args_stageN.append( WithProperties( "-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++" )) cmake_args_stageN.append( WithProperties( "-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang" )) addCmakeSteps( f, True, # We always do a clean build for the staged builds. obj_dir=stage_objdirs[stage_idx], install_dir=stage_installdirs[stage_idx], extra_configure_args=cmake_args_stageN, env=env, stage_name=stage_names[stage_idx], **kwargs) addNinjaSteps( f, obj_dir=stage_objdirs[stage_idx], checks=checks, install_dir=stage_installdirs[stage_idx], env=env, stage_name=stage_names[stage_idx], **kwargs) return f
def _getClangCMakeBuildFactory( clean=True, test=True, cmake='cmake', jobs=None, # VS tools environment variable if using MSVC. For example, # %VS120COMNTOOLS% selects the 2013 toolchain. vs=None, vs_target_arch='x86', # Multi-stage compilation useTwoStage=False, testStage1=True, stage1_config='Release', stage2_config='Release', # Test-suite runTestSuite=False, nt_flags=[], testsuite_flags=[], submitURL=None, testerName=None, # Environmental variables for all steps. env={}, extra_cmake_args=[], # Extra repositories checkout_clang_tools_extra=True, checkout_compiler_rt=True, checkout_lld=True, checkout_libcxx=False, checkout_test_suite=False, # Upload artifacts to Google Cloud Storage (for the llvmbisect tool) stage1_upload_directory=None, # Use a lower compression level to generate the build-cache package faster # default is 6 according to documentation xz_compression_factor=6, use_pixz_compression=False, # Triggers trigger_after_stage1=[]): ############# PREPARING clean_build_requested = lambda step: \ step.build.getProperty( \ "clean", \ default=step.build.getProperty("clean_obj") \ ) or clean # We *must* checkout at least Clang+LLVM depends_on_projects = ['llvm', 'clang'] if checkout_clang_tools_extra: depends_on_projects.append('clang-tools-extra') if checkout_compiler_rt: depends_on_projects.append('compiler-rt') if checkout_lld: depends_on_projects.append('lld') if runTestSuite or checkout_test_suite: depends_on_projects.append('lnt') depends_on_projects.append('test-suite') if checkout_libcxx: depends_on_projects.append('libcxx') depends_on_projects.append('libcxxabi') depends_on_projects.append('libunwind') f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir='llvm') f.addSVNSteps() # If jobs not defined, Ninja will choose a suitable value jobs_cmd = [] lit_args = "'-v" if jobs is not None: jobs_cmd = ["-j"+str(jobs)] lit_args += " -j"+str(jobs)+"'" else: lit_args += "'" ninja_cmd = ['ninja'] + jobs_cmd ninja_install_cmd = ['ninja', 'install'] + jobs_cmd ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd # Global configurations stage1_build = 'stage1' stage1_install = 'stage1.install' stage2_build = 'stage2' stage2_install = 'stage2.install' # Set up VS environment, if appropriate. if vs: f.addStep(SetProperty( command=builders_util.getVisualStudioEnvironment(vs, vs_target_arch), extract_fn=builders_util.extractSlaveEnvironment)) assert not env, "Can't have custom builder env vars with VS" env = Property('slave_env') ############# CLEANING f.addStep(ShellCommand(name='clean stage 1', command=['rm','-rf',stage1_build], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='cleaning stage 1', descriptionDone='clean', workdir='.', doStepIf=clean_build_requested)) ############# STAGE 1 f.addStep(ShellCommand(name='cmake stage 1', command=[cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE="+stage1_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS="+lit_args, "-DCMAKE_INSTALL_PREFIX=../"+stage1_install] + extra_cmake_args, haltOnFailure=True, description='cmake stage 1', workdir=stage1_build, doStepIf=FileDoesNotExist("build.ninja"), env=env)) f.addStep(WarningCountingShellCommand(name='build stage 1', command=ninja_cmd, haltOnFailure=True, description='ninja all', workdir=stage1_build, env=env)) if test and testStage1: haltOnStage1Check = not useTwoStage and not runTestSuite f.addStep(lit_test_command.LitTestCommand(name='ninja check 1', command=ninja_check_cmd, haltOnFailure=haltOnStage1Check, description=["checking stage 1"], descriptionDone=["stage 1 checked"], workdir=stage1_build, env=env)) if useTwoStage or runTestSuite or stage1_upload_directory: f.addStep(ShellCommand(name='clean stage 1 install', command=['rm','-rf',stage1_install], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='cleaning stage 1 install', descriptionDone='clean', workdir='.')) f.addStep(ShellCommand(name='install stage 1', command=ninja_install_cmd, description='ninja install', workdir=stage1_build, env=env)) if stage1_upload_directory: addGCSUploadSteps(f, 'stage 1', stage1_install, stage1_upload_directory, env, gcs_url_property='stage1_package_gcs_url', use_pixz_compression=use_pixz_compression, xz_compression_factor=xz_compression_factor) # Compute the cmake define flag to set the C and C++ compiler to clang. Use # clang-cl if we used MSVC for stage1. if not vs: cc = 'clang' cxx = 'clang++' else: cc = 'clang-cl.exe' cxx = 'clang-cl.exe' ############# STAGE 2 if useTwoStage: # We always cleanly build the stage 2. If the compiler has been # changed on the stage 1, we cannot trust any of the intermediate file # from the old compiler. And if the stage 1 compiler is the same, we # should not build in the first place. f.addStep(ShellCommand(name='clean stage 2', command=['rm','-rf',stage2_build], warnOnFailure=True, description='cleaning stage 2', descriptionDone='clean', workdir='.')) # Set the compiler using the CC and CXX environment variables to work around # backslash string escaping bugs somewhere between buildbot and cmake. The # env.exe helper is required to run the tests, so hopefully it's already on # PATH. cmake_cmd2 = ['env', WithProperties('CC=%(workdir)s/'+stage1_install+'/bin/'+cc), WithProperties('CXX=%(workdir)s/'+stage1_install+'/bin/'+cxx), cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE="+stage2_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS="+lit_args, "-DCMAKE_INSTALL_PREFIX=../"+stage2_install] + extra_cmake_args f.addStep(ShellCommand(name='cmake stage 2', command=cmake_cmd2, haltOnFailure=True, description='cmake stage 2', workdir=stage2_build, env=env)) f.addStep(WarningCountingShellCommand(name='build stage 2', command=ninja_cmd, haltOnFailure=True, description='ninja all', workdir=stage2_build, env=env)) if test: f.addStep(lit_test_command.LitTestCommand(name='ninja check 2', command=ninja_check_cmd, haltOnFailure=not runTestSuite, description=["checking stage 2"], descriptionDone=["stage 2 checked"], workdir=stage2_build, env=env)) ############# TEST SUITE ## Test-Suite (stage 2 if built, stage 1 otherwise) if runTestSuite: compiler_path = stage1_install if useTwoStage: compiler_path=stage2_install f.addStep(ShellCommand(name='clean stage 2 install', command=['rm','-rf',stage2_install], warnOnFailure=True, description='cleaning stage 2 install', descriptionDone='clean', workdir='.')) f.addStep(ShellCommand(name='install stage 2', command=ninja_install_cmd, description='ninja install 2', workdir=stage2_build, env=env)) # Get generated python, lnt python = WithProperties('%(workdir)s/test/sandbox/bin/python') lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt') lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py') # Paths sandbox = WithProperties('%(workdir)s/test/sandbox') test_suite_dir = WithProperties('%(workdir)s/test/test-suite') # Get latest built Clang (stage1 or stage2) cc = WithProperties('%(workdir)s/'+compiler_path+'/bin/'+cc) cxx = WithProperties('%(workdir)s/'+compiler_path+'/bin/'+cxx) # LNT Command line (don't pass -jN. Users need to pass both --threads # and --build-threads in nt_flags/test_suite_flags to get the same effect) use_runtest_testsuite = len(nt_flags) == 0 if not use_runtest_testsuite: test_suite_cmd = [python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx] # Append any option provided by the user test_suite_cmd.extend(nt_flags) else: lit = WithProperties('%(workdir)s/'+stage1_build+'/bin/llvm-lit') test_suite_cmd = [python, lnt, 'runtest', 'test-suite', '--no-timestamp', '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx, '--use-lit', lit] # Append any option provided by the user test_suite_cmd.extend(testsuite_flags) # Only submit if a URL has been specified if submitURL is not None: if not isinstance(submitURL, list): submitURL = [submitURL] for url in submitURL: test_suite_cmd.extend(['--submit', url]) # lnt runtest test-suite doesn't understand --no-machdep-info: if testerName and not use_runtest_testsuite: test_suite_cmd.extend(['--no-machdep-info', testerName]) # CC and CXX are needed as env for build-tools test_suite_env = copy.deepcopy(env) test_suite_env['CC'] = cc test_suite_env['CXX'] = cxx # Steps to prepare, build and run LNT f.addStep(ShellCommand(name='clean sandbox', command=['rm', '-rf', 'sandbox'], haltOnFailure=True, description='removing sandbox directory', workdir='test', env=env)) f.addStep(ShellCommand(name='recreate sandbox', command=['virtualenv', 'sandbox'], haltOnFailure=True, description='recreating sandbox', workdir='test', env=env)) f.addStep(ShellCommand(name='setup lit', command=[python, lnt_setup, 'develop'], haltOnFailure=True, description='setting up LNT in sandbox', workdir='test/sandbox', env=env)) f.addStep(commands.LitTestCommand.LitTestCommand( name='test-suite', command=test_suite_cmd, haltOnFailure=True, description=['running the test suite'], workdir='test/sandbox', logfiles={'configure.log' : 'build/configure.log', 'build-tools.log' : 'build/build-tools.log', 'test.log' : 'build/test.log', 'report.json' : 'build/report.json'}, env=test_suite_env)) return f
def getLLDBCMakeBuildFactory( clean=False, jobs="%(jobs)s", # Source directory containing a built python python_source_dir=None, # Default values for VS devenv and build configuration vs=None, config='Release', target_arch='x86', extra_cmake_args=None, test=False, testTimeout=2400, install=False): ############# PREPARING # Global configurations build_dir = 'build' f = LLVMBuildFactory(is_legacy_mode=False, depends_on_projects=["llvm", "clang", "lldb", "lld"], obj_dir=build_dir) # Determine Slave Environment and Set MSVC environment. if vs: f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f.addGetSourcecodeSteps() build_cmd = ['ninja'] install_cmd = ['ninja', 'install'] test_cmd = ['ninja', 'check-lldb'] if jobs: build_cmd.append(WithProperties("-j%s" % jobs)) install_cmd.append(WithProperties("-j%s" % jobs)) test_cmd.append(WithProperties("-j%s" % jobs)) ############# CLEANING cleanBuildRequested = lambda step: clean or step.build.getProperty( "clean", default=step.build.getProperty("clean_obj")) f.addStep( RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, f.obj_dir) cmake_options = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=" + config, "-DCMAKE_INSTALL_PREFIX=../install", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), ] if python_source_dir: cmake_options.append("-DPYTHON_HOME=" + python_source_dir) if extra_cmake_args: cmake_options += extra_cmake_args f.addStep( CmakeCommand(name="cmake-configure", description=["cmake configure"], haltOnFailure=True, options=cmake_options, path=rel_src_dir, env=Property('slave_env'), workdir=build_dir)) f.addStep( WarningCountingShellCommand(name='build', command=build_cmd, haltOnFailure=True, description='ninja build', workdir=build_dir, env=Property('slave_env'))) ignoreInstallFail = bool(install != 'ignoreFail') f.addStep( ShellCommand(name='install', command=install_cmd, flunkOnFailure=ignoreInstallFail, description='ninja install', workdir=build_dir, doStepIf=bool(install), env=Property('slave_env'))) ignoreTestFail = bool(test != 'ignoreFail') f.addStep( ShellCommand(name='test', command=test_cmd, flunkOnFailure=ignoreTestFail, timeout=testTimeout, description='ninja test', workdir=build_dir, doStepIf=bool(test), env=Property('slave_env'))) return f
def addCmakeSteps( f, cleanBuildRequested, obj_dir, install_dir = None, extra_configure_args = None, env = None, stage_name = None, **kwargs): # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # This is an incremental build, unless otherwise has been requested. # Remove obj and install dirs for a clean build. # TODO: Some Windows slaves do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeToBuild( install_dir, obj_dir) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel), ]) f.addStep(RemoveDirectory(name='clean-%s-dir' % install_dir, dir=install_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Reconcile the cmake options for this build. # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ('-DLLVM_LIT_ARGS=', '"-v"'), ]) # Create configuration files with cmake, unless this has been already done # for an incremental build. if stage_name: step_name = "cmake-configure-%s" % stage_name else: stage_name = "" step_name = "cmake-configure" src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) f.addStep(CmakeCommand(name=step_name, description=["Cmake", "configure", stage_name], options=cmake_args, path=src_dir, haltOnFailure=kwargs.get('haltOnFailure', True), env=env, workdir=obj_dir, **kwargs # Pass through all the extra arguments. ))
def getCmakeWithNinjaMultistageBuildFactory(depends_on_projects=None, llvm_srcdir=None, obj_dir=None, checks=None, install_dir=None, clean=False, extra_configure_args=None, env=None, stages=2, stage_names=None, **kwargs): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() assert stages > 1, "It should be at least 2 stages in a multistage build." if stage_names is None: stage_names = list() for i in range(1, stages + 1): stage_names.append("stage%s" % i) else: assert len( stage_names ) == stages, "Please specify names for none or all of the requested stages." if obj_dir is None: obj_dir = "build" if install_dir is None: install_dir = "install" if checks is None: checks = ['check-all'] stage_objdirs = list() stage_installdirs = list() for s in stage_names: stage_objdirs.append("%s/%s" % (obj_dir, s)) stage_installdirs.append("%s/%s" % (install_dir, s)) f = getLLVMBuildFactoryAndSVNSteps( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, env=env, stage_objdirs=stage_objdirs, stage_installdirs=stage_installdirs, stage_names=stage_names, **kwargs) # Pass through all the extra arguments. # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ]) # Some options are required for this build no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ]) # The stage 1 is special, though. We use the system compiler and # do incremental build, unless a clean one has been requested. cmake_args_stage1 = cmake_args[:] CmakeCommand.applyDefaultOptions( cmake_args_stage1, [ # Do not expect warning free build by the system toolchain. ('-DLLVM_ENABLE_WERROR=', 'OFF'), ]) cleanBuildRequested = lambda step: step.build.getProperty( "clean", default=step.build.getProperty("clean_obj")) or clean addCmakeSteps(f, cleanBuildRequested=cleanBuildRequested, obj_dir=stage_objdirs[0], install_dir=stage_installdirs[0], extra_configure_args=cmake_args_stage1, env=env, stage_name=stage_names[0], **kwargs) addNinjaSteps(f, obj_dir=stage_objdirs[0], checks=checks, install_dir=stage_installdirs[0], env=env, stage_name=stage_names[0], **kwargs) # Build the rest stage by stage, using just built compiler to compile # the next stage. CmakeCommand.applyDefaultOptions( cmake_args, [ # We should be warnings free when use just built compiler. ('-DLLVM_ENABLE_WERROR=', 'ON'), ]) # If we build LLD, we would link with LLD. # Otherwise we link with a system linker. if 'lld' in f.depends_on_projects: CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DLLVM_ENABLE_LLD=', 'ON'), ]) for stage_idx in range(1, stages): # Directories to use in this stage. obj_dir = f.stage_objdirs[stage_idx] src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) install_dir = LLVMBuildFactory.pathRelativeToBuild( f.stage_installdirs[stage_idx], obj_dir) staged_install = f.stage_installdirs[stage_idx - 1] # Configure the compiler to use in this stage. cmake_args_stageN = cmake_args[:] CmakeCommand.applyRequiredOptions(cmake_args_stageN, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) cmake_args_stageN.append( WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++")) cmake_args_stageN.append( WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang")) addCmakeSteps( f, True, # We always do a clean build for the staged builds. obj_dir=stage_objdirs[stage_idx], install_dir=stage_installdirs[stage_idx], extra_configure_args=cmake_args_stageN, env=env, stage_name=stage_names[stage_idx], **kwargs) addNinjaSteps(f, obj_dir=stage_objdirs[stage_idx], checks=checks, install_dir=stage_installdirs[stage_idx], env=env, stage_name=stage_names[stage_idx], **kwargs) return f
def getToolchainBuildFactory( clean=False, test=True, env=None, # Environmental variables for all steps. extra_configure_args=None): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { "TERM" : "dumb", # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) src_dir = "llvm.src" obj_dir = "llvm.obj" install_dir = "llvm.install" f = LLVMBuildFactory( depends_on_projects=[ "llvm", "clang", "clang-tools-extra", "compiler-rt", "libcxx", "libcxxabi", "libunwind", "lld" ]) # Get Fuchsia SDK. sdk_dir = "fuchsia.sdk" sdk_platform = { "Linux": "linux-amd64", "Darwin": "mac-amd64", }[platform.system()] sdk_version = "latest" sdk_url = WithProperties( "https://chrome-infra-packages.appspot.com/dl/" "fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s", sdk_platform=lambda _: sdk_platform, sdk_version=lambda _: sdk_version) f.addStep(RemoveDirectory(name="clean-sdk", dir=sdk_dir, haltOnFailure=True)) f.addStep(ShellCommand(name="fetch-sdk", command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url], description=["download", "fuchsia sdk"], workdir=sdk_dir)) f.addStep(ShellCommand(name="extract-sdk", command=["unzip", "-fo", "sdk.cipd"], description=["extract", "fuchsia sdk"], workdir=sdk_dir)) cleanCheckoutRequested = lambda step: step.build.getProperty("clean", default=False) or clean # Clean up llvm sources. f.addStep(RemoveDirectory(name="clean-llvm.src", dir=src_dir, haltOnFailure=True, doStepIf=cleanCheckoutRequested)) # Get sources. for project in f.depends_on_projects: _, baseURL = svn_repos[project] f.addStep(SVN(name="svn-%s" % project, workdir=src_dir + "/" + project, baseURL=baseURL)) cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean # Clean up llvm build. f.addStep(RemoveDirectory(name="clean-llvm.obj", dir=obj_dir, haltOnFailure=True, doStepIf=cleanBuildRequested)) # Configure. if extra_configure_args is None: cmake_options = [] else: cmake_options = extra_configure_args[:] # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_options, [ ("-G", "Ninja"), ("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"), ("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"), ]) # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_options, [ ("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"), ("-DLLVM_ENABLE_LTO=", "OFF"), ]) cmake_options.append( WithProperties( "-DCMAKE_INSTALL_PREFIX=%(workdir)s/" + install_dir )) cmake_options.append( WithProperties( "-DFUCHSIA_SDK=%(workdir)s/" + sdk_dir )) CmakeCommand.applyRequiredOptions(cmake_options, [ ("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"), ]) f.addStep(CmakeCommand(name="cmake-configure", options=cmake_options, path='../' + src_dir + '/llvm', haltOnFailure=True, description=["configure"], workdir=obj_dir, env=merged_env, doStepIf=FileDoesNotExist("CMakeCache.txt"))) # Build distribution. f.addStep(NinjaCommand(name="ninja-build", targets=["stage2-distribution"], haltOnFailure=True, description=["build"], workdir=obj_dir, env=merged_env)) # Test llvm, clang and lld. f.addStep(NinjaCommand(name="check", targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")], haltOnFailure=True, description=["check"], workdir=obj_dir, env=merged_env, doStepIf=test)) # Install distribution. f.addStep(NinjaCommand(name="install", targets=["stage2-install-distribution"], haltOnFailure=True, description=["install"], workdir=obj_dir, env=merged_env)) return f
def getClangWithLTOBuildFactory( depends_on_projects = None, clean = False, jobs = None, extra_configure_args = None, compare_last_2_stages = True, lto = None, # The string gets passed to -flto flag as is. Like -flto=thin. env = None): # Set defaults if depends_on_projects: depends_on_projects = list(depends_on_projects) else: # By default we link with LLD. depends_on_projects = ['llvm', 'clang', 'lld'] if lto is None: lto = 'ON' if jobs is None: jobs = "%(jobs)s" if extra_configure_args is None: extra_configure_args = [] else: extra_configure_args = list(extra_configure_args) # Make sure CMAKE_INSTALL_PREFIX and -G are not specified # in the extra_configure_args. We set them internally as needed. # TODO: assert extra_configure_args. install_prefix_specified = any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) assert True, "Please do not explicitly specify the install prefix for multi-stage build." # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir="llvm.src", stage_objdirs=[ "build/stage1", "build/stage2", "build/stage3", "build/stage4", ], stage_installdirs=[ "install/stage1", "install/stage2", "install/stage3", "install/stage4", ], staged_compiler_idx = 1) cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean # Do a clean checkout if requested. f.addStep(RemoveDirectory(name='clean-src-dir', dir=f.llvm_srcdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Get the source code. f.addSVNSteps() # Build with the system compiler first _addSteps4SystemCompiler(f, stage_idx=0, clean=cleanBuildRequested, jobs=jobs, extra_configure_args=extra_configure_args, env=merged_env) # Then build the compiler we would use for the bootstrap. _addSteps4StagedCompiler(f, stage_idx=1, withLTOSupport=True, jobs=jobs, extra_configure_args=extra_configure_args, env=merged_env) # Build all the remaining stages with exactly the same configuration. CmakeCommand.applyRequiredOptions(extra_configure_args, [ ('-DLLVM_ENABLE_LTO=', lto), ]) # If we build LLD, we would link with LLD. # Otherwise we link with the system linker. if 'lld' in depends_on_projects: CmakeCommand.applyRequiredOptions(extra_configure_args, [ ('-DLLVM_ENABLE_LLD=', 'ON'), ]) # The rest are test stages, which depend on the staged compiler we are ultimately after. s = f.staged_compiler_idx + 1 staged_install = f.stage_installdirs[f.staged_compiler_idx] for i in range(s, len(f.stage_objdirs[s:]) + s): configure_args = extra_configure_args[:] configure_args.append( WithProperties( "-DCMAKE_AR=%(workdir)s/" + staged_install + "/bin/llvm-ar" )) configure_args.append( WithProperties( "-DCMAKE_RANLIB=%(workdir)s/" + staged_install + "/bin/llvm-ranlib" )) _addSteps4StagedCompiler(f, stage_idx=i, use_stage_idx=f.staged_compiler_idx, jobs=jobs, extra_configure_args=configure_args, env=merged_env) if compare_last_2_stages: # Compare the compilers built on the last 2 stages if requested. diff_command = [ "diff", "-q", f.stage_installdirs[-2] + "/bin/clang", f.stage_installdirs[-1] + "/bin/clang", ] f.addStep( ShellCommand( name="compare-compilers", description=[ "compare", "stage%d" % (len(f.stage_installdirs)-1), "and", "stage%d" % len(f.stage_installdirs), "compilers", ], haltOnFailure=True, command=WithProperties(" ".join(diff_command)), workdir=".", env=merged_env ) ) return f
def _addSteps4StagedCompiler(f, stage_idx=1, use_stage_idx=-1, jobs=None, extra_configure_args=None, env=None): if use_stage_idx < 0: use_stage_idx = stage_idx - 1 # Index is zero-based, so we want to use a human friendly number instead. stage_num = stage_idx + 1 # Directories to use on this stage. obj_dir = f.stage_objdirs[stage_idx] src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir) install_dir = LLVMBuildFactory.pathRelativeTo( f.stage_installdirs[stage_idx], obj_dir) staged_install = f.stage_installdirs[use_stage_idx] # Always do a clean build for the staged compiler. f.addStep( steps.RemoveDirectory( name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, )) f.addStep( steps.RemoveDirectory( name='clean-%s-dir' % f.stage_installdirs[stage_idx], dir=f.stage_installdirs[stage_idx], haltOnFailure=False, flunkOnFailure=False, )) # Reconcile the cmake options for this stage. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ]) # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) cmake_args.append( WithProperties("-DCMAKE_CXX_COMPILER=%(builddir)s/" + staged_install + "/bin/clang++")) cmake_args.append( WithProperties("-DCMAKE_C_COMPILER=%(builddir)s/" + staged_install + "/bin/clang")) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) # Create configuration files with cmake f.addStep( CmakeCommand( name="cmake-configure-stage%s" % stage_num, description=["stage%s cmake configure" % stage_num], haltOnFailure=True, options=cmake_args, path=src_dir, env=env, workdir=obj_dir, )) # Build clang by the staged compiler f.addStep( NinjaCommand( name="build-stage%s-compiler" % stage_num, jobs=jobs, haltOnFailure=True, description=["build stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, )) # Test just built compiler f.addStep( NinjaCommand( name="test-stage%s-compiler" % stage_num, targets=["check-all"], jobs=jobs, haltOnFailure=True, description=["test stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, )) # Install just built compiler f.addStep( NinjaCommand( name="install-stage%s-compiler" % stage_num, targets=["install"], jobs=jobs, haltOnFailure=True, description=["install stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, ))
def getLibcxxAndAbiBuilder(f=None, env=None, cmake_extra_opts=None, lit_extra_opts=None, lit_extra_args=None, check_libcxx_abilist=False, check_libcxx_benchmarks=None, depends_on_projects=None, use_cache=None, **kwargs): if env is None: env = {} if cmake_extra_opts is None: cmake_extra_opts = {} if lit_extra_opts is None: lit_extra_opts = {} if lit_extra_args is None: lit_extra_args = [] if depends_on_projects is None: depends_on_projects = ['libcxx','libcxxabi','libunwind'] src_root = 'llvm' build_path = 'build' if f is None: f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, llvm_srcdir=src_root, obj_dir=build_path, **kwargs) # Pass through all the extra arguments. rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, build_path) # Specify the max number of threads using properties so LIT doesn't use # all the threads on the system. litTestArgs = '-vv --show-unsupported --show-xfail --threads=%(jobs)s' if lit_extra_args: litTestArgs += ' ' + ' '.join(lit_extra_args) for key in lit_extra_opts: litTestArgs += (' --param=' + key + '=' + lit_extra_opts[key]) cmake_opts = [properties.WithProperties('-DLLVM_LIT_ARGS='+litTestArgs)] for key in cmake_extra_opts: cmake_opts.append('-D' + key + '=' + cmake_extra_opts[key]) if use_cache: libcxx_cache_dir = '%s/../libcxx/cmake/caches' % rel_src_dir cache = '%s/%s' % (libcxx_cache_dir, use_cache) cmake_opts.append('-C' + cache) # FIXME: The libc++ abilist's are generated in release mode with debug # symbols Other configurations may contain additional non-inlined symbols. if check_libcxx_abilist and not 'CMAKE_BUILD_TYPE' in cmake_extra_opts: cmake_opts.append('-DCMAKE_BUILD_TYPE=RELWITHDEBINFO') # Force libc++ to use the in-tree libc++abi unless otherwise specified. if 'LIBCXX_CXX_ABI' not in cmake_extra_opts: cmake_opts.append('-DLIBCXX_CXX_ABI=libcxxabi') # Nuke/remake build directory and run CMake f.addStep(buildbot.steps.shell.ShellCommand( name='rm.builddir', command=['rm', '-rf', build_path], workdir=".", haltOnFailure=False)) CmakeCommand.applyRequiredOptions(cmake_opts, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) f.addStep(buildbot.steps.shell.ShellCommand( name='cmake', command=['cmake', rel_src_dir] + cmake_opts, haltOnFailure=True, workdir=build_path, env=env)) # Build libcxxabi jobs_flag = properties.WithProperties('-j%(jobs)s') f.addStep(buildbot.steps.shell.ShellCommand( name='build.libcxxabi', command=['make', jobs_flag, 'cxxabi'], haltOnFailure=True, workdir=build_path)) # Build libcxx f.addStep(buildbot.steps.shell.ShellCommand( name='build.libcxx', command=['make', jobs_flag, 'cxx'], haltOnFailure=True, workdir=build_path)) # Test libc++abi f.addStep(LitTestCommand( name = 'test.libcxxabi', command = ['make', jobs_flag, 'check-cxxabi'], description = ['testing', 'libcxxabi'], descriptionDone = ['test', 'libcxxabi'], workdir = build_path)) # Test libc++ f.addStep(LitTestCommand( name = 'test.libcxx', command = ['make', jobs_flag, 'check-cxx'], description = ['testing', 'libcxx'], descriptionDone = ['test', 'libcxx'], workdir = build_path)) if check_libcxx_abilist: f.addStep(buildbot.steps.shell.ShellCommand( name = 'test.libcxx.abilist', command = ['make', 'check-cxx-abilist'], description = ['testing', 'libcxx', 'abi'], descriptionDone = ['test', 'libcxx', 'abi'], workdir = build_path)) if check_libcxx_benchmarks: # Build the libc++ benchmarks f.addStep(buildbot.steps.shell.ShellCommand( name='build.libcxx.benchmarks', command=['make', jobs_flag, 'cxx-benchmarks'], haltOnFailure=True, workdir=build_path)) # Run the benchmarks f.addStep(LitTestCommand( name = 'test.libcxx.benchmarks', command = ['make', jobs_flag, 'check-cxx-benchmarks'], description = ['testing', 'libcxx', 'benchmarks'], descriptionDone = ['test', 'libcxx', 'benchmarks'], workdir = build_path)) return f
def getPollyBuildFactory( clean=False, install=False, make='make', jobs=None, checkAll=False, env=None, extraCmakeArgs=None, testsuite=False,extraTestsuiteCmakeArgs=None, **kwargs): if extraCmakeArgs is None: extraCmakeArgs=[] if extraTestsuiteCmakeArgs is None: extraTestsuiteCmakeArgs = [] llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" llvm_instdir = "llvm.inst" testsuite_srcdir = "test-suite.src" testsuite_builddir = "test-suite.build" jobs_cmd = [] if jobs is not None: jobs_cmd = ["-j{}".format(jobs)] build_cmd = [make] + jobs_cmd install_cmd = [make, 'install'] + jobs_cmd check_all_cmd = [make, 'check-all'] + jobs_cmd check_polly_cmd = [make, 'check-polly'] + jobs_cmd cmake_install = [] if install: cmake_install = ["-DCMAKE_INSTALL_PREFIX=../%s" % llvm_instdir] # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env: merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything. depends_on_projects = ['llvm','clang','polly'] if testsuite: # XRay tests in test-suite require compiler-rt depends_on_projects += ['compiler-rt'] cleanBuildRequestedByProperty = lambda step: step.build.getProperty("clean", False) cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=llvm_objdir, install_dir=llvm_instdir, cleanBuildRequested=cleanBuildRequested, **kwargs) # Pass through all the extra arguments. f.addStep(steps.RemoveDirectory(name='clean-src-dir', dir=f.monorepo_dir, warnOnFailure=True, doStepIf=cleanBuildRequestedByProperty)) # Get the source code. f.addGetSourcecodeSteps(**kwargs) # Clean build dir f.addStep(steps.RemoveDirectory(name='clean-build-dir', dir=llvm_objdir, warnOnFailure=True, doStepIf=cleanBuildRequested)) # Create configuration files with cmake cmakeCommand = ["cmake", "../%s/llvm" % llvm_srcdir, "-DCMAKE_COLOR_MAKEFILE=OFF", "-DPOLLY_TEST_DISABLE_BAR=ON", "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_POLLY_LINK_INTO_TOOLS=ON", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), ] + cmake_install + extraCmakeArgs f.addStep(ShellCommand(name="cmake-configure", command=cmakeCommand, haltOnFailure=False, description=["cmake configure"], workdir=llvm_objdir, env=merged_env)) # Build f.addStep(WarningCountingShellCommand(name="build", command=build_cmd, haltOnFailure=True, description=["build"], workdir=llvm_objdir, env=merged_env)) clangexe = "%(builddir)s/" + llvm_objdir + "/bin/clang" clangxxexe = "%(builddir)s/" + llvm_objdir + "/bin/clang++" litexe = "%(builddir)s/" + llvm_objdir + "/bin/llvm-lit" sizeexe = "%(builddir)s/" + llvm_objdir + "/bin/llvm-size" # Clean install dir if install: f.addStep(steps.RemoveDirectory(name='clean-install-dir', dir=llvm_instdir, haltOnFailure=False, doStepIf=cleanBuildRequested)) f.addStep(ShellCommand(name="install", command=install_cmd, haltOnFailure=True, description=["install"], workdir=llvm_objdir, env=merged_env)) # If installing, use the installed version of clang. clangexe = "%(builddir)s/" + llvm_instdir + "/bin/clang" clangxxexe = "%(builddir)s/" + llvm_instdir + "/bin/clang++" sizeexe = "%(builddir)s/" + llvm_instdir + "/bin/llvm-size" # Test if checkAll: f.addStep(LitTestCommand(name="check_all", command=check_all_cmd, haltOnFailure=False, description=["check all"], workdir=llvm_objdir, env=merged_env)) else: f.addStep(LitTestCommand(name="check_polly", command=check_polly_cmd, haltOnFailure=False, description=["check polly"], workdir=llvm_objdir, env=merged_env)) if testsuite: f.addStep(steps.RemoveDirectory(name='test-suite_clean-src-dir', dir=testsuite_srcdir, haltOnFailure=False, warnOnFailure=True, doStepIf=cleanBuildRequestedByProperty)) f.addGetSourcecodeForProject( project='test-suite', src_dir=testsuite_srcdir, alwaysUseLatest=True) f.addStep(steps.RemoveDirectory(name='test-suite_clean-build-dir', dir=testsuite_builddir, haltOnFailure=False, warnOnFailure=True)) # -Wno-unused-command-line-argument is needed because linking will not uses the "-mllvm -polly" argument. f.addStep(ShellCommand(name='test-suite_cmake-configure', description=["Test-Suite: cmake"], command=["cmake", '-B', testsuite_builddir, '-S', testsuite_srcdir, "-DCMAKE_BUILD_TYPE=Release", "-DTEST_SUITE_COLLECT_STATS=ON", "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", "-DTEST_SUITE_EXTRA_CXX_FLAGS=-mllvm -polly", "-DTEST_SUITE_LIT_FLAGS=-vv;-o;report.json", WithProperties("-DCMAKE_C_COMPILER=" + clangexe), WithProperties("-DCMAKE_CXX_COMPILER=" + clangxxexe), WithProperties("-DTEST_SUITE_LLVM_SIZE=" + sizeexe), WithProperties("-DTEST_SUITE_LIT=" + litexe), ] + extraTestsuiteCmakeArgs, haltOnFailure=True, workdir='.', env=merged_env)) f.addStep(WarningCountingShellCommand(name='test-suite_build', description=["Test-Suite: build"], # Continue building; programs that don't compile will fail with NOEXE. command=[make, 'all', '-k0'] + jobs_cmd, haltOnFailure=False, flunkOnFailure=True, workdir=testsuite_builddir, env=merged_env)) f.addStep(LitTestCommand(name='test-suite_run', description=['Test-Suite: run'], command=[WithProperties(litexe), '-vv', '-o', 'report.json', '.'], haltOnFailure=True, workdir=testsuite_builddir, logfiles={ 'test.log' : 'test.log', 'report.json': 'report.json' }, env=merged_env)) return f
def addCmakeSteps(f, cleanBuildRequested, obj_dir, generator=None, install_dir=None, extra_configure_args=None, env=None, stage_name=None, **kwargs): # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() if obj_dir is None: obj_dir = f.obj_dir # This is an incremental build, unless otherwise has been requested. # Remove obj and install dirs for a clean build. # TODO: Some Windows workers do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep( steps.RemoveDirectory( name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeTo(install_dir, obj_dir) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel), ]) f.addStep( steps.RemoveDirectory( name='clean-%s-dir' % install_dir, dir=install_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Reconcile the cmake options for this build. # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_LIT_ARGS=', '-v -vv'), ]) # Create configuration files with cmake, unless this has been already done # for an incremental build. if stage_name: step_name = "cmake-configure-%s" % stage_name else: stage_name = "" step_name = "cmake-configure" src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir) # Make a local copy of the configure args, as we are going to modify that. definitions = dict() options = list() for d in cmake_args: if isinstance(d, str) and d.startswith("-D"): k, v = d[2:].split('=', 1) definitions[k] = v else: options.append(d) f.addStep( CmakeCommand( name=step_name, haltOnFailure=True, description=["Cmake", "configure", stage_name], generator=generator, definitions=definitions, options=options, path=src_dir, env=env or {}, workdir=obj_dir, **kwargs # Pass through all the extra arguments. ))
def getLLDBuildFactory( clean = True, jobs = None, extra_configure_args = None, env = None): # Set defaults if jobs is None: jobs = "%(jobs)s" if extra_configure_args is None: extra_configure_args = [] # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'CC' : "clang", 'CXX' : "clang++", 'TERM' : 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) f = LLVMBuildFactory( depends_on_projects=['llvm', 'lld'], llvm_srcdir="llvm.src", llvm_objdir="llvm.obj") # Get LLVM and Lld f.addSVNSteps() # Clean directory, if requested. cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir, dir=f.llvm_objdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested )) # Create configuration files with cmake f.addStep(CmakeCommand(name="cmake-configure", description=["cmake configure"], haltOnFailure=True, options=extra_configure_args, path="../%s" % f.llvm_srcdir, env=merged_env, workdir=f.llvm_objdir, doStepIf=FileDoesNotExist( "./%s/CMakeCache.txt" % f.llvm_objdir))) # Build Lld f.addStep(ShellCommand(name="build_Lld", command=['nice', '-n', '10', 'make', WithProperties("-j%s" % jobs)], haltOnFailure=True, description=["build lld"], env=merged_env, workdir=f.llvm_objdir)) # Test Lld f.addStep(ShellCommand(name="test_lld", command=["make", "lld-test"], haltOnFailure=True, description=["test lld"], env=merged_env, workdir=f.llvm_objdir)) return f
def getLLDWinBuildFactory( clean = True, # Default values for VS devenv and build configuration vs = None, # What to run to configure Visual Studio utils. target_arch = None, # Native. extra_configure_args = None, env = None): # Set defaults if vs is None: vs = r"""%VS140COMNTOOLS%""" # Visual Studio 2015. if extra_configure_args is None: extra_configure_args = [] if env is None: env = {} f = LLVMBuildFactory( depends_on_projects=['llvm', 'lld'], llvm_srcdir="llvm.src", llvm_objdir="llvm.obj") # Get LLVM and Lld f.addSVNSteps() # Clean directory, if requested. cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir, dir=f.llvm_objdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested )) # If set up environment step is requested, do this now. if vs: f.addStep(SetProperty( command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) assert not env, "Can't have custom builder env vars with VS" env = Property('slave_env') # Always build with ninja. cmake_options = ["-G", "Ninja"] # Reconsile configure args with the defaults we want. if not any(a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args): cmake_options.append('-DCMAKE_BUILD_TYPE=Release') if not any(a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args): cmake_options.append('-DLLVM_ENABLE_WERROR=ON') if not any(a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args): cmake_options.append('-DLLVM_ENABLE_ASSERTIONS=ON') if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args): cmake_options.append('-DLLVM_LIT_ARGS=\"-v\"') cmake_options += extra_configure_args # Note: ShellCommand does not pass the params with special symbols right. # The " ".join is a workaround for this bug. f.addStep(CmakeCommand(name="cmake-configure", description=["cmake configure"], haltOnFailure=True, warnOnWarnings=True, options=cmake_options, path="../%s" % f.llvm_srcdir, env=env, workdir=f.llvm_objdir, doStepIf=FileDoesNotExist( "./%s/CMakeCache.txt" % f.llvm_objdir))) # Build Lld. f.addStep(NinjaCommand(name='build lld', haltOnFailure=True, warnOnWarnings=True, description='build lld', workdir=f.llvm_objdir, env=env)) # Test Lld f.addStep(NinjaCommand(name='test lld', targets=['lld-test'], haltOnFailure=True, warnOnWarnings=True, description='test lld', workdir=f.llvm_objdir, env=env)) return f
def _getClangCMakeBuildFactory( clean=True, test=True, cmake='cmake', jobs=None, # VS tools environment variable if using MSVC. For example, # %VS120COMNTOOLS% selects the 2013 toolchain. vs=None, vs_target_arch='x86', # Multi-stage compilation useTwoStage=False, testStage1=True, stage1_config='Release', stage2_config='Release', # Test-suite runTestSuite=False, nt_flags=None, testsuite_flags=None, submitURL=None, testerName=None, # Environmental variables for all steps. env=None, extra_cmake_args=None, # Extra repositories checkout_clang_tools_extra=True, checkout_compiler_rt=True, checkout_lld=True, checkout_libcxx=False, checkout_test_suite=False, checkout_flang=False, # Upload artifacts to Google Cloud Storage (for the llvmbisect tool) stage1_upload_directory=None, # Use a lower compression level to generate the build-cache package faster # default is 6 according to documentation xz_compression_factor=6, use_pixz_compression=False, # Triggers trigger_after_stage1=None): ############# PREPARING if nt_flags is None: nt_flags = [] if testsuite_flags is None: testsuite_flags = [] if env is None: env = {} if extra_cmake_args is None: extra_cmake_args = [] if trigger_after_stage1 is None: trigger_after_stage1 = [] clean_build_requested = lambda step: \ step.build.getProperty( \ "clean", \ default=step.build.getProperty("clean_obj") \ ) or clean # We *must* checkout at least Clang+LLVM depends_on_projects = ['llvm', 'clang'] if checkout_clang_tools_extra: depends_on_projects.append('clang-tools-extra') if checkout_compiler_rt: depends_on_projects.append('compiler-rt') if checkout_lld: depends_on_projects.append('lld') if checkout_libcxx: depends_on_projects.append('libcxx') depends_on_projects.append('libcxxabi') depends_on_projects.append('libunwind') if checkout_flang: depends_on_projects.append('flang') depends_on_projects.append('mlir') f = LLVMBuildFactory(depends_on_projects=depends_on_projects, llvm_srcdir='llvm') # Checkout the latest code for LNT # and the test-suite separately. Le's do this first, # so we wouldn't poison got_revision property. if runTestSuite or checkout_test_suite: f.addGetSourcecodeForProject(project='lnt', src_dir='test/lnt', alwaysUseLatest=True) f.addGetSourcecodeForProject(project='test-suite', src_dir='test/test-suite', alwaysUseLatest=True) # Then get the LLVM source code revision this particular build is for. f.addGetSourcecodeSteps() # If jobs not defined, Ninja will choose a suitable value jobs_cmd = [] lit_args = "'-v" if jobs is not None: jobs_cmd = ["-j" + str(jobs)] lit_args += " -j" + str(jobs) + "'" else: lit_args += "'" ninja_cmd = ['ninja'] + jobs_cmd ninja_install_cmd = ['ninja', 'install'] + jobs_cmd ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd # Global configurations stage1_build = 'stage1' stage1_install = 'stage1.install' stage2_build = 'stage2' stage2_install = 'stage2.install' # Set up VS environment, if appropriate. if vs and vs != "manual": f.addStep( SetProperty(command=builders_util.getVisualStudioEnvironment( vs, vs_target_arch), extract_fn=builders_util.extractSlaveEnvironment)) assert not env, "Can't have custom builder env vars with VS" env = Property('slave_env') ############# CLEANING f.addStep( ShellCommand(name='clean stage 1', command=['rm', '-rf', stage1_build], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='cleaning stage 1', descriptionDone='clean', workdir='.', doStepIf=clean_build_requested)) ############# STAGE 1 CmakeCommand.applyRequiredOptions(extra_cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage1_build) f.addStep( ShellCommand( name='cmake stage 1', command=[ cmake, "-G", "Ninja", rel_src_dir, "-DCMAKE_BUILD_TYPE=" + stage1_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args, "-DCMAKE_INSTALL_PREFIX=../" + stage1_install ] + extra_cmake_args, haltOnFailure=True, description='cmake stage 1', workdir=stage1_build, doStepIf=FileDoesNotExist("build.ninja"), env=env)) f.addStep( WarningCountingShellCommand(name='build stage 1', command=ninja_cmd, haltOnFailure=True, description='ninja all', workdir=stage1_build, env=env)) if test and testStage1: haltOnStage1Check = not useTwoStage and not runTestSuite f.addStep( LitTestCommand(name='ninja check 1', command=ninja_check_cmd, haltOnFailure=haltOnStage1Check, description=["checking stage 1"], descriptionDone=["stage 1 checked"], workdir=stage1_build, env=env)) if useTwoStage or runTestSuite or stage1_upload_directory: f.addStep( ShellCommand(name='clean stage 1 install', command=['rm', '-rf', stage1_install], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='cleaning stage 1 install', descriptionDone='clean', workdir='.')) f.addStep( ShellCommand(name='install stage 1', command=ninja_install_cmd, description='ninja install', workdir=stage1_build, env=env)) if stage1_upload_directory: addGCSUploadSteps(f, 'stage 1', stage1_install, stage1_upload_directory, env, gcs_url_property='stage1_package_gcs_url', use_pixz_compression=use_pixz_compression, xz_compression_factor=xz_compression_factor) # Compute the cmake define flag to set the C and C++ compiler to clang. Use # clang-cl if we used MSVC for stage1. if not vs: cc = 'clang' cxx = 'clang++' else: cc = 'clang-cl.exe' cxx = 'clang-cl.exe' ############# STAGE 2 if useTwoStage: # We always cleanly build the stage 2. If the compiler has been # changed on the stage 1, we cannot trust any of the intermediate file # from the old compiler. And if the stage 1 compiler is the same, we # should not build in the first place. f.addStep( ShellCommand(name='clean stage 2', command=['rm', '-rf', stage2_build], warnOnFailure=True, description='cleaning stage 2', descriptionDone='clean', workdir='.')) # Set the compiler using the CC and CXX environment variables to work around # backslash string escaping bugs somewhere between buildbot and cmake. The # env.exe helper is required to run the tests, so hopefully it's already on # PATH. rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage2_build) cmake_cmd2 = [ cmake, "-G", "Ninja", rel_src_dir, WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + stage1_install + "/bin/" + cc), WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" + stage1_install + "/bin/" + cxx), "-DCMAKE_BUILD_TYPE=" + stage2_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args, "-DCMAKE_INSTALL_PREFIX=../" + stage2_install ] + extra_cmake_args f.addStep( ShellCommand(name='cmake stage 2', command=cmake_cmd2, haltOnFailure=True, description='cmake stage 2', workdir=stage2_build, env=env)) f.addStep( WarningCountingShellCommand(name='build stage 2', command=ninja_cmd, haltOnFailure=True, description='ninja all', workdir=stage2_build, env=env)) if test: f.addStep( LitTestCommand(name='ninja check 2', command=ninja_check_cmd, haltOnFailure=not runTestSuite, description=["checking stage 2"], descriptionDone=["stage 2 checked"], workdir=stage2_build, env=env)) ############# TEST SUITE ## Test-Suite (stage 2 if built, stage 1 otherwise) if runTestSuite: compiler_path = stage1_install if useTwoStage: compiler_path = stage2_install f.addStep( ShellCommand(name='clean stage 2 install', command=['rm', '-rf', stage2_install], warnOnFailure=True, description='cleaning stage 2 install', descriptionDone='clean', workdir='.')) f.addStep( ShellCommand(name='install stage 2', command=ninja_install_cmd, description='ninja install 2', workdir=stage2_build, env=env)) # Get generated python, lnt python = WithProperties('%(workdir)s/test/sandbox/bin/python') lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt') lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py') # Paths sandbox = WithProperties('%(workdir)s/test/sandbox') test_suite_dir = WithProperties('%(workdir)s/test/test-suite') # Get latest built Clang (stage1 or stage2) cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc) cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx) # LNT Command line (don't pass -jN. Users need to pass both --threads # and --build-threads in nt_flags/test_suite_flags to get the same effect) use_runtest_testsuite = len(nt_flags) == 0 if not use_runtest_testsuite: test_suite_cmd = [ python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx ] # Append any option provided by the user test_suite_cmd.extend(nt_flags) else: lit = WithProperties('%(workdir)s/' + stage1_build + '/bin/llvm-lit') test_suite_cmd = [ python, lnt, 'runtest', 'test-suite', '--no-timestamp', '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx, '--use-lit', lit ] # Append any option provided by the user test_suite_cmd.extend(testsuite_flags) # Only submit if a URL has been specified if submitURL is not None: if not isinstance(submitURL, list): submitURL = [submitURL] for url in submitURL: test_suite_cmd.extend(['--submit', url]) # lnt runtest test-suite doesn't understand --no-machdep-info: if testerName and not use_runtest_testsuite: test_suite_cmd.extend(['--no-machdep-info', testerName]) # CC and CXX are needed as env for build-tools test_suite_env = copy.deepcopy(env) test_suite_env['CC'] = cc test_suite_env['CXX'] = cxx # Steps to prepare, build and run LNT f.addStep( ShellCommand(name='clean sandbox', command=['rm', '-rf', 'sandbox'], haltOnFailure=True, description='removing sandbox directory', workdir='test', env=env)) f.addStep( ShellCommand(name='recreate sandbox', command=['virtualenv', 'sandbox'], haltOnFailure=True, description='recreating sandbox', workdir='test', env=env)) f.addStep( ShellCommand(name='setup lit', command=[python, lnt_setup, 'develop'], haltOnFailure=True, description='setting up LNT in sandbox', workdir='test/sandbox', env=env)) f.addStep( LitTestCommand(name='test-suite', command=test_suite_cmd, haltOnFailure=True, description=['running the test suite'], workdir='test/sandbox', logfiles={ 'configure.log': 'build/configure.log', 'build-tools.log': 'build/build-tools.log', 'test.log': 'build/test.log', 'report.json': 'build/report.json' }, env=test_suite_env)) return f
def getOpenMPCMakeBuildFactory( jobs = '%(jobs)s', # Number of concurrent jobs. clean = True, # "clean" step is requested if true env = None, # Environmental variables for all steps. ompt = False, # Whether to enable the OpenMP Tools Interface. test = True, # Test the built libraries. depends_on_projects = None, **kwargs): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } # Overwrite pre-set items with the given ones, so user can set anything. if env is not None: merged_env.update(env) llvm_srcdir = 'llvm.src' llvm_builddir = 'llvm.build' cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) if depends_on_projects is None: # Monorepo configuration requires llvm and clang to get cmake work. depends_on_projects = ['llvm', 'clang', 'openmp'] f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=llvm_builddir, cleanBuildRequested=cleanBuildRequested, env=merged_env, **kwargs) # Pass through all the extra arguments. f.addStep( ShellCommand( name = 'clean', command = ['rm', '-rf', f.obj_dir], warnOnFailure = True, description = ['clean'], doStepIf = cleanBuildRequested, workdir = '.', env = merged_env)) # Configure LLVM and OpenMP (and Clang, if requested). cmake_args = ['cmake', '-G', 'Ninja'] cmake_args += ['-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_ASSERTIONS=ON'] if ompt: cmake_args += ['-DLIBOMP_OMPT_SUPPORT=ON'] if test: lit_args = '-vv --show-unsupported --show-xfail -j %s' % jobs cmake_args += [WithProperties('-DLLVM_LIT_ARGS=%s' % lit_args)] CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) # Add llvm-lit and clang (if built) to PATH merged_env.update({ 'PATH': WithProperties('%(workdir)s/' + llvm_builddir + '/bin:${PATH}')}) src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, f.obj_dir) f.addStep(CmakeCommand(name='configure-openmp', description=['configure','openmp'], options=cmake_args, path=src_dir, env=merged_env, workdir=f.obj_dir, haltOnFailure=True, **kwargs # Pass through all the extra arguments. )) # Build OpenMP runtime libraries. f.addStep( NinjaCommand( name = 'compile-openmp', description = 'compile openmp', workdir = f.obj_dir, env = merged_env, haltOnFailure=True)) # Test OpenMP runtime libraries, if requested. if test: # Add llvm-lit and clang (if built) to PATH merged_env.update({ 'PATH': WithProperties('%(workdir)s/' + llvm_builddir + '/bin:${PATH}')}) ninja_test_args = ['ninja', WithProperties('-j %s' % jobs)] f.addStep( LitTestCommand( name = 'test-openmp', command = ninja_test_args + ['check-openmp'], description = 'test openmp', workdir = f.obj_dir, env = merged_env, haltOnFailure=True)) return f
def _addSteps4SystemCompiler(f, stage_idx=0, clean=True, jobs=None, extra_configure_args=None, env=None): # Index is zero-based, so we want to use a human friendly number instead. stage_num = stage_idx + 1 # Directories to use on this stage. obj_dir = f.stage_objdirs[stage_idx] src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir) install_dir = LLVMBuildFactory.pathRelativeTo( f.stage_installdirs[stage_idx], obj_dir) # This stage could use incremental build. # Clean stage1, only if requested. f.addStep( steps.RemoveDirectory(name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=clean)) f.addStep( steps.RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx], dir=f.stage_installdirs[stage_idx], haltOnFailure=False, flunkOnFailure=False, doStepIf=clean)) # Reconcile the cmake options for this stage. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults. CmakeCommand.applyDefaultOptions( cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), # Do not expect warning free build by the system toolchain. ('-DLLVM_ENABLE_WERROR=', 'OFF'), ]) # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) # Note: On this stage we do not care of warnings, as we build with # a system toolchain and cannot control the environment. # Warnings are likely, and we ignore them. # Create configuration files with cmake f.addStep( CmakeCommand( name="cmake-configure-stage%s" % stage_num, description=["stage%s cmake configure" % stage_num], haltOnFailure=True, flunkOnWarnings=False, options=cmake_args, path=src_dir, env=env, workdir=obj_dir, )) # Build clang by the system compiler f.addStep( NinjaCommand( name="build-stage%s-compiler" % stage_num, jobs=jobs, haltOnFailure=True, flunkOnWarnings=False, description=["build stage%s compiler" % stage_num], env=env, workdir=obj_dir, )) # Test stage1 compiler f.addStep( NinjaCommand( name="test-stage%s-compiler" % stage_num, targets=["check-all"], # or "check-llvm", "check-clang" jobs=jobs, haltOnFailure=True, flunkOnWarnings=False, description=["test stage%s compiler" % stage_num], env=env, workdir=obj_dir, )) # Install stage1 compiler f.addStep( NinjaCommand( name="install-stage%s-compiler" % stage_num, targets=["install"], jobs=jobs, haltOnFailure=True, description=["install stage%s compiler" % stage_num], env=env, workdir=obj_dir, ))
def addCmakeSteps(f, cleanBuildRequested, obj_dir, install_dir=None, extra_configure_args=None, env=None, stage_name=None, **kwargs): # Consume is_legacy_mode if given. # TODO: Remove this once legacy mode gets dropped. kwargs.pop('is_legacy_mode', None) # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() if obj_dir is None: obj_dir = f.obj_dir # This is an incremental build, unless otherwise has been requested. # Remove obj and install dirs for a clean build. # TODO: Some Windows slaves do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep( RemoveDirectory( name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) if not f.is_legacy_mode: CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeToBuild( install_dir, obj_dir) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel), ]) f.addStep( RemoveDirectory( name='clean-%s-dir' % install_dir, dir=install_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Reconcile the cmake options for this build. # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_LIT_ARGS=', '-v -vv'), ]) # Create configuration files with cmake, unless this has been already done # for an incremental build. if stage_name: step_name = "cmake-configure-%s" % stage_name else: stage_name = "" step_name = "cmake-configure" src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) f.addStep( CmakeCommand( name=step_name, haltOnFailure=True, description=["Cmake", "configure", stage_name], options=cmake_args, path=src_dir, env=env, workdir=obj_dir, **kwargs # Pass through all the extra arguments. ))
def _addSteps4SystemCompiler( f, stage_idx = 0, clean = True, jobs = None, extra_configure_args = None, env = None): # Index is zero-based, so we want to use a human friendly number instead. stage_num = stage_idx + 1 # Directories to use on this stage. obj_dir = f.stage_objdirs[stage_idx] src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir) # This stage could use incremental build. # Clean stage1, only if requested. f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=clean )) f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx], dir=f.stage_installdirs[stage_idx], haltOnFailure=False, flunkOnFailure=False, doStepIf=clean )) # Reconcile the cmake options for this stage. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), # Do not expect warning free build by the system toolchain. ('-DLLVM_ENABLE_WERROR=', 'OFF'), ]) # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) # Note: On this stage we do not care of warnings, as we build with # a system toolchain and cannot control the environment. # Warnings are likely, and we ignore them. # Create configuration files with cmake f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num, description=["stage%s cmake configure" % stage_num], haltOnFailure=True, flunkOnWarnings=False, options=cmake_args, path=src_dir, env=env, workdir=obj_dir, doStepIf=FileDoesNotExist("CMakeCache.txt") )) # Build clang by the system compiler f.addStep(NinjaCommand(name="build-stage%s-compiler" % stage_num, jobs=jobs, haltOnFailure=True, flunkOnWarnings=False, description=["build stage%s compiler" % stage_num], env=env, workdir=obj_dir, )) # Test stage1 compiler f.addStep(NinjaCommand(name="test-stage%s-compiler"% stage_num, targets=["check-all"], # or "check-llvm", "check-clang" jobs=jobs, haltOnFailure=True, flunkOnWarnings=False, description=["test stage%s compiler" % stage_num], env=env, workdir=obj_dir, )) # Install stage1 compiler f.addStep(NinjaCommand(name="install-stage%s-compiler"% stage_num, targets=["install"], jobs=jobs, haltOnFailure=True, description=["install stage%s compiler" % stage_num], env=env, workdir=obj_dir, ))
def _addSteps4StagedCompiler( f, stage_idx = 1, use_stage_idx = -1, withLTOSupport = False, jobs = None, extra_configure_args = None, env = None): if use_stage_idx < 0: use_stage_idx = stage_idx - 1 # Index is zero-based, so we want to use a human friendly number instead. stage_num = stage_idx + 1 # Directories to use on this stage. obj_dir = f.stage_objdirs[stage_idx] src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir) staged_install = f.stage_installdirs[use_stage_idx] # Always do a clean build for the staged compiler. f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, )) f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx], dir=f.stage_installdirs[stage_idx], haltOnFailure=False, flunkOnFailure=False, )) # Reconcile the cmake options for this stage. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ]) if withLTOSupport: CmakeCommand.applyDefaultOptions(cmake_args, [ # LTO Plugin dependency: ('-DLLVM_BINUTILS_INCDIR=', '/opt/binutils/include'), ]) # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) cmake_args.append( WithProperties( "-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++" )) cmake_args.append( WithProperties( "-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang" )) # Create configuration files with cmake f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num, description=["stage%s cmake configure" % stage_num], haltOnFailure=True, options=cmake_args, path=src_dir, env=env, workdir=obj_dir, doStepIf=FileDoesNotExist("CMakeCache.txt") )) if withLTOSupport: # Build LTO plugin if requested. f.addStep(NinjaCommand(name="build-stage%s-LLVMgold.so" % stage_num, targets=['lib/LLVMgold.so'], jobs=jobs, haltOnFailure=True, description=["stage%s build LLVMgold.so" % stage_num], env=env, workdir=obj_dir, )) # Build clang by the staged compiler f.addStep(NinjaCommand(name="build-stage%s-compiler" % stage_num, jobs=jobs, haltOnFailure=True, description=["build stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, )) # Test just built compiler f.addStep(NinjaCommand(name="test-stage%s-compiler"% stage_num, targets=["check-all"], jobs=jobs, haltOnFailure=True, description=["test stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, )) # Install just built compiler f.addStep(NinjaCommand(name="install-stage%s-compiler"% stage_num, targets=["install"], jobs=jobs, haltOnFailure=True, description=["install stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, ))
def addCmakeSteps(f, cleanBuildRequested, obj_dir, install_dir=None, extra_configure_args=None, env=None, stage_name=None, **kwargs): # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # This is an incremental build, unless otherwise has been requested. # Remove obj and install dirs for a clean build. # TODO: Some Windows slaves do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep( RemoveDirectory( name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeToBuild( install_dir, obj_dir) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel), ]) f.addStep( RemoveDirectory( name='clean-%s-dir' % install_dir, dir=install_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Reconcile the cmake options for this build. # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ('-DLLVM_LIT_ARGS=', '"-v"'), ]) # Create configuration files with cmake, unless this has been already done # for an incremental build. if stage_name: step_name = "cmake-configure-%s" % stage_name else: stage_name = "" step_name = "cmake-configure" src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) f.addStep( CmakeCommand( name=step_name, description=["Cmake", "configure", stage_name], options=cmake_args, path=src_dir, haltOnFailure=kwargs.get('haltOnFailure', True), env=env, workdir=obj_dir, **kwargs # Pass through all the extra arguments. ))
def getClangWithLTOBuildFactory( depends_on_projects = None, clean = False, jobs = None, extra_configure_args = None, compare_last_2_stages = True, lto = None, # The string gets passed to -flto flag as is. Like -flto=thin. env = None): # Set defaults if depends_on_projects: depends_on_projects = list(depends_on_projects) else: # By default we link with LLD. depends_on_projects = ['llvm', 'clang', 'lld'] if lto is None: lto = 'ON' if jobs is None: jobs = "%(jobs)s" if extra_configure_args is None: extra_configure_args = [] else: extra_configure_args = list(extra_configure_args) # Make sure CMAKE_INSTALL_PREFIX and -G are not specified # in the extra_configure_args. We set them internally as needed. # TODO: assert extra_configure_args. install_prefix_specified = any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) assert not install_prefix_specified, "Please do not explicitly specify the install prefix for multi-stage build." # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir="llvm.src", stage_objdirs=[ "build/stage1", "build/stage2", "build/stage3", "build/stage4", ], stage_installdirs=[ "install/stage1", "install/stage2", "install/stage3", "install/stage4", ], staged_compiler_idx = 1) cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean # Do a clean checkout if requested. f.addStep(RemoveDirectory(name='clean-src-dir', dir=f.llvm_srcdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Get the source code. f.addSVNSteps() # Build with the system compiler first _addSteps4SystemCompiler(f, stage_idx=0, clean=cleanBuildRequested, jobs=jobs, extra_configure_args=extra_configure_args, env=merged_env) # Then build the compiler we would use for the bootstrap. _addSteps4StagedCompiler(f, stage_idx=1, withLTOSupport=True, jobs=jobs, extra_configure_args=extra_configure_args, env=merged_env) # Build all the remaining stages with exactly the same configuration. CmakeCommand.applyRequiredOptions(extra_configure_args, [ ('-DLLVM_ENABLE_LTO=', lto), ]) # If we build LLD, we would link with LLD. # Otherwise we link with the system linker. if 'lld' in depends_on_projects: CmakeCommand.applyRequiredOptions(extra_configure_args, [ ('-DLLVM_ENABLE_LLD=', 'ON'), ]) # The rest are test stages, which depend on the staged compiler we are ultimately after. s = f.staged_compiler_idx + 1 staged_install = f.stage_installdirs[f.staged_compiler_idx] for i in range(s, len(f.stage_objdirs[s:]) + s): configure_args = extra_configure_args[:] configure_args.append( WithProperties( "-DCMAKE_AR=%(workdir)s/" + staged_install + "/bin/llvm-ar" )) configure_args.append( WithProperties( "-DCMAKE_RANLIB=%(workdir)s/" + staged_install + "/bin/llvm-ranlib" )) _addSteps4StagedCompiler(f, stage_idx=i, use_stage_idx=f.staged_compiler_idx, jobs=jobs, extra_configure_args=configure_args, env=merged_env) if compare_last_2_stages: # Compare the compilers built on the last 2 stages if requested. diff_command = [ "diff", "-q", f.stage_installdirs[-2] + "/bin/clang", f.stage_installdirs[-1] + "/bin/clang", ] f.addStep( ShellCommand( name="compare-compilers", description=[ "compare", "stage%d" % (len(f.stage_installdirs)-1), "and", "stage%d" % len(f.stage_installdirs), "compilers", ], haltOnFailure=False, command=WithProperties(" ".join(diff_command)), workdir=".", env=merged_env ) ) # Only if the compare-compilers step has failed. def _prevStepFailed(step): steps = step.build.getStatus().getSteps() prev_step = steps[-2] (result, _) = prev_step.getResults() return (result == FAILURE) dir1 = f.stage_objdirs[-2] dir2 = f.stage_objdirs[-1] inc_pattern = "-type f -not -name *.inc -printf '%f\n'" find_cmd = "find %s %s" % (dir1, dir2) diff_cmd = "diff -ru %s %s -x '*.tmp*' -X -" % (dir1, dir2) # Note: Use a string here as we want the command executed by a shell. diff_tablegen_inc_files_command = "%s %s | %s" % (find_cmd, inc_pattern, diff_cmd) f.addStep( ShellCommand( name="compare-tablegen-inc-files", description=[ "compare", "stage%d" % (len(f.stage_installdirs)-1), "and", "stage%d" % len(f.stage_installdirs), "Tablegen inc files", ], command=diff_tablegen_inc_files_command, workdir=".", env=merged_env, doStepIf=_prevStepFailed, ) ) return f
def getFlangOutOfTreeBuildFactory( checks = None, clean = False, llvm_extra_configure_args = None, flang_extra_configure_args = None, env = None, **kwargs): if env is None: env = dict() f = getCmakeWithNinjaBuildFactory( depends_on_projects=['llvm','mlir'], obj_dir="build_llvm", checks=[], clean=clean, extra_configure_args=llvm_extra_configure_args, env=env, **kwargs) if checks is None: checks = ['check-all'] # Make a local copy of the flang configure args, as we are going to modify that. if flang_extra_configure_args: flang_cmake_args = flang_extra_configure_args[:] else: flang_cmake_args = list() # Some options are required for this build no matter what. CmakeCommand.applyRequiredOptions(flang_cmake_args, [ ('-G', 'Ninja'), ]) flang_obj_dir = "build_flang" flang_src_dir = "{}/flang".format(f.monorepo_dir) # Add LLVM_DIR and MLIR_DIR to the CMake invocation. llvm_dir = "{}/lib/cmake/llvm".format(f.obj_dir) mlir_dir = "{}/lib/cmake/mlir".format(f.obj_dir) CmakeCommand.applyRequiredOptions(flang_cmake_args, [ # We actually need the paths to be relative to the source directory, # otherwise find_package can't locate the config files. ('-DLLVM_DIR:PATH=', LLVMBuildFactory.pathRelativeTo(llvm_dir, flang_src_dir)), ('-DMLIR_DIR:PATH=', LLVMBuildFactory.pathRelativeTo(mlir_dir, flang_src_dir)), ]) # We can't use addCmakeSteps as that would use the path in f.llvm_srcdir. f.addStep(CmakeCommand(name="cmake-configure-flang", haltOnFailure=True, description=["CMake", "configure", "flang"], options=flang_cmake_args, path=LLVMBuildFactory.pathRelativeTo(flang_src_dir, flang_obj_dir), env=env, workdir=flang_obj_dir, **kwargs)) addNinjaSteps( f, obj_dir=flang_obj_dir, checks=checks, env=env, stage_name="flang", **kwargs) return f
def getLLDBScriptCommandsFactory( downloadBinary=True, buildAndroid=False, runTest=True, scriptExt='.sh', extra_cmake_args=None, depends_on_projects=None, ): if scriptExt is '.bat': pathSep = '.\\' else: pathSep = './' if extra_cmake_args is None: extra_cmake_args = [] if depends_on_projects is None: f = buildbot.process.factory.BuildFactory() else: f = LLVMBuildFactory( depends_on_projects=depends_on_projects) # Update scripts getShellCommandStep(f, name='update scripts', command=['updateScripts' + scriptExt]) # Acquire lock if downloadBinary: getShellCommandStep(f, name='acquire lock', command=[pathSep + 'acquireLock' + scriptExt, 'totBuild'], description='get') # Checkout source code getShellCommandStep(f, name='checkout source code', command=[pathSep + 'checkoutSource' + scriptExt, WithProperties('%(revision)s')]) # Set source revision f.addStep(SetProperty(name="set revision", command=[pathSep + 'getRevision' + scriptExt], property="got_revision", workdir="scripts")) # Configure getShellCommandStep(f, name='cmake local', command=[pathSep + 'cmake' + scriptExt] + extra_cmake_args) # Build getShellCommandStep(f, name='ninja build local', command=[pathSep + 'buildLocal' + scriptExt]) if buildAndroid: getShellCommandStep(f, name='build android', command=[pathSep + 'buildAndroid' + scriptExt]) # Get lldb-server binaries if downloadBinary: getShellCommandStep(f, name='get lldb-server binaries', command=[pathSep + 'downloadBinaries' + scriptExt, WithProperties('%(got_revision)s')]) # Test if runTest: f.addStep(LitTestCommand(name="run unit tests", command=[pathSep + 'testUnit' + scriptExt], description=["testing"], descriptionDone=["unit test"], workdir='scripts')) getTestSteps(f, scriptExt, pathSep) # upload test traces getShellCommandStep(f, name='upload test traces', command=[pathSep + 'uploadTestTrace' + scriptExt, WithProperties('%(buildnumber)s'), WithProperties('%(buildername)s')], flunkOnFailure=False) # Upload lldb-server binaries and trigger android builders if buildAndroid: getShellCommandStep(f, name='upload lldb-server binaries', command=[pathSep + 'uploadBinaries' + scriptExt]) f.addStep(trigger.Trigger(schedulerNames=['lldb_android_scheduler'], updateSourceStamp=False, waitForFinish=False)) # Release lock if downloadBinary: getShellCommandStep(f, name='release lock', command=[pathSep + 'releaseLock' + scriptExt, 'totBuild'], description='release', alwaysRun=True) return f
def getLLDBScriptCommandsFactory( downloadBinary=True, buildAndroid=False, runTest=True, scriptExt='.sh', extra_cmake_args=None, depends_on_projects=None, ): if scriptExt is '.bat': pathSep = '.\\' else: pathSep = './' if extra_cmake_args is None: extra_cmake_args = [] if depends_on_projects is None: f = buildbot.process.factory.BuildFactory() else: f = LLVMBuildFactory(depends_on_projects=depends_on_projects) # Update scripts getShellCommandStep(f, name='update scripts', command=['updateScripts' + scriptExt]) # Acquire lock if downloadBinary: getShellCommandStep( f, name='acquire lock', command=[pathSep + 'acquireLock' + scriptExt, 'totBuild'], description='get') # Checkout source code getShellCommandStep(f, name='checkout source code', command=[ pathSep + 'checkoutSource' + scriptExt, WithProperties('%(revision)s') ]) # Set source revision f.addStep( SetProperty(name="set revision", command=[pathSep + 'getRevision' + scriptExt], property="got_revision", workdir="scripts")) # Configure getShellCommandStep(f, name='cmake local', command=[pathSep + 'cmake' + scriptExt] + extra_cmake_args) # Build getShellCommandStep(f, name='ninja build local', command=[pathSep + 'buildLocal' + scriptExt]) if buildAndroid: getShellCommandStep(f, name='build android', command=[pathSep + 'buildAndroid' + scriptExt]) # Get lldb-server binaries if downloadBinary: getShellCommandStep(f, name='get lldb-server binaries', command=[ pathSep + 'downloadBinaries' + scriptExt, WithProperties('%(got_revision)s') ]) # Test if runTest: f.addStep( LitTestCommand(name="run unit tests", command=[pathSep + 'testUnit' + scriptExt], description=["testing"], descriptionDone=["unit test"], workdir='scripts')) getTestSteps(f, scriptExt, pathSep) # upload test traces getShellCommandStep(f, name='upload test traces', command=[ pathSep + 'uploadTestTrace' + scriptExt, WithProperties('%(buildnumber)s'), WithProperties('%(buildername)s') ], flunkOnFailure=False) # Upload lldb-server binaries and trigger android builders if buildAndroid: getShellCommandStep(f, name='upload lldb-server binaries', command=[pathSep + 'uploadBinaries' + scriptExt]) f.addStep( trigger.Trigger(schedulerNames=['lldb_android_scheduler'], updateSourceStamp=False, waitForFinish=False)) # Release lock if downloadBinary: getShellCommandStep( f, name='release lock', command=[pathSep + 'releaseLock' + scriptExt, 'totBuild'], description='release', alwaysRun=True) return f
def getAnnotatedBuildFactory( script, clean=False, depends_on_projects=None, env=None, extra_args=None, timeout=1200, is_legacy_mode=False, checkout_llvm_sources=True): """ Returns a new build factory that uses AnnotatedCommand, which allows the build to be run by version-controlled scripts that do not require a buildmaster restart to update. script: script under "builders/annotated" to be run by python clean: set to true for a clean build of llvm depends_on_projects: which subprojects to enable llvm must be first in the list (default: ["llvm", "clang", "compiler-rt", "libcxx", "libcxxabi", "libunwind", "lld"]) env: environment overrides (map; default is no overrides) extra_args: extra arguments to pass to the script (default: []) timeout: specifies the builder's timeout in seconds (default: 1200) """ if depends_on_projects is None: depends_on_projects = [ "llvm", "clang", "compiler-rt", "libcxx", "libcxxabi", "libunwind", "lld"] if extra_args is None: extra_args = [] f = LLVMBuildFactory( is_legacy_mode=is_legacy_mode, clean=clean, depends_on_projects=depends_on_projects) if clean: f.addStep(SetProperty(property='clean', command='echo 1')) # We normally use the clean property to indicate that we want a # clean build, but AnnotatedCommand uses the clobber property # instead. Therefore, set clobber if clean is set to a truthy # value. This will cause AnnotatedCommand to set # BUILDBOT_CLOBBER=1 in the environment, which is how we # communicate to the script that we need a clean build. f.addStep(SetProperty( property='clobber', command='echo 1', doStepIf=lambda step: step.build.getProperty('clean', False))) merged_env = { 'TERM': 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set # anything. merged_env.update(env) scripts_dir = "annotated" # Check out zorg so we can run the annotator scripts. f.addGetSourcecodeForProject( name='update-annotated-scripts', project='zorg', src_dir='llvm-zorg', alwaysUseLatest=True) if checkout_llvm_sources: f.addGetSourcecodeSteps() extra_args_with_props = [WithProperties(arg) for arg in extra_args] # Explicitly use '/' as separator, because it works on *nix and Windows. if script.startswith('/'): command = [script] else: script_path = "../llvm-zorg/zorg/buildbot/builders/annotated/%s" % (script) command = ["python", script_path, WithProperties("--jobs=%(jobs:-)s")] command += extra_args_with_props f.addStep(AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, command=command, env=merged_env)) return f