def addNinjaSteps( f, obj_dir = None, targets = None, checks = None, install_dir = None, env = None, stage_name = None, **kwargs): # Build the unified tree. if stage_name: step_name = "%s-" % stage_name else: stage_name = "" step_name = "" if obj_dir is None: obj_dir = f.obj_dir f.addStep(NinjaCommand(name="build-%sunified-tree" % step_name, haltOnFailure=True, targets=targets, description=["Build", stage_name, "unified", "tree"], env=env, workdir=obj_dir, **kwargs # Pass through all the extra arguments. )) # Test just built components if requested. for check in checks: f.addStep(LitTestCommand(name="test-%s%s" % (step_name, check), command=['ninja', check], description=[ "Test", "just", "built", "components", "for", check, ], env=env, workdir=obj_dir, **kwargs # Pass through all the extra arguments. )) # Install just built components if install_dir: # TODO: Run this step only if none of the prevous failed. f.addStep(NinjaCommand(name="install-%sall" % step_name, targets=["install"], description=["Install", "just", "built", "components"], env=env, workdir=obj_dir, **kwargs # Pass through all the extra arguments. ))
def getCmakeWithMSVCBuildFactory( clean = True, # False for incremental builds. depends_on_projects = None, # List of projects to listen. cmake_cache = None, # Path to a cmake cache file. extra_configure_args = None, # Extra CMake args if any. llvm_srcdir = None, # Source code root directory. obj_dir = None, # Build tree root directory. install_dir = None, # Directory to install the results to. checks = None, # List of checks to test the build. checks_on_target = None, # [(<name>,[<command tokens>])] array of # check name and command to run on target. jobs = None, # Restrict a degree of parallelism. env = None, # Environmental variables for all steps. # VS tools environment variable if using MSVC. # For example, "autodetect" to auto detect, %VS140COMNTOOLS% to select # the VS 2015 toolchain, or empty string if environment is already set. vs=None, **kwargs): if vs is None: # We autodetect Visual Studio, unless otherwise is requested. vs = "autodetect" if install_dir is None: install_dir = 'install' # Prepare environmental variables. Set here all env we want for all steps. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: assert not vs, "Cannot have custom builder env vars with VS setup." # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() if depends_on_projects is None: depends_on_projects = [ 'llvm', 'compiler-rt', 'clang', 'clang-tools-extra', 'libunwind', 'libcxx', 'libcxxabi', 'lld', ] if checks is None: # Check only host-side tools. Target-side tests should run on a target. checks=[ "check-llvm", "check-clang", "check-lld", ] source_remove_requested = lambda step: step.build.getProperty("clean") clean_build_requested = lambda step: \ clean or \ step.build.getProperty("clean", \ default=step.build.getProperty("clean_obj") \ ) f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, cleanBuildRequested=clean_build_requested, **kwargs) # Pass through all the extra arguments. # Remove the source code tree if requested. # NOTE: Somehow RemoveDirectory buildbot command often fails on Windows, # as somthing keeps a lock. We use rm command instead realying on a shell # to support that. f.addStep(ShellCommand(name='clean-%s-dir' % f.monorepo_dir, command=['rm','-rf', f.monorepo_dir], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='Remove the source code', workdir='.', env=merged_env, doStepIf=source_remove_requested)) # Get the source code. f.addGetSourcecodeSteps(**kwargs) # Remove the build directory if requested. f.addStep(ShellCommand(name='clean-%s-dir' % f.obj_dir, command=['rm','-rf', f.obj_dir], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='Remove build directory', workdir='.', env=merged_env, doStepIf=clean_build_requested)) if vs: # Configure MSVC environment if requested. f.addStep(SetProperty( command=builders_util.getVisualStudioEnvironment(vs, None), extract_fn=builders_util.extractSlaveEnvironment)) merged_env = Property('slave_env') # Since this is a build of a cross toolchain, we build only the host-side # tools first by the host system compiler. Libraries will be cross-compiled. cmake_args.append(InterpolateToPosixPath( '-DLLVM_AR=%(workdir)s/' + f.obj_dir + '/bin/llvm-ar.exe')), CmakeCommand.applyDefaultOptions(cmake_args, [ ('-G', 'Ninja'), ('-DLLVM_ENABLE_PROJECTS=', 'llvm;clang;clang-tools-extra;lld'), ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCMAKE_CXX_FLAGS=', '-D__OPTIMIZE__'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_LIT_ARGS=', '-v -vv'), ]) if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeTo( install_dir, f.obj_dir) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel), ]) # Remove the build directory if requested. f.addStep(ShellCommand(name='clean-%s-dir' % install_dir, command=['rm','-rf', install_dir], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='Remove install directory', workdir='.', env=merged_env, doStepIf=clean_build_requested)) src_dir_rel = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, f.obj_dir) # Add given cmake cache at the very end. if cmake_cache: cmake_args.append('-C%s' % cmake_cache) f.addStep(CmakeCommand(name="cmake-configure", haltOnFailure=True, description=["Cmake", "configure"], options=cmake_args, path=src_dir_rel, workdir=f.obj_dir, env=merged_env, **kwargs # Pass through all the extra arguments. )) f.addStep(NinjaCommand(name="build-%s" % f.monorepo_dir, haltOnFailure=True, description=["Build", f.monorepo_dir], workdir=f.obj_dir, env=merged_env, **kwargs # Pass through all the extra arguments. )) # Test the components if requested one check at a time. for check in checks: f.addStep( LitTestCommand( haltOnFailure=False, # We want to test as much as we could. name='test-%s' % check, command=["ninja", WithProperties(check)], description=[ "Testing", "just", "built", "components", "for", check], descriptionDone=[ "Test", "just", "built", "components", "for", check, "completed"], env=merged_env, **kwargs # Pass through all the extra arguments. )) # Run commands on a target if requested. if checks_on_target: for check, cmd in checks_on_target: f.addStep( LitTestCommand( haltOnFailure=False, # We want to test as much as we could. name='test-%s' % check, command=cmd, description=[ "Testing", "just", "built", "components", "for", check], descriptionDone=[ "Test", "just", "built", "components", "for", check, "completed"], env=merged_env, **kwargs # Pass through all the extra arguments. )) # Install just built components if install_dir: f.addStep(NinjaCommand(name="install-all", haltOnFailure=True, targets=["install"], description=[ "Install", "just", "built", "components"], workdir=f.obj_dir, env=merged_env, **kwargs # Pass through all the extra arguments. )) return f
def getToolchainBuildFactory( clean=False, test=True, env=None, # Environmental variables for all steps. extra_configure_args=None): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { "TERM" : "dumb", # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) src_dir = "llvm.src" obj_dir = "llvm.obj" install_dir = "llvm.install" f = LLVMBuildFactory( depends_on_projects=[ "llvm", "clang", "clang-tools-extra", "compiler-rt", "libcxx", "libcxxabi", "libunwind", "lld" ]) # Get Fuchsia SDK. sdk_dir = "fuchsia.sdk" sdk_platform = { "Linux": "linux-amd64", "Darwin": "mac-amd64", }[platform.system()] sdk_version = "latest" sdk_url = WithProperties( "https://chrome-infra-packages.appspot.com/dl/" "fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s", sdk_platform=lambda _: sdk_platform, sdk_version=lambda _: sdk_version) f.addStep(RemoveDirectory(name="clean-sdk", dir=sdk_dir, haltOnFailure=True)) f.addStep(ShellCommand(name="fetch-sdk", command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url], description=["download", "fuchsia sdk"], workdir=sdk_dir)) f.addStep(ShellCommand(name="extract-sdk", command=["unzip", "-fo", "sdk.cipd"], description=["extract", "fuchsia sdk"], workdir=sdk_dir)) cleanCheckoutRequested = lambda step: step.build.getProperty("clean", default=False) or clean # Clean up llvm sources. f.addStep(RemoveDirectory(name="clean-llvm.src", dir=src_dir, haltOnFailure=True, doStepIf=cleanCheckoutRequested)) # Get sources. for project in f.depends_on_projects: _, baseURL = svn_repos[project] f.addStep(SVN(name="svn-%s" % project, workdir=src_dir + "/" + project, baseURL=baseURL)) cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean # Clean up llvm build. f.addStep(RemoveDirectory(name="clean-llvm.obj", dir=obj_dir, haltOnFailure=True, doStepIf=cleanBuildRequested)) # Configure. if extra_configure_args is None: cmake_options = [] else: cmake_options = extra_configure_args[:] # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_options, [ ("-G", "Ninja"), ("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"), ("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"), ]) # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_options, [ ("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"), ("-DLLVM_ENABLE_LTO=", "OFF"), ]) cmake_options.append( WithProperties( "-DCMAKE_INSTALL_PREFIX=%(workdir)s/" + install_dir )) cmake_options.append( WithProperties( "-DFUCHSIA_SDK=%(workdir)s/" + sdk_dir )) CmakeCommand.applyRequiredOptions(cmake_options, [ ("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"), ]) f.addStep(CmakeCommand(name="cmake-configure", options=cmake_options, path='../' + src_dir + '/llvm', haltOnFailure=True, description=["configure"], workdir=obj_dir, env=merged_env, doStepIf=FileDoesNotExist("CMakeCache.txt"))) # Build distribution. f.addStep(NinjaCommand(name="ninja-build", targets=["stage2-distribution"], haltOnFailure=True, description=["build"], workdir=obj_dir, env=merged_env)) # Test llvm, clang and lld. f.addStep(NinjaCommand(name="check", targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")], haltOnFailure=True, description=["check"], workdir=obj_dir, env=merged_env, doStepIf=test)) # Install distribution. f.addStep(NinjaCommand(name="install", targets=["stage2-install-distribution"], haltOnFailure=True, description=["install"], workdir=obj_dir, env=merged_env)) return f
def get3StageClangLTOBuildFactory( clean=True, env=None, build_gold=False, cmake_cache_file=None, extra_cmake_options=None, ): llvm_srcdir = "llvm.src" llvm_objdir = "llvm-stage1.obj" merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: merged_env.update(env) f = BuildFactory() f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir)) # We have to programatically determine the current llvm version. def getClangVer(exit_status, stdout, stderr): # We expect something like this: # release = '3.9' if exit_status: return {} k, v = stdout.split('=') clang_ver = v.strip().strip("\'\"") return {'clang_ver': clang_ver} f.addStep( SetProperty( name="get_clang_ver", command=["grep", "release =", "./tools/clang/docs/conf.py"], extract_fn=getClangVer, description="get clang release ver", workdir=llvm_srcdir, env=merged_env)) # Clean directory, if requested. cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( ShellCommand(doStepIf=cleanBuildRequested, name="rm-llvm_objdir", command=["rm", "-rf", llvm_objdir], haltOnFailure=True, description=["rm build dir", "llvm"], workdir=".", env=merged_env)) cmake_command = ["cmake"] if cmake_cache_file: cmake_command += ['-C', cmake_cache_file] if extra_cmake_options: cmake_command += extra_cmake_options cmake_command += ["../%s" % llvm_srcdir] # Note: ShellCommand does not pass the params with special symbols right. # The " ".join is a workaround for this bug. f.addStep( WarningCountingShellCommand(name="cmake-configure", description=["cmake configure"], haltOnFailure=True, command=WithProperties( " ".join(cmake_command)), workdir=llvm_objdir, env=merged_env)) if build_gold: f.addStep( NinjaCommand(name='build-LLVMgold.so', targets=['lib/LLVMgold.so'], haltOnFailure=True, warnOnWarnings=True, description=["3 Stage Build Clang"], workdir=llvm_objdir, env=merged_env)) f.addStep( NinjaCommand( name='build-stage3-clang', targets=['stage3-clang'], haltOnFailure=True, warnOnWarnings=True, description=["3 Stage Build Clang"], timeout=3600, # Default value is not enough. workdir=llvm_objdir, env=merged_env)) f.addStep( NinjaCommand(name='build-stage3-check-clang', targets=['stage3-check-clang'], haltOnFailure=True, warnOnWarnings=True, description=["Check Clang"], workdir=llvm_objdir, env=merged_env)) # Compare stage2 & stage3 clang shell_command = [ "diff", "-q", "tools/clang/stage2-bins/bin/clang-%(clang_ver)s", "tools/clang/stage2-bins/tools/clang/stage3-bins/bin/clang-%(clang_ver)s" ] f.addStep( ShellCommand(name="compare-clang", description=["comapre stage2 & stage3 clang"], haltOnFailure=True, command=WithProperties(" ".join(shell_command)), workdir=llvm_objdir, env=merged_env)) return f
def getSphinxDocsBuildFactory( llvm_html=False, # Build LLVM HTML documentation llvm_man=False, # Build LLVM man pages clang_html=False, # Build Clang HTML documentation clang_tools_html=False, # Build Clang Extra Tools HTML documentation lld_html=False, # Build LLD HTML documentation libcxx_html=False, # Build Libc++ HTML documentation libunwind_html=False, # Build libunwind HTML documentation lldb_html=False, # Build LLDB HTML documentation extra_configure_args=None, **kwargs): if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults for the config flags. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-G', 'Ninja'), ('-DLLVM_ENABLE_SPHINX=', 'ON'), ('-DSPHINX_OUTPUT_HTML=', 'ON'), ('-DSPHINX_OUTPUT_MAN=', 'ON'), ('-DLLDB_INCLUDE_TESTS=', 'OFF'), ('-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), ]) llvm_srcdir = 'llvm/src' llvm_objdir = 'llvm/build' depends_on_projects = ['llvm'] if clang_html or clang_tools_html or lldb_html: depends_on_projects.append('clang') if clang_tools_html: depends_on_projects.append('clang-tools-extra') if lld_html: depends_on_projects.append('lld') if lldb_html: depends_on_projects.append('lldb') if libcxx_html: depends_on_projects.append('libcxx') depends_on_projects.append('libcxxabi') if libunwind_html: depends_on_projects.append('libunwind') f = UnifiedTreeBuilder.getCmakeBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=llvm_objdir, extra_configure_args=cmake_args, **kwargs) # Pass through all the extra arguments. if llvm_html: f.addStep( NinjaCommand(name="docs-llvm-html", haltOnFailure=True, description=["Build LLVM Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-llvm-html'])) if llvm_man: f.addStep( NinjaCommand(name="docs-llvm-man", haltOnFailure=True, description=["Build LLVM Sphinx man pages"], workdir=llvm_objdir, targets=['docs-llvm-man'])) if clang_html: f.addStep( NinjaCommand(name="docs-clang-html", haltOnFailure=True, description=["Build Clang Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-clang-html'])) if clang_tools_html: f.addStep( NinjaCommand( name="docs-clang-tools-html", haltOnFailure=True, description=[ "Build Clang Extra Tools Sphinx HTML documentation" ], workdir=llvm_objdir, targets=['docs-clang-tools-html'])) if lld_html: f.addStep( NinjaCommand(name="docs-lld-html", haltOnFailure=True, description=["Build LLD Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-lld-html'])) if lldb_html: f.addStep( NinjaCommand(name="docs-lldb-html", haltOnFailure=True, description=["Build LLDB Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-lldb-html'])) if libcxx_html: f.addStep( NinjaCommand( name="docs-libcxx-html", haltOnFailure=True, description=["Build Libc++ Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-libcxx-html'])) if libunwind_html: f.addStep( NinjaCommand( name="docs-libunwind-html", haltOnFailure=True, description=["Build libunwind Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-libunwind-html'])) return f
def _addSteps4SystemCompiler(f, stage_idx=0, clean=True, jobs=None, extra_configure_args=None, env=None): # Index is zero-based, so we want to use a human friendly number instead. stage_num = stage_idx + 1 # Directories to use on this stage. obj_dir = f.stage_objdirs[stage_idx] src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir) install_dir = LLVMBuildFactory.pathRelativeTo( f.stage_installdirs[stage_idx], obj_dir) # This stage could use incremental build. # Clean stage1, only if requested. f.addStep( steps.RemoveDirectory(name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=clean)) f.addStep( steps.RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx], dir=f.stage_installdirs[stage_idx], haltOnFailure=False, flunkOnFailure=False, doStepIf=clean)) # Reconcile the cmake options for this stage. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults. CmakeCommand.applyDefaultOptions( cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), # Do not expect warning free build by the system toolchain. ('-DLLVM_ENABLE_WERROR=', 'OFF'), ]) # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) # Note: On this stage we do not care of warnings, as we build with # a system toolchain and cannot control the environment. # Warnings are likely, and we ignore them. # Create configuration files with cmake f.addStep( CmakeCommand( name="cmake-configure-stage%s" % stage_num, description=["stage%s cmake configure" % stage_num], haltOnFailure=True, flunkOnWarnings=False, options=cmake_args, path=src_dir, env=env, workdir=obj_dir, )) # Build clang by the system compiler f.addStep( NinjaCommand( name="build-stage%s-compiler" % stage_num, jobs=jobs, haltOnFailure=True, flunkOnWarnings=False, description=["build stage%s compiler" % stage_num], env=env, workdir=obj_dir, )) # Test stage1 compiler f.addStep( NinjaCommand( name="test-stage%s-compiler" % stage_num, targets=["check-all"], # or "check-llvm", "check-clang" jobs=jobs, haltOnFailure=True, flunkOnWarnings=False, description=["test stage%s compiler" % stage_num], env=env, workdir=obj_dir, )) # Install stage1 compiler f.addStep( NinjaCommand( name="install-stage%s-compiler" % stage_num, targets=["install"], jobs=jobs, haltOnFailure=True, description=["install stage%s compiler" % stage_num], env=env, workdir=obj_dir, ))
def getSphinxDocsBuildFactory( llvm_html=False, # Build LLVM HTML documentation llvm_man=False, # Build LLVM man pages clang_html=False, # Build Clang HTML documentation clang_tools_html=False, # Build Clang Extra Tools HTML documentation lld_html=False, # Build LLD HTML documentation libcxx_html=False, # Build Libc++ HTML documentation libunwind_html=False, # Build libunwind HTML documentation lldb_html=False # Build LLDB HTML documentation ): f = buildbot.process.factory.BuildFactory() llvm_srcdir = 'llvm/src' llvm_objdir = 'llvm/build' clang_srcdir = llvm_srcdir + '/tools/clang' clang_tools_srcdir = llvm_srcdir + '/tools/clang/tools/extra' lld_srcdir = llvm_srcdir + '/tools/lld' lldb_srcdir = llvm_srcdir + '/tools/lldb' libcxx_srcdir = llvm_srcdir + '/projects/libcxx' libcxxabi_srcdir = llvm_srcdir + '/projects/libcxxabi' libunwind_srcdir = llvm_srcdir + '/projects/libunwind' # Get LLVM. This is essential for all builds # because we build all subprojects in tree f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) if clang_html or clang_tools_html or lldb_html: f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir=clang_srcdir)) if clang_tools_html: f.addStep( SVN(name='svn-clang-tools', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', defaultBranch='trunk', workdir=clang_tools_srcdir)) if lld_html: f.addStep( SVN(name='svn-lld', mode='update', baseURL='http://llvm.org/svn/llvm-project/lld/', defaultBranch='trunk', workdir=lld_srcdir)) if lldb_html: f.addStep( SVN(name='svn-lldb', mode='update', baseURL='http://llvm.org/svn/llvm-project/lldb/', defaultBranch='trunk', workdir=lldb_srcdir)) if libcxx_html: f.addStep( SVN(name='svn-libcxx', mode='update', baseURL='http://llvm.org/svn/llvm-project/libcxx/', defaultBranch='trunk', workdir=libcxx_srcdir)) f.addStep( SVN(name='svn-libcxxabi', mode='update', baseURL='http://llvm.org/svn/llvm-project/libcxxabi/', defaultBranch='trunk', workdir=libcxxabi_srcdir)) if libunwind_html: f.addStep( SVN(name='svn-libunwind', mode='update', baseURL='http://llvm.org/svn/llvm-project/libunwind/', defaultBranch='trunk', workdir=libunwind_srcdir)) # Use CMake to configure cmakeCommand = [ "cmake", WithProperties('%s/' + llvm_srcdir, 'workdir'), '-G', 'Ninja', '-DLLVM_ENABLE_SPHINX:BOOL=ON', '-DSPHINX_OUTPUT_HTML:BOOL=ON', '-DSPHINX_OUTPUT_MAN:BOOL=ON', '-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON', '-DLLDB_INCLUDE_TESTS=OFF', ] f.addStep( ShellCommand(name="cmake-configure", command=cmakeCommand, description=["cmake configure"], workdir=llvm_objdir)) if llvm_html: f.addStep( NinjaCommand(name="docs-llvm-html", haltOnFailure=True, description=["Build LLVM Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-llvm-html'])) if llvm_man: f.addStep( NinjaCommand(name="docs-llvm-man", haltOnFailure=True, description=["Build LLVM Sphinx man pages"], workdir=llvm_objdir, targets=['docs-llvm-man'])) if clang_html: f.addStep( NinjaCommand(name="docs-clang-html", haltOnFailure=True, description=["Build Clang Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-clang-html'])) if clang_tools_html: f.addStep( NinjaCommand( name="docs-clang-tools-html", haltOnFailure=True, description=[ "Build Clang Extra Tools Sphinx HTML documentation" ], workdir=llvm_objdir, targets=['docs-clang-tools-html'])) if lld_html: f.addStep( NinjaCommand(name="docs-lld-html", haltOnFailure=True, description=["Build LLD Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-lld-html'])) if lldb_html: f.addStep( NinjaCommand(name="docs-lldb-html", haltOnFailure=True, description=["Build LLDB Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-lldb-html'])) if libcxx_html: f.addStep( NinjaCommand( name="docs-libcxx-html", haltOnFailure=True, description=["Build Libc++ Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-libcxx-html'])) if libunwind_html: f.addStep( NinjaCommand( name="docs-libunwind-html", haltOnFailure=True, description=["Build libunwind Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-libunwind-html'])) return f
def getSanitizerWindowsBuildFactory( clean=False, cmake='cmake', # Default values for VS devenv and build configuration vs=r"""%VS120COMNTOOLS%""", config='Release', target_arch='x86', extra_cmake_args=[]): ############# PREPARING f = buildbot.process.factory.BuildFactory() # Kill any stale symbolizer processes for the last run. If there are any # stale processes, the build will fail during linking. This can happen to # any process, but it is most likely to happen to llvm-symbolizer if its # pipe isn't closed. taskkill_cmd = 'taskkill /f /im llvm-symbolizer.exe || exit /b 0' f.addStep( ShellCommand(name='taskkill', description='kill stale processes', command=['cmd', '/c', taskkill_cmd], haltOnFailure=False)) # Determine Slave Environment and Set MSVC environment. f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f = getSource(f, 'llvm') # Global configurations build_dir = 'build' ############# CLEANING cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) f.addStep( ShellCommand( name='cmake', command=[ cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config, "-DLLVM_ENABLE_ASSERTIONS=ON" ] + extra_cmake_args, haltOnFailure=True, workdir=build_dir, env=Property('slave_env'))) # Build compiler-rt first to speed up detection of Windows-specific # compiler-time errors in the sanitizer runtimes. f.addStep( NinjaCommand(name='build compiler-rt', targets=['compiler-rt'], haltOnFailure=True, description='ninja compiler-rt', workdir=build_dir, env=Property('slave_env'))) # Build Clang and LLD next so that most compilation errors occur in a build # step. f.addStep( NinjaCommand(name='build clang lld', targets=['clang', 'lld'], haltOnFailure=True, description='ninja clang lld', workdir=build_dir, env=Property('slave_env'))) # Only run sanitizer tests. # Don't build targets that are not required in order to speed up the cycle. for target in [ 'check-asan', 'check-asan-dynamic', 'check-sanitizer', 'check-cfi' ]: f.addStep( NinjaCommand(name='run %s' % target, targets=[target], haltOnFailure=False, description='ninja %s' % target, workdir=build_dir, env=Property('slave_env'))) return f
def getLLGoBuildFactory( clean=True, build_type='Release+Asserts', test_libgo=True, # run 'check-libgo' target if True ): llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" llgo_srcdir = '%s/tools/llgo' % llvm_srcdir clang_srcdir = '%s/tools/clang' % llvm_srcdir f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".")) # Get LLVM, clang and llgo f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir=clang_srcdir)) f.addStep( SVN(name='svn-llgo', mode='update', baseURL='http://llvm.org/svn/llvm-project/llgo/', defaultBranch='trunk', workdir=llgo_srcdir)) # Clean build directory, if requested. f.addStep( ShellCommand(name="rm-llvm_objdir", command=["rm", "-rf", llvm_objdir], haltOnFailure=True, description=["rm build dir", "llvm"], workdir=".", doStepIf=clean)) # Create configuration files with cmake f.addStep( ShellCommand(name="create-build-dir", command=["mkdir", "-p", llvm_objdir], haltOnFailure=False, description=["create build dir"], workdir=".")) cmakeCommand = [ "cmake", "-G", "Ninja", "../%s" % llvm_srcdir, "-DCMAKE_BUILD_TYPE=" + build_type, ] f.addStep( ShellCommand(name="cmake-configure", command=cmakeCommand, haltOnFailure=False, description=["cmake configure"], workdir=llvm_objdir)) # Build llgo f.addStep( NinjaCommand(name="build_llgo", targets=["llgo"], haltOnFailure=True, description=["build llgo"], workdir=llvm_objdir)) # Build libgo f.addStep( NinjaCommand( name="build_libgo", targets=["libgo"], haltOnFailure=True, logfiles={ 'libgo-build-out': 'tools/llgo/libgo-prefix/src/libgo-stamp/libgo-build-out.log', 'libgo-build-err': 'tools/llgo/libgo-prefix/src/libgo-stamp/libgo-build-err.log', 'libgo-configure-out': 'tools/llgo/libgo-prefix/src/libgo-stamp/libgo-configure-out.log', 'libgo-configure-err': 'tools/llgo/libgo-prefix/src/libgo-stamp/libgo-configure-err.log', }, lazylogfiles=True, description=["build libgo"], workdir=llvm_objdir)) # Build llgoi f.addStep( NinjaCommand(name="build_llgoi", targets=["llgoi"], haltOnFailure=True, description=["build llgoi"], workdir=llvm_objdir)) # Test llgo f.addStep( NinjaCommand(name="test_llgo", targets=["check-llgo"], haltOnFailure=True, description=["test llgo"], workdir=llvm_objdir)) # Test libgo f.addStep( NinjaCommand(name="test_libgo", targets=["check-libgo"], haltOnFailure=True, description=["test libgo"], workdir=llvm_objdir)) return f
def addNinjaSteps(f, obj_dir=None, targets=None, checks=None, install_dir=None, env=None, stage_name=None, **kwargs): # Build the unified tree. if stage_name: step_name = "%s-" % stage_name else: stage_name = "" step_name = "" if obj_dir is None: obj_dir = f.obj_dir f.addStep( NinjaCommand( name="build-%sunified-tree" % step_name, haltOnFailure=True, targets=targets, description=["Build", stage_name, "unified", "tree"], env=env or {}, workdir=obj_dir, **kwargs # Pass through all the extra arguments. )) # Test just built components if requested. # Note: At this point env could be None, a dictionary, or a Property object. if isinstance(env, dict): check_env = env.copy() if env else dict() check_env['NINJA_STATUS'] = check_env.get('NINJA_STATUS', "%e [%u/%r/%f] ") else: check_env = env or {} for check in checks: f.addStep( LitTestCommand( name="test-%s%s" % (step_name, check), command=['ninja', check], description=[ "Test", "just", "built", "components", "for", check, ], env=check_env, workdir=obj_dir, **kwargs # Pass through all the extra arguments. )) # Install just built components if install_dir: # TODO: Run this step only if none of the prevous failed. f.addStep( NinjaCommand( name="install-%sall" % step_name, targets=["install"], description=["Install", "just", "built", "components"], env=env or {}, workdir=obj_dir, **kwargs # Pass through all the extra arguments. ))
def getSanitizerBuildFactoryII( clean=False, sanity_check=True, sanitizers=[ 'sanitizer', 'asan', 'lsan', 'msan', 'tsan', 'ubsan', 'dfsan' ], common_cmake_options=None, # FIXME: For backward compatibility. Will be removed. extra_configure_args=[], prefixCommand=["nice", "-n", "10"], # For backward compatibility. env=None, jobs="%(jobs)s", timeout=1200): llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb', # Make sure Clang doesn't use color escape sequences. } if env: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) f = BuildFactory() # Clean directory, if requested. cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( RemoveDirectory(name='clean ' + llvm_objdir, dir=llvm_objdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) # Get llvm, clang, ompiler-rt, libcxx, libcxxabi, libunwind f.addStep( SVN(name='svn-llvm', mode='update', description='svn-llvm', descriptionDone='svn-llvm', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) f.addStep( SVN(name='svn-clang', mode='update', description='svn-clang', descriptionDone='svn-clang', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir)) f.addStep( SVN(name='svn-compiler-rt', mode='update', description='svn-compiler-rt', descriptionDone='svn--compiler-rt', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', defaultBranch='trunk', workdir='%s/projects/compiler-rt' % llvm_srcdir)) f.addStep( SVN(name='svn-libcxx', mode='update', description='svn-libcxx', descriptionDone='svn-libcxx', baseURL='http://llvm.org/svn/llvm-project/libcxx/', defaultBranch='trunk', workdir='%s/projects/libcxx' % llvm_srcdir)) f.addStep( SVN(name='svn-libcxxabi', mode='update', description='svn-libcxxabi', descriptionDone='svn-libcxxabi', baseURL='http://llvm.org/svn/llvm-project/libcxxabi/', defaultBranch='trunk', workdir='%s/projects/libcxxabi' % llvm_srcdir)) f.addStep( SVN(name='svn-libunwind', mode='update', description='svn-libunwind', descriptionDone='svn-libunwind', baseURL='http://llvm.org/svn/llvm-project/libunwind/', defaultBranch='trunk', workdir='%s/projects/libunwind' % llvm_srcdir)) # Run annotated command for sanitizer. if sanity_check: f.addStep( AnnotatedCommand( name="lint check", description="lint check", timeout=timeout, haltOnFailure=False, #True, warnOnWarnings=True, command=["./check_lint.sh"], workdir="%s/projects/compiler-rt/lib/sanitizer_common/scripts" % llvm_srcdir, env=merged_env)) # Always build with ninja. cmakeCommand = ["cmake", "-G", "Ninja"] # Reconsile configure args with the defaults we want. if not any( a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args): cmakeCommand.append('-DCMAKE_BUILD_TYPE=Release') if not any( a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args): cmakeCommand.append('-DLLVM_ENABLE_WERROR=OFF') if not any( a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args): cmakeCommand.append('-DLLVM_ENABLE_ASSERTIONS=ON') if not any( a.startswith('-DCMAKE_C_COMPILER') for a in extra_configure_args): cmakeCommand.append('-DCMAKE_C_COMPILER=clang') if not any( a.startswith('-DCMAKE_CXX_COMPILER') for a in extra_configure_args): cmakeCommand.append('-DCMAKE_CXX_COMPILER=clang++') if not any( a.startswith('-DCMAKE_CXX_FLAGS') for a in extra_configure_args): cmakeCommand.append('-DCMAKE_CXX_FLAGS=\"-std=c++11 -stdlib=libc++\"') if not any( a.startswith('-DCMAKE_EXE_LINKER_FLAGS') for a in extra_configure_args): cmakeCommand.append('-DCMAKE_EXE_LINKER_FLAGS=-lcxxrt') if not any( a.startswith('-DLIBCXXABI_USE_LLVM_UNWINDER=') for a in extra_configure_args): cmakeCommand.append('-DLIBCXXABI_USE_LLVM_UNWINDER=ON') if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args): cmakeCommand.append('-DLLVM_LIT_ARGS=\"-v\"') cmakeCommand += extra_configure_args + ["../%s" % llvm_srcdir] # Note: ShellCommand does not pass the params with special symbols right. # The " ".join is a workaround for this bug. f.addStep( ShellCommand( name="cmake-configure", description=["cmake configure"], haltOnFailure=False, #True, warnOnWarnings=True, command=WithProperties(" ".join(cmakeCommand)), env=merged_env, workdir=llvm_objdir, doStepIf=FileDoesNotExist("./%s/CMakeCache.txt" % llvm_objdir))) # Build everything. f.addStep( NinjaCommand( name='build', haltOnFailure=False, #True, warnOnWarnings=True, description=['building', 'with', 'ninja'], descriptionDone=['built', 'with', 'ninja'], workdir=llvm_objdir, env=merged_env)) # Run tests for each of the requested sanitizers. if sanitizers: for s in sanitizers: f.addStep( NinjaCommand( name='test %s' % s, targets=['check-%s' % s], haltOnFailure=False, #True, description=['testing', '%s' % s], descriptionDone=['test', '%s' % s], workdir=llvm_objdir, env=merged_env)) return f
def getLLDWinBuildFactory( clean = True, # Default values for VS devenv and build configuration vs = None, # What to run to configure Visual Studio utils. target_arch = None, # Native. extra_configure_args = None, env = None): # Set defaults if vs is None: vs = r"""%VS140COMNTOOLS%""" # Visual Studio 2015. if extra_configure_args is None: extra_configure_args = [] if env is None: env = {} f = LLVMBuildFactory( depends_on_projects=['llvm', 'lld'], llvm_srcdir="llvm.src", llvm_objdir="llvm.obj") # Get LLVM and Lld f.addSVNSteps() # Clean directory, if requested. cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir, dir=f.llvm_objdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested )) # If set up environment step is requested, do this now. if vs: f.addStep(SetProperty( command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) assert not env, "Can't have custom builder env vars with VS" env = Property('slave_env') # Always build with ninja. cmake_options = ["-G", "Ninja"] # Reconsile configure args with the defaults we want. if not any(a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args): cmake_options.append('-DCMAKE_BUILD_TYPE=Release') if not any(a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args): cmake_options.append('-DLLVM_ENABLE_WERROR=ON') if not any(a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args): cmake_options.append('-DLLVM_ENABLE_ASSERTIONS=ON') if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args): cmake_options.append('-DLLVM_LIT_ARGS=\"-v\"') cmake_options += extra_configure_args # Note: ShellCommand does not pass the params with special symbols right. # The " ".join is a workaround for this bug. f.addStep(CmakeCommand(name="cmake-configure", description=["cmake configure"], haltOnFailure=True, warnOnWarnings=True, options=cmake_options, path="../%s" % f.llvm_srcdir, env=env, workdir=f.llvm_objdir, doStepIf=FileDoesNotExist( "./%s/CMakeCache.txt" % f.llvm_objdir))) # Build Lld. f.addStep(NinjaCommand(name='build lld', haltOnFailure=True, warnOnWarnings=True, description='build lld', workdir=f.llvm_objdir, env=env)) # Test Lld f.addStep(NinjaCommand(name='test lld', targets=['lld-test'], haltOnFailure=True, warnOnWarnings=True, description='test lld', workdir=f.llvm_objdir, env=env)) return f
def getABITestsuitBuildFactory( clean = True, depends_on_projects = None, extra_configure_args = None, # Extra CMake args for all stages. jobs = None, # Restrict a degree of parallelism if needed. env = None, # Environmental variables for all steps. **kwargs): # Prepare environmental variables. Set here all env we want for all steps. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) if depends_on_projects: depends_on_projects = list(depends_on_projects) else: depends_on_projects = ['llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld'] if extra_configure_args is None: cmake_args = list() else: cmake_args = list(extra_configure_args) # Some options are required for this build no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ]) f = UnifiedTreeBuilder.getCmakeBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir="llvm", obj_dir="build", clean=clean, extra_configure_args=cmake_args, env=merged_env, **kwargs) # Pass through all the extra arguments. f.addStep(NinjaCommand(name="build-unified-tree", haltOnFailure=True, description=["Build", "unified", "tree"], env=merged_env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) # Checkout the test-suite. f.addStep(SVN(name='svn-test-suite', mode='update', baseURL='http://llvm.org/svn/llvm-project/test-suite/', defaultBranch='trunk', workdir='test-suite')) # Run the ABI test. abi_test_env = { 'PYTHONPATH' : WithProperties("%(workdir)s/" + f.llvm_srcdir + "/utils/lit:${PYTHONPATH}"), 'PATH' : WithProperties("%(workdir)s/" + f.obj_dir + "/bin:${PATH}"), } merged_env.update(abi_test_env) abi_test_cmd = ["python", "linux-x86.py", "clang", "test", "-v"] if jobs: abi_test_cmd.append("-j" + str(jobs)) f.addStep(LitTestCommand(name='abi-test-suite', command=abi_test_cmd, description=["running", "ABI", "test-suite"], descriptionDone=["ABI", "test-suite", "completed"], workdir='test-suite/ABI-Testsuite', env=merged_env)) return f
def getLibompCMakeBuildFactory(clean=True, env=None, ompt=False, test=True, c_compiler="gcc", cxx_compiler="g++"): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) openmp_srcdir = "openmp.src" openmp_builddir = "openmp.build" llvm_srcdir = "llvm.src" llvm_builddir = "llvm.build" f = buildbot.process.factory.BuildFactory() # Get libomp f.addStep( SVN(name='svn-libomp', mode='update', baseURL='http://llvm.org/svn/llvm-project/openmp/', defaultBranch='trunk', workdir=openmp_srcdir)) # Get llvm to build llvm-lit f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) # Get clang if c/c++ compilers is clang/clang++ if c_compiler == "clang": f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir)) # Clean directory, if requested. if clean: f.addStep( ShellCommand(name="clean", command=["rm", "-rf", openmp_builddir, llvm_builddir], warnOnFailure=True, description=["clean"], workdir='.', env=merged_env)) # CMake llvm f.addStep( ShellCommand(name='cmake llvm', command=[ "cmake", "../" + llvm_srcdir, "-G", "Ninja", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_ENABLE_ASSERTIONS=ON", "-DCMAKE_C_COMPILER=" + c_compiler, "-DCMAKE_CXX_COMPILER=" + cxx_compiler ], haltOnFailure=True, description='cmake llvm', workdir=llvm_builddir, env=merged_env)) # Make clang build or just llvm utils build if c_compiler == "clang": f.addStep( NinjaCommand(name="make clang build", haltOnFailure=True, description=["make clang build"], workdir=llvm_builddir, env=merged_env)) else: f.addStep( NinjaCommand(name="make llvm utils build", targets=["LLVMX86Utils"], haltOnFailure=True, description=["make llvm utils build"], workdir=llvm_builddir, env=merged_env)) if ompt: f.addStep( NinjaCommand(name="make ompt test utils", targets=["FileCheck"], haltOnFailure=True, description=["make ompt test utils"], workdir=llvm_builddir, env=merged_env)) # Add clang/llvm-lit to PATH merged_env.update({ 'PATH': WithProperties("%(workdir)s/" + llvm_builddir + "/bin:" + "${PATH}") }) # CMake libomp command = [ "cmake", "../" + openmp_srcdir, "-DCMAKE_C_COMPILER=" + c_compiler, "-DCMAKE_CXX_COMPILER=" + cxx_compiler ] if ompt: command.append("-DLIBOMP_OMPT_SUPPORT=ON") f.addStep( ShellCommand(name='cmake libomp', command=command, haltOnFailure=True, description='cmake libomp', workdir=openmp_builddir, env=merged_env)) # Make libomp f.addStep( WarningCountingShellCommand(name='make libomp build', command=['make'], haltOnFailure=True, description='make libomp build', workdir=openmp_builddir, env=merged_env)) # Test, if requested if test: f.addStep( LitTestCommand(name="make check-libomp", command=["make", "check-libomp"], haltOnFailure=True, description=["make check-libomp"], descriptionDone=["build checked"], workdir=openmp_builddir, env=merged_env)) return f
def get3StageClangLTOBuildFactory( clean=True, env=None, build_gold=False, cmake_cache_file=None, extra_cmake_options=None, ): llvm_srcdir = "llvm.src" llvm_objdir = "llvm-stage1.obj" merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: merged_env.update(env) f = BuildFactory() f.addStep( SVN( name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir ) ) f.addStep( SVN( name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir ) ) # Clean directory, if requested. cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( ShellCommand( doStepIf=cleanBuildRequested, name="rm-llvm_objdir", command=["rm", "-rf", llvm_objdir], haltOnFailure=True, description=["rm build dir", "llvm"], workdir=".", env=merged_env ) ) cmake_command = ["cmake"] if cmake_cache_file: cmake_command += ['-C', cmake_cache_file] if extra_cmake_options: cmake_command += extra_cmake_options cmake_command += [ "../%s" % llvm_srcdir ] # Note: ShellCommand does not pass the params with special symbols right. # The " ".join is a workaround for this bug. f.addStep( WarningCountingShellCommand( name="cmake-configure", description=["cmake configure"], haltOnFailure=True, command=WithProperties(" ".join(cmake_command)), workdir=llvm_objdir, env=merged_env ) ) if build_gold: f.addStep( NinjaCommand(name='build-LLVMgold.so', targets=['lib/LLVMgold.so'], haltOnFailure=True, warnOnWarnings=True, description=["3 Stage Build Clang"], workdir=llvm_objdir, env=merged_env) ) f.addStep( NinjaCommand(name='build-stage3-clang', targets=['stage3-clang'], haltOnFailure=True, warnOnWarnings=True, description=["3 Stage Build Clang"], timeout=3600, # Default value is not enough. workdir=llvm_objdir, env=merged_env) ) f.addStep( NinjaCommand(name='build-stage3-check-clang', targets=['stage3-check-clang'], haltOnFailure=True, warnOnWarnings=True, description=["Check Clang"], timeout=3600, # Default value is not enough. workdir=llvm_objdir, env=merged_env) ) # Compare stage2 & stage3 clang shell_command = [ "diff", "-q", "tools/clang/stage2-bins/bin/clang", "tools/clang/stage2-bins/tools/clang/stage3-bins/bin/clang" ] f.addStep( ShellCommand( name="compare-clang", description=["comapre stage2 & stage3 clang"], haltOnFailure=True, command=WithProperties(" ".join(shell_command)), workdir=llvm_objdir, env=merged_env ) ) return f
def getClangAndLLDBuildFactory( clean=True, env=None, withLLD=True, extraCmakeOptions=None, extraCompilerOptions=None, buildWithSanitizerOptions=None, triple=None, isMSVC=False, prefixCommand=["nice", "-n", "10"], # For backward compatibility. extraLitArgs=None): llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( buildbot.steps.shell.SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".", env=merged_env)) # Get LLVM, Clang and LLD. f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) # Sanitizer runtime in compiler-rt, and cannot be built with sanitizer compiler. if buildWithSanitizerOptions is None: f.addStep( SVN(name='svn-compiler-rt', mode='update', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', defaultBranch='trunk', workdir='%s/projects/compiler-rt' % llvm_srcdir)) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir)) f.addStep( SVN(name='svn-clang-tools-extra', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', defaultBranch='trunk', workdir='%s/tools/clang/tools/extra' % llvm_srcdir)) if withLLD: f.addStep( SVN(name='svn-lld', mode='update', baseURL='http://llvm.org/svn/llvm-project/lld/', defaultBranch='trunk', workdir='%s/tools/lld' % llvm_srcdir)) # Clean directory, if requested. if clean: shellCommand = ["rm", "-rf", llvm_objdir] if isMSVC: shellCommand = ["rmdir", "/S", "/Q", llvm_objdir] f.addStep( ShellCommand(name="rm-llvm_objdir", command=shellCommand, haltOnFailure=False, flunkOnFailure=False, description=["rm build dir", "llvm"], workdir=".", env=merged_env)) # Create configuration files with cmake. options = ["-Wdocumentation", "-Wno-documentation-deprecated-sync"] if isMSVC: options = [] if extraCompilerOptions: options += extraCompilerOptions if buildWithSanitizerOptions: options += buildWithSanitizerOptions lit_args = ["-v"] if extraLitArgs: lit_args += extraLitArgs lit_args = ["-DLLVM_LIT_ARGS=\"%s\"" % " ".join(lit_args)] cmakeCommand = [ "cmake", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_ENABLE_ASSERTIONS=ON" ] if buildWithSanitizerOptions: cmakeCommand += [ "-DCMAKE_C_COMPILER=clang", "-DCMAKE_CXX_COMPILER=clang++" ] if triple: cmakeCommand += ["-DLLVM_DEFAULT_TARGET_TRIPLE=%s" % triple] if extraCmakeOptions: assert not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extraCmakeOptions), \ "Please use extraLitArgs for LIT arguments instead of defining them in extraCmakeOptions." cmakeCommand += extraCmakeOptions if not isMSVC: cmakeCommand += [ "-DCMAKE_C_FLAGS=\"%s\"" % (" ".join(options)), "-DCMAKE_CXX_FLAGS=\"-std=c++11 %s\"" % (" ".join(options)), ] cmakeCommand += lit_args cmakeCommand += ["-GNinja", "../%s" % llvm_srcdir] # Note: ShellCommand does not pass the params with special symbols right. # The " ".join is a workaround for this bug. f.addStep( ShellCommand(name="cmake-configure", description=["cmake configure"], haltOnFailure=True, command=WithProperties(" ".join(cmakeCommand)), workdir=llvm_objdir, env=merged_env)) # Build everything. f.addStep( NinjaCommand(name="build", prefixCommand=prefixCommand, haltOnFailure=True, description=["build"], workdir=llvm_objdir, env=merged_env)) # Test everything. f.addStep( NinjaCommand(name="test", prefixCommand=prefixCommand, targets=["check-all"], haltOnFailure=True, description=["test"], workdir=llvm_objdir, env=merged_env)) return f
def getOpenMPCMakeBuildFactory( c_compiler='gcc', # The C compiler to use. cxx_compiler='g++', # The C++ compiler to use. jobs='%(jobs)s', # Number of concurrent jobs. clean=True, # "clean" step is requested if true env=None, # Environmental variables for all steps. ompt=False, # Whether to enable the OpenMP Tools Interface. test=True, # Test the built libraries. ): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } # Overwrite pre-set items with the given ones, so user can set anything. if env is not None: merged_env.update(env) openmp_srcdir = 'openmp.src' openmp_builddir = 'openmp.build' llvm_srcdir = 'llvm.src' llvm_builddir = 'llvm.build' build_clang = c_compiler == 'clang' f = buildbot.process.factory.BuildFactory() # Checkout sources. f.addStep( SVN(name='svn-openmp', mode='update', baseURL='http://llvm.org/svn/llvm-project/openmp/', defaultBranch='trunk', workdir=openmp_srcdir)) # LLVM sources are needed to build utilities for testing. f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) # Get Clang sources, if requested. if build_clang: f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir)) cleanBuildRequested = lambda step: clean or step.build.getProperty("clean") f.addStep( ShellCommand(name='clean', command=['rm', '-rf', openmp_builddir, llvm_builddir], warnOnFailure=True, description=['clean'], doStepIf=cleanBuildRequested, workdir='.', env=merged_env)) # Configure LLVM (and Clang, if requested). cmake_args = ['cmake', '-G', 'Ninja', '../%s' % llvm_srcdir] cmake_args += ['-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_ASSERTIONS=ON'] cmake_args += ['-DCMAKE_C_COMPILER=%s' % c_compiler] cmake_args += ['-DCMAKE_CXX_COMPILER=%s' % cxx_compiler] f.addStep( Configure(name='configure-llvm', command=cmake_args, description='configure llvm', workdir=llvm_builddir, env=merged_env)) # Build LLVM utils. f.addStep( NinjaCommand(name='compile-llvm-utils', targets=['FileCheck'], description='compile llvm utils', workdir=llvm_builddir, env=merged_env)) # Build Clang if requested. if build_clang: f.addStep( NinjaCommand(name='compile-clang', description='compile clang', workdir=llvm_builddir, env=merged_env)) # Add llvm-lit and clang (if built) to PATH merged_env.update({ 'PATH': WithProperties('%(workdir)s/' + llvm_builddir + '/bin:${PATH}') }) # Configure OpenMP runtime libraries. cmake_args = ['cmake', '-G', 'Ninja', '../%s' % openmp_srcdir] cmake_args += ['-DCMAKE_C_COMPILER=%s' % c_compiler] cmake_args += ['-DCMAKE_CXX_COMPILER=%s' % cxx_compiler] if ompt: cmake_args += ['-DLIBOMP_OMPT_SUPPORT=ON'] if test: lit_args = '-v --show-unsupported --show-xfail -j %s' % jobs cmake_args += [WithProperties('-DOPENMP_LIT_ARGS=%s' % lit_args)] f.addStep( Configure(name='configure-openmp', command=cmake_args, description='configure openmp', workdir=openmp_builddir, env=merged_env)) # Build OpenMP runtime libraries. f.addStep( NinjaCommand(name='compile-openmp', description='compile openmp', workdir=openmp_builddir, env=merged_env)) # Test OpenMP runtime libraries, if requested. if test: ninja_test_args = ['ninja', WithProperties('-j %s' % jobs)] f.addStep( LitTestCommand(name='test-openmp', command=ninja_test_args + ['check-openmp'], description='test openmp', workdir=openmp_builddir, env=merged_env)) return f
def getCmakeWithNinjaBuildFactory(depends_on_projects=None, llvm_srcdir=None, obj_dir=None, install_dir=None, clean=False, extra_configure_args=None, env=None, **kwargs): # Reconcile the cmake options for this build. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # To set proper defaults, uncomment and modify the following lines. #CmakeCommand.applyDefaultOptions(cmake_args, [ # ('-DCMAKE_BUILD_TYPE=', 'Release'), # ]) # Some options are required for this build no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ]) f = getCmakeBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir or "install", extra_configure_args=cmake_args, env=env, **kwargs # Pass through all the extra arguments. ) # Directories to use for this build. # obj_dir = f.obj_dir # src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) # install_dir = LLVMBuildFactory.pathRelativeToBuild(f.install_dir, obj_dir) # Build the unified tree. f.addStep( NinjaCommand( name="build-unified-tree", haltOnFailure=True, description=["build unified tree"], env=env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) # Test just built components f.addStep( NinjaCommand( name="test-check-all", targets=["check-all"], haltOnFailure=True, description=["test just built components"], env=env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) # Install just built components f.addStep( NinjaCommand( name="install-all", targets=["install"], haltOnFailure=True, description=["install just built components"], env=env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) return f
def getOpenMPCMakeBuildFactory( jobs = '%(jobs)s', # Number of concurrent jobs. clean = True, # "clean" step is requested if true env = None, # Environmental variables for all steps. ompt = False, # Whether to enable the OpenMP Tools Interface. test = True, # Test the built libraries. depends_on_projects = None, **kwargs): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } # Overwrite pre-set items with the given ones, so user can set anything. if env is not None: merged_env.update(env) llvm_srcdir = 'llvm.src' llvm_builddir = 'llvm.build' cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) if depends_on_projects is None: # Monorepo configuration requires llvm and clang to get cmake work. depends_on_projects = ['llvm', 'clang', 'openmp'] f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=llvm_builddir, cleanBuildRequested=cleanBuildRequested, env=merged_env, **kwargs) # Pass through all the extra arguments. f.addStep( ShellCommand( name = 'clean', command = ['rm', '-rf', f.obj_dir], warnOnFailure = True, description = ['clean'], doStepIf = cleanBuildRequested, workdir = '.', env = merged_env)) # Configure LLVM and OpenMP (and Clang, if requested). cmake_args = ['cmake', '-G', 'Ninja'] cmake_args += ['-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_ASSERTIONS=ON'] if ompt: cmake_args += ['-DLIBOMP_OMPT_SUPPORT=ON'] if test: lit_args = '-vv --show-unsupported --show-xfail -j %s' % jobs cmake_args += [WithProperties('-DLLVM_LIT_ARGS=%s' % lit_args)] CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) # Add llvm-lit and clang (if built) to PATH merged_env.update({ 'PATH': WithProperties('%(workdir)s/' + llvm_builddir + '/bin:${PATH}')}) src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, f.obj_dir) f.addStep(CmakeCommand(name='configure-openmp', description=['configure','openmp'], options=cmake_args, path=src_dir, env=merged_env, workdir=f.obj_dir, haltOnFailure=True, **kwargs # Pass through all the extra arguments. )) # Build OpenMP runtime libraries. f.addStep( NinjaCommand( name = 'compile-openmp', description = 'compile openmp', workdir = f.obj_dir, env = merged_env, haltOnFailure=True)) # Test OpenMP runtime libraries, if requested. if test: # Add llvm-lit and clang (if built) to PATH merged_env.update({ 'PATH': WithProperties('%(workdir)s/' + llvm_builddir + '/bin:${PATH}')}) ninja_test_args = ['ninja', WithProperties('-j %s' % jobs)] f.addStep( LitTestCommand( name = 'test-openmp', command = ninja_test_args + ['check-openmp'], description = 'test openmp', workdir = f.obj_dir, env = merged_env, haltOnFailure=True)) return f
def getFactory( depends_on_projects = None, targets = None, checks = None, clean = False, extra_configure_args = None, llvm_srcdir = None, obj_dir = None, env = None, **kwargs): # Prepare environmental variables. Set here all env we want for all steps. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) if depends_on_projects: depends_on_projects = list(depends_on_projects) else: depends_on_projects = ['llvm', 'lld'] if checks is None: checks = [] # No check unless requested specifically. if extra_configure_args is None: cmake_args = list() else: cmake_args = list(extra_configure_args) # Some options are required for this build no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DLLVM_OPTIMIZED_TABLEGEN=', 'OFF'), ('-DLLVM_BUILD_STATIC=', 'ON'), ('-DLLVM_ENABLE_PIC=', 'OFF'), ]) # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-G', 'Ninja'), ('-DLLVM_LIT_ARGS=', '-v -vv'), ]) f = UnifiedTreeBuilder.getCmakeBuildFactory( depends_on_projects=depends_on_projects, clean=clean, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, extra_configure_args=cmake_args, env=merged_env, **kwargs) # Pass through all the extra arguments. if targets: step_name = "build-{}".format("-".join(targets)) step_description=["Build"] step_description.extend(targets) else: step_name = "build-unified-tree" step_description=["Build", "unified", "tree"] f.addStep(NinjaCommand(name=step_name, targets=targets, description=step_description, haltOnFailure=kwargs.get('haltOnFailure', True), env=merged_env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) # Test just built components if requested. if checks: f.addStep(NinjaCommand(name="test-{}".format("-".join(checks)), targets=checks, description=[ "Test", "just", "built", "components"], haltOnFailure=kwargs.get('haltOnFailure', True), env=merged_env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) # Copy just built LLD executable to the test suite directory # to avoid load from a hard drive overhead. f.addStep( ShellCommand( name="copy-lld-to-test-suite", description=[ "Copy", "LLD", "executable", "to", "the", "performance", "test", "suite", ], command=[ "cp", "-aL", "./{}/bin/ld.lld".format(f.obj_dir), "./lld-speed-test/ld.lld" ], env=merged_env, workdir='.' ) ) # Run the performance test suite. perf_command = [ "python", "%(builddir)s/lld-benchmark.py", "--machine=%(workername)s", "--revision=%(got_revision)s", "--linker=./ld.lld", ".", ] f.addStep( ShellCommand( name="performance-test-suite", description=[ "LLD", "performance","test","suite", ], command=WithProperties(" ".join(perf_command)), workdir="./lld-speed-test", env=merged_env ) ) return f
def getSphinxDocsBuildFactory( llvm_html=False, # Build LLVM HTML documentation llvm_man=False, # Build LLVM man pages clang_html=False, # Build Clang HTML documentation lld_html=False, # Build LLD HTML documentation libcxx_html=False # Build Libc++ HTML documentation ): f = buildbot.process.factory.BuildFactory() llvm_srcdir = 'llvm/src' llvm_objdir = 'llvm/build' clang_srcdir = llvm_srcdir + '/tools/clang' lld_srcdir = llvm_srcdir + '/tools/lld' libcxx_srcdir = llvm_srcdir + '/projects/libcxx' libcxxabi_srcdir = llvm_srcdir + '/projects/libcxxabi' # Get LLVM. This is essential for all builds # because we build all subprojects in tree f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) if clang_html: f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir=clang_srcdir)) if lld_html: f.addStep( SVN(name='svn-lld', mode='update', baseURL='http://llvm.org/svn/llvm-project/lld/', defaultBranch='trunk', workdir=lld_srcdir)) if libcxx_html: f.addStep( SVN(name='svn-libcxx', mode='update', baseURL='http://llvm.org/svn/llvm-project/libcxx/', defaultBranch='trunk', workdir=libcxx_srcdir)) f.addStep( SVN(name='svn-libcxxabi', mode='update', baseURL='http://llvm.org/svn/llvm-project/libcxxabi/', defaultBranch='trunk', workdir=libcxxabi_srcdir)) f.addStep( ShellCommand( name="create-build-dir", command=["mkdir", "-p", llvm_objdir], haltOnFailure= False, # We might of already created the directory in a previous build description=["create build dir"], workdir=".")) # Use CMake to configure cmakeCommand = [ "cmake", WithProperties('%s/' + llvm_srcdir, 'workdir'), '-G', 'Ninja', '-DLLVM_ENABLE_SPHINX:BOOL=ON', '-DSPHINX_OUTPUT_HTML:BOOL=ON', '-DSPHINX_OUTPUT_MAN:BOOL=ON' ] f.addStep( ShellCommand(name="cmake-configure", command=cmakeCommand, description=["cmake configure"], workdir=llvm_objdir)) if llvm_html: f.addStep( NinjaCommand(name="docs-llvm-html", haltOnFailure=True, description=["Build LLVM Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-llvm-html'])) if llvm_man: f.addStep( NinjaCommand(name="docs-llvm-man", haltOnFailure=True, description=["Build LLVM Sphinx man pages"], workdir=llvm_objdir, targets=['docs-llvm-man'])) if clang_html: f.addStep( NinjaCommand(name="docs-clang-html", haltOnFailure=True, description=["Build Clang Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-clang-html'])) if lld_html: f.addStep( NinjaCommand(name="docs-lld-html", haltOnFailure=True, description=["Build LLD Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-lld-html'])) if libcxx_html: f.addStep( NinjaCommand( name="docs-libcxx-html", haltOnFailure=True, description=["Build Libc++ Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-libcxx-html'])) return f
def _addSteps4StagedCompiler(f, stage_idx=1, use_stage_idx=-1, jobs=None, extra_configure_args=None, env=None): if use_stage_idx < 0: use_stage_idx = stage_idx - 1 # Index is zero-based, so we want to use a human friendly number instead. stage_num = stage_idx + 1 # Directories to use on this stage. obj_dir = f.stage_objdirs[stage_idx] src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir) install_dir = LLVMBuildFactory.pathRelativeTo( f.stage_installdirs[stage_idx], obj_dir) staged_install = f.stage_installdirs[use_stage_idx] # Always do a clean build for the staged compiler. f.addStep( steps.RemoveDirectory( name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, )) f.addStep( steps.RemoveDirectory( name='clean-%s-dir' % f.stage_installdirs[stage_idx], dir=f.stage_installdirs[stage_idx], haltOnFailure=False, flunkOnFailure=False, )) # Reconcile the cmake options for this stage. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ]) # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) cmake_args.append( WithProperties("-DCMAKE_CXX_COMPILER=%(builddir)s/" + staged_install + "/bin/clang++")) cmake_args.append( WithProperties("-DCMAKE_C_COMPILER=%(builddir)s/" + staged_install + "/bin/clang")) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) # Create configuration files with cmake f.addStep( CmakeCommand( name="cmake-configure-stage%s" % stage_num, description=["stage%s cmake configure" % stage_num], haltOnFailure=True, options=cmake_args, path=src_dir, env=env, workdir=obj_dir, )) # Build clang by the staged compiler f.addStep( NinjaCommand( name="build-stage%s-compiler" % stage_num, jobs=jobs, haltOnFailure=True, description=["build stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, )) # Test just built compiler f.addStep( NinjaCommand( name="test-stage%s-compiler" % stage_num, targets=["check-all"], jobs=jobs, haltOnFailure=True, description=["test stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, )) # Install just built compiler f.addStep( NinjaCommand( name="install-stage%s-compiler" % stage_num, targets=["install"], jobs=jobs, haltOnFailure=True, description=["install stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, ))
def getABITestsuitBuildFactory( clean=True, depends_on_projects=None, extra_configure_args=None, # Extra CMake args for all stages. jobs=None, # Restrict a degree of parallelism if needed. env=None, # Environmental variables for all steps. **kwargs): # Prepare environmental variables. Set here all env we want for all steps. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) if depends_on_projects: depends_on_projects = list(depends_on_projects) else: depends_on_projects = [ 'llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld' ] if extra_configure_args is None: cmake_args = list() else: cmake_args = list(extra_configure_args) # Some options are required for this build no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ]) cleanBuildRequested = lambda step: step.build.getProperty( "clean", default=step.build.getProperty("clean_obj")) or clean f = UnifiedTreeBuilder.getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects=depends_on_projects, llvm_srcdir="llvm", obj_dir="build", cleanBuildRequested=cleanBuildRequested, env=merged_env, **kwargs) # Pass through all the extra arguments. # First of all, we shall checkout the latest test-suite. f.addGetSourcecodeForProject(project='test-suite', src_dir='test-suite', alwaysUseLatest=True, **kwargs) # Then get the LLVM source code revision this particular build is for. f.addGetSourcecodeSteps(**kwargs) UnifiedTreeBuilder.addCmakeSteps(f, cleanBuildRequested=cleanBuildRequested, obj_dir=f.obj_dir, extra_configure_args=cmake_args, env=env, **kwargs) f.addStep( NinjaCommand( name="build-unified-tree", haltOnFailure=True, description=["Build", "unified", "tree"], env=merged_env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) # Run the ABI test. abi_test_env = { 'PYTHONPATH': WithProperties("%(workdir)s/" + f.llvm_srcdir + "/utils/lit:${PYTHONPATH}"), 'PATH': WithProperties("%(workdir)s/" + f.obj_dir + "/bin:${PATH}"), } merged_env.update(abi_test_env) abi_test_cmd = ["python", "linux-x86.py", "clang", "test", "-v"] if jobs: abi_test_cmd.append("-j" + str(jobs)) f.addStep( LitTestCommand(name='abi-test-suite', command=abi_test_cmd, description=["running", "ABI", "test-suite"], descriptionDone=["ABI", "test-suite", "completed"], workdir='test-suite/ABI-Testsuite', env=merged_env)) return f
def getCUDATestsuiteBuildFactory( externals, # Directory with CUDA, thrust and gcc versions for testing. always_clean=True, test=False, useTwoStage=False, cmake='cmake', extra_cmake_args=None, # Extra CMake args for all stages. extra_ts_cmake_args=None, # extra cmake args for testsuite. jobs=None, cuda_jobs=1, # number of simultaneous CUDA apps to run env=None, # Environmental variables for all steps. enable_thrust_tests=False, split_thrust_tests=False, # Each thrust test is a separate executable. run_thrust_tests=False, enable_libcxx=True, # checkout/build libcxx to test with. gpu_arch_list=None, gpu_devices=None, # List of devices to make visible to CUDA stage1_config='Release', stage2_config='Release'): if extra_cmake_args is None: extra_cmake_args = [] if extra_ts_cmake_args is None: extra_ts_cmake_args = [] # Prepare environmental variables. Set here all env we want for all steps. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set # anything. merged_env.update(env) source_dir = 'llvm' # Should match the one used in getClangCMakeBuildFactory. stage1_build_dir = 'stage1' # Should match the one defined in getClangCMakeBuildFactory. stage2_build_dir = 'stage2' # Should match the one defined in getClangCMakeBuildFactory. install_dir = 'clang.install' if useTwoStage: clang_build_dir = stage2_build_dir else: clang_build_dir = stage1_build_dir # Build clang. f = ClangBuilder.getClangCMakeBuildFactory( clean=always_clean, test=test, cmake=cmake, extra_cmake_args=extra_cmake_args, jobs=jobs, env=merged_env, useTwoStage=useTwoStage, stage1_config=stage1_config, stage2_config=stage2_config, checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_libcxx=enable_libcxx, checkout_lld=False, checkout_test_suite=True) cuda_test_env = { 'PYTHONPATH': WithProperties("%(workdir)s/" + source_dir + "/utils/lit:${PYTHONPATH}"), 'DESTDIR': WithProperties("%(workdir)s/" + install_dir), 'PATH': WithProperties("%(workdir)s/" + install_dir + "/usr/local/bin:${PATH}"), } merged_env.update(cuda_test_env) ts_build_dir = 'test-suite-build' f.addStep( RemoveDirectory(name="Remove old clang install directory", dir=install_dir)) # Install clang into directory pointed by $DESTDIR f.addStep( NinjaCommand(name='ninja install clang', targets=["install"], jobs=jobs, haltOnFailure=True, description=split("installing clang"), descriptionDone=split("Clang installation is done."), workdir=clang_build_dir, env=merged_env)) # Completely remove test suite build dir. f.addStep( RemoveDirectory(name="Remove old test-suite build directory", dir=ts_build_dir)) if extra_ts_cmake_args: cmake_args = extra_ts_cmake_args[:] else: cmake_args = [] # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ]) # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ]) cmake_args.append( WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" + clang_build_dir + "/bin/clang++")) cmake_args.append( WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + clang_build_dir + "/bin/clang")) cmake_args.append('-DTEST_SUITE_SUBDIRS=External'), # Limit to External tests only. if externals: cmake_args.append('-DTEST_SUITE_EXTERNALS_DIR=' + externals) if split_thrust_tests: cmake_args.append('-DTHRUST_SPLIT_TESTS=1') if gpu_arch_list: cmake_args.append('-DCUDA_GPU_ARCH=' + ';'.join(gpu_arch_list)) if cuda_jobs: cmake_args.append('-DCUDA_JOBS=%s' % cuda_jobs) # Then do fresh cmake configuration. f.addStep( CmakeCommand(name='cmake test-suite', description='cmake test-suite', haltOnFailure=True, options=cmake_args, path="../test/test-suite", workdir=ts_build_dir, env=merged_env)) # Always build simple CUDA tests. They serve as compilation # smoketests and will fail quickly if compiler has obvious issues # compiling CUDA files. f.addStep( NinjaCommand(name='ninja build simple CUDA tests', targets=["cuda-tests-simple"], jobs=jobs, haltOnFailure=True, description=split("building simple CUDA tests"), descriptionDone=split("simple CUDA tests built."), workdir=ts_build_dir, env=merged_env)) # Limit GPUs visible to CUDA. if gpu_devices: for gpu_id in gpu_devices: # make ID a string as it may be either an integer or a UUID string. gpu_id = str(gpu_id) gpu_env = dict(merged_env) gpu_env["CUDA_VISIBLE_DEVICES"] = gpu_id f.addStep( NinjaCommand( name='run simple CUDA tests on gpu %s' % gpu_id, targets=["check-cuda-simple"], jobs=1, # lit will parallelize the jobs haltOnFailure=True, description=split("Running simple CUDA tests on GPU %s" % gpu_id), descriptionDone=split("simple CUDA tests on GPU %s done." % gpu_id), workdir=ts_build_dir, env=gpu_env)) else: f.addStep( NinjaCommand( name='run simple CUDA tests', targets=["check-cuda-simple"], jobs=1, # lit will parallelize the jobs haltOnFailure=True, description=split("Running simple CUDA tests"), descriptionDone=split("simple CUDA tests done."), workdir=ts_build_dir, env=merged_env)) # If we've enabled thrust tests, build them now. # WARNING: This takes a lot of time to build. if (enable_thrust_tests): f.addStep( NinjaCommand(name='ninja build thrust', targets=["cuda-tests-thrust"], jobs=jobs, haltOnFailure=True, description=split("building thrust tests"), descriptionDone=split("thrust tests built."), workdir=ts_build_dir, env=merged_env)) # Run them. That also takes a while. # TODO(tra) we may want to run more than one instance so one # can be compiling tests while another is running them on GPU. if run_thrust_tests: f.addStep( NinjaCommand( name='run all CUDA tests', targets=["check"], jobs=1, # lit will parallelize the jobs. haltOnFailure=True, description=split("running all CUDA tests."), descriptionDone=split("all cuda tests done."), workdir=ts_build_dir, env=merged_env)) return f
def getSanitizerWindowsBuildFactory( clean=False, cmake='cmake', # Default values for VS devenv and build configuration vs=r"""%VS120COMNTOOLS%""", config='Release', target_arch='x86', extra_cmake_args=[]): ############# PREPARING f = buildbot.process.factory.BuildFactory() # Determine Slave Environment and Set MSVC environment. f.addStep(SetProperty( command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f = getSource(f,'llvm') # Global configurations build_dir='build' ############# CLEANING cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep(RemoveDirectory(name='clean '+build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested )) f.addStep(ShellCommand(name='cmake', command=[cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE="+config, "-DLLVM_ENABLE_ASSERTIONS=ON"] + extra_cmake_args, haltOnFailure=True, workdir=build_dir, env=Property('slave_env'))) # Build compiler-rt first to speed up detection of Windows-specific # compiler-time errors in the sanitizers runtime. f.addStep(NinjaCommand(name='build compiler-rt', targets=['compiler-rt'], haltOnFailure=True, description='ninja compiler-rt', workdir=build_dir, env=Property('slave_env'))) # Only run sanitizer tests. # Don't build targets that are not required in order to speed up the cycle. test_targets = ['check-asan','check-asan-dynamic','check-sanitizer', 'check-cfi'] f.addStep(NinjaCommand(name='run tests', targets=test_targets, haltOnFailure=True, description='ninja test', workdir=build_dir, env=Property('slave_env'))) return f