def getFactory( depends_on_projects = None, targets = None, checks = None, clean = False, extra_configure_args = None, llvm_srcdir = None, obj_dir = None, env = None, **kwargs): # Prepare environmental variables. Set here all env we want for all steps. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) if depends_on_projects: depends_on_projects = list(depends_on_projects) else: depends_on_projects = ['llvm', 'lld'] if checks is None: checks = [] # No check unless requested specifically. if extra_configure_args is None: cmake_args = list() else: cmake_args = list(extra_configure_args) # Some options are required for this build no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DLLVM_OPTIMIZED_TABLEGEN=', 'OFF'), ('-DLLVM_BUILD_STATIC=', 'ON'), ('-DLLVM_ENABLE_PIC=', 'OFF'), ]) # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-G', 'Ninja'), ('-DLLVM_LIT_ARGS=', '-v -vv'), ]) f = UnifiedTreeBuilder.getCmakeBuildFactory( depends_on_projects=depends_on_projects, clean=clean, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, extra_configure_args=cmake_args, env=merged_env, **kwargs) # Pass through all the extra arguments. if targets: step_name = "build-{}".format("-".join(targets)) step_description=["Build"] step_description.extend(targets) else: step_name = "build-unified-tree" step_description=["Build", "unified", "tree"] f.addStep(NinjaCommand(name=step_name, targets=targets, description=step_description, haltOnFailure=kwargs.get('haltOnFailure', True), env=merged_env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) # Test just built components if requested. if checks: f.addStep(NinjaCommand(name="test-{}".format("-".join(checks)), targets=checks, description=[ "Test", "just", "built", "components"], haltOnFailure=kwargs.get('haltOnFailure', True), env=merged_env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) # Copy just built LLD executable to the test suite directory # to avoid load from a hard drive overhead. f.addStep( ShellCommand( name="copy-lld-to-test-suite", description=[ "Copy", "LLD", "executable", "to", "the", "performance", "test", "suite", ], command=[ "cp", "-aL", "./{}/bin/ld.lld".format(f.obj_dir), "./lld-speed-test/ld.lld" ], env=merged_env, workdir='.' ) ) # Run the performance test suite. perf_command = [ "python", "%(builddir)s/lld-benchmark.py", "--machine=%(workername)s", "--revision=%(got_revision)s", "--linker=./ld.lld", ".", ] f.addStep( ShellCommand( name="performance-test-suite", description=[ "LLD", "performance","test","suite", ], command=WithProperties(" ".join(perf_command)), workdir="./lld-speed-test", env=merged_env ) ) return f
def getSphinxDocsBuildFactory( llvm_html=False, # Build LLVM HTML documentation llvm_man=False, # Build LLVM man pages clang_html=False, # Build Clang HTML documentation clang_tools_html=False, # Build Clang Extra Tools HTML documentation lld_html=False, # Build LLD HTML documentation libcxx_html=False, # Build Libc++ HTML documentation libunwind_html=False, # Build libunwind HTML documentation lldb_html=False, # Build LLDB HTML documentation extra_configure_args=None, **kwargs): if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults for the config flags. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-G', 'Ninja'), ('-DLLVM_ENABLE_SPHINX=', 'ON'), ('-DSPHINX_OUTPUT_HTML=', 'ON'), ('-DSPHINX_OUTPUT_MAN=', 'ON'), ('-DLLDB_INCLUDE_TESTS=', 'OFF'), ('-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), ]) llvm_srcdir = 'llvm/src' llvm_objdir = 'llvm/build' depends_on_projects = ['llvm'] if clang_html or clang_tools_html or lldb_html: depends_on_projects.append('clang') if clang_tools_html: depends_on_projects.append('clang-tools-extra') if lld_html: depends_on_projects.append('lld') if lldb_html: depends_on_projects.append('lldb') if libcxx_html: depends_on_projects.append('libcxx') depends_on_projects.append('libcxxabi') if libunwind_html: depends_on_projects.append('libunwind') f = UnifiedTreeBuilder.getCmakeBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=llvm_objdir, extra_configure_args=cmake_args, **kwargs) # Pass through all the extra arguments. if llvm_html: f.addStep( NinjaCommand(name="docs-llvm-html", haltOnFailure=True, description=["Build LLVM Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-llvm-html'])) if llvm_man: f.addStep( NinjaCommand(name="docs-llvm-man", haltOnFailure=True, description=["Build LLVM Sphinx man pages"], workdir=llvm_objdir, targets=['docs-llvm-man'])) if clang_html: f.addStep( NinjaCommand(name="docs-clang-html", haltOnFailure=True, description=["Build Clang Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-clang-html'])) if clang_tools_html: f.addStep( NinjaCommand( name="docs-clang-tools-html", haltOnFailure=True, description=[ "Build Clang Extra Tools Sphinx HTML documentation" ], workdir=llvm_objdir, targets=['docs-clang-tools-html'])) if lld_html: f.addStep( NinjaCommand(name="docs-lld-html", haltOnFailure=True, description=["Build LLD Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-lld-html'])) if lldb_html: f.addStep( NinjaCommand(name="docs-lldb-html", haltOnFailure=True, description=["Build LLDB Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-lldb-html'])) if libcxx_html: f.addStep( NinjaCommand( name="docs-libcxx-html", haltOnFailure=True, description=["Build Libc++ Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-libcxx-html'])) if libunwind_html: f.addStep( NinjaCommand( name="docs-libunwind-html", haltOnFailure=True, description=["Build libunwind Sphinx HTML documentation"], workdir=llvm_objdir, targets=['docs-libunwind-html'])) return f
def getLLVMDocsBuildFactory(clean=False, depends_on_projects=None, extra_configure_args=None, env=None, **kwargs): if depends_on_projects is None: # All the projects by default. _depends_on_projects = [ "llvm", "clang", "clang-tools-extra", "libcxx", "libcxxabi", "libunwind", "lld", "lldb", "flang", "openmp", "polly", ] else: # Make a local copy of depends_on_projects, as we are going to modify # that. _depends_on_projects = depends_on_projects[:] # Some projects are interdependent for the purpose of documentation. # Enforce the dependencies. if ("clang-tools-extra" in _depends_on_projects or \ "lldb" in _depends_on_projects ) and "clang" not in _depends_on_projects: _depends_on_projects.append("clang") if "libcxx" in _depends_on_projects and \ "libcxxabi" not in _depends_on_projects: _depends_on_projects.append("libcxxabi") if "libcxxabi" in _depends_on_projects and \ "libcxx" not in _depends_on_projects: _depends_on_projects.append("libcxx") # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) CmakeCommand.applyDefaultOptions(cmake_args, [ ("-G", "Ninja"), ("-DLLVM_ENABLE_SPHINX=", "ON"), ("-DSPHINX_OUTPUT_HTML=", "ON"), ("-DSPHINX_OUTPUT_MAN=", "OFF"), ("-DSPHINX_WARNINGS_AS_ERRORS=", "OFF"), ("-DLLVM_ENABLE_ASSERTIONS=", "OFF"), ("-DCMAKE_BUILD_TYPE=", "Release"), ]) # Build docs for each of the projects this builder depends on docs = [ llvm_docs[project] for project in llvm_docs.keys() if project in _depends_on_projects ] f = UnifiedTreeBuilder.getCmakeBuildFactory( clean=clean, depends_on_projects=_depends_on_projects, extra_configure_args=cmake_args, env=merged_env, **kwargs) # Pass through all the extra arguments. UnifiedTreeBuilder.addNinjaSteps(f, targets=[d[0] for d in docs], checks=[], env=merged_env, **kwargs) # Publish just built documentation for target, local_path, remote_path in docs: f.addStep( ShellCommand( name="Publish %s" % target, description=[ "Publish", "just", "built", "documentation", "fior", "%s" % target, ], command=[ 'rsync', '-vrl', '--delete', '--force', '--delay-updates', '--delete-delay', '--ignore-times', '--checksum', '-p', '--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r', "%s" % local_path, "lists.llvm.org:web/%s" % remote_path, ], env=merged_env, )) return f
def getFactory( depends_on_projects = None, targets = None, checks = None, clean = False, extra_configure_args = None, env = None, **kwargs): # Prepare environmental variables. Set here all env we want for all steps. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) if depends_on_projects: depends_on_projects = list(depends_on_projects) else: depends_on_projects = ['llvm', 'lld'] if checks is None: checks = [] # No check unless requested specifically. if extra_configure_args is None: cmake_args = list() else: cmake_args = list(extra_configure_args) # Some options are required for this build no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'OFF'), ('-DLLVM_BUILD_STATIC=', 'ON'), ('-DLLVM_ENABLE_PIC=', 'OFF'), ]) f = UnifiedTreeBuilder.getCmakeBuildFactory( depends_on_projects=depends_on_projects, clean=clean, extra_configure_args=cmake_args, env=merged_env, **kwargs) # Pass through all the extra arguments. if targets: step_name = "build-%s" % ("-".join(targets)) step_description=["Build"] step_description.extend(targets) else: step_name = "build-unified-tree" step_description=["Build", "unified", "tree"] f.addStep(NinjaCommand(name=step_name, targets=targets, description=step_description, haltOnFailure=kwargs.get('haltOnFailure', True), env=merged_env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) # Test just built components if requested. if checks: f.addStep(NinjaCommand(name="test-%s" % ("-".join(checks)), targets=checks, description=[ "Test", "just", "built", "components"], haltOnFailure=kwargs.get('haltOnFailure', True), env=merged_env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) # Copy just built LLD executable to the test suite directory # to avoid load from a hard drive overhead. f.addStep( ShellCommand( name="copy-lld-to-test-suite", description=[ "Copy", "LLD", "executable", "to", "the", "performance", "test", "suite", ], command=[ "cp", "-aL", "./bin/ld.lld", "../lld-speed-test/ld.lld" ], workdir=f.obj_dir, env=merged_env ) ) # Run the performance test suite. perf_command = [ "python", "%(workdir)s/lld-benchmark.py", "--machine=%(slavename)s", "--revision=%(got_revision)s", "--linker=./ld.lld", ".", ] f.addStep( ShellCommand( name="performance-test-suite", description=[ "LLD", "performance","test","suite", ], command=WithProperties(" ".join(perf_command)), workdir="./lld-speed-test", env=merged_env ) ) return f
def getLLVMDocsBuildFactory(clean=True, depends_on_projects=None, extra_configure_args=None, env=None, **kwargs): if depends_on_projects is None: # All the projects from llvm_docs, and remove all duplicates. _depends_on_projects = list( set([project for project in llvm_docs if project])) else: # Make a local copy of depends_on_projects, as we are going to modify # that. _depends_on_projects = depends_on_projects[:] # Some projects are interdependent for the purpose of documentation. # Enforce the dependencies. # TODO: Check later the dependencies for doxygen docs and enforce them # here if needed. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) CmakeCommand.CmakeCommand.applyDefaultOptions(cmake_args, [ ("-G", "Ninja"), ("-DLLVM_ENABLE_DOXYGEN=", "ON"), ("-DLLVM_BUILD_DOCS=", "ON"), ("-DCLANG_TOOLS_EXTRA_INCLUDE_DOCS=", "ON"), ("-DLLVM_ENABLE_ASSERTIONS=", "OFF"), ("-DCMAKE_BUILD_TYPE=", "Release"), ]) f = UnifiedTreeBuilder.getCmakeBuildFactory( clean=clean, depends_on_projects=_depends_on_projects, extra_configure_args=cmake_args, env=merged_env, **kwargs) # Pass through all the extra arguments. # Build the documentation for all the projects. for project in llvm_docs: target = llvm_docs[project][0] # Build only those with specifies targets. if target: UnifiedTreeBuilder.addNinjaSteps( f, # Doxygen builds the final result for really # long time without any output. # We have to have a long timeout at this step. timeout=10800, targets=[target], checks=[], env=merged_env, **kwargs) # Publish just built documentation for project in llvm_docs: target, local_path, remote_path = llvm_docs[project] f.addStep( ShellCommand( name="Publish {}".format(project or target), description=[ "Publish", "just", "built", "documentation", "for", "{}".format(project or target) ], command=[ 'rsync', '-vrl', '--delete', '--force', '--delay-updates', '--delete-delay', '--ignore-times', '--checksum', '-p', '--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r', "{}".format(local_path), "lists.llvm.org:web/doxygen/{}".format(remote_path), ], env=merged_env, )) return f
def getABITestsuitBuildFactory( clean = True, depends_on_projects = None, extra_configure_args = None, # Extra CMake args for all stages. jobs = None, # Restrict a degree of parallelism if needed. env = None, # Environmental variables for all steps. **kwargs): # Prepare environmental variables. Set here all env we want for all steps. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) if depends_on_projects: depends_on_projects = list(depends_on_projects) else: depends_on_projects = ['llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld'] if extra_configure_args is None: cmake_args = list() else: cmake_args = list(extra_configure_args) # Some options are required for this build no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ]) f = UnifiedTreeBuilder.getCmakeBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir="llvm", obj_dir="build", clean=clean, extra_configure_args=cmake_args, env=merged_env, **kwargs) # Pass through all the extra arguments. f.addStep(NinjaCommand(name="build-unified-tree", haltOnFailure=True, description=["Build", "unified", "tree"], env=merged_env, workdir=f.obj_dir, **kwargs # Pass through all the extra arguments. )) # Checkout the test-suite. f.addStep(SVN(name='svn-test-suite', mode='update', baseURL='http://llvm.org/svn/llvm-project/test-suite/', defaultBranch='trunk', workdir='test-suite')) # Run the ABI test. abi_test_env = { 'PYTHONPATH' : WithProperties("%(workdir)s/" + f.llvm_srcdir + "/utils/lit:${PYTHONPATH}"), 'PATH' : WithProperties("%(workdir)s/" + f.obj_dir + "/bin:${PATH}"), } merged_env.update(abi_test_env) abi_test_cmd = ["python", "linux-x86.py", "clang", "test", "-v"] if jobs: abi_test_cmd.append("-j" + str(jobs)) f.addStep(LitTestCommand(name='abi-test-suite', command=abi_test_cmd, description=["running", "ABI", "test-suite"], descriptionDone=["ABI", "test-suite", "completed"], workdir='test-suite/ABI-Testsuite', env=merged_env)) return f