def __init__(self, prefixCommand=None, options=None, path=None, **kwargs): self.prefixCommand = prefixCommand self.path = [path] if options is None: self.options = list() else: self.options = list(options) command = [] if prefixCommand: command += prefixCommand command += ["cmake"] if self.options: command += self.options if self.path: command += self.path # Remove here all the kwargs any of our LLVM buildbot command could consume. # Note: We will remove all the empty items from the command at start, as we # still didn't get yet WithProperties rendered. sanitized_kwargs = self.sanitize_kwargs(kwargs) sanitized_kwargs["command"] = command # And upcall to let the base class do its work WarningCountingShellCommand.__init__(self, **sanitized_kwargs) self.addFactoryArguments(prefixCommand=prefixCommand, options=self.options, path=path)
def __init__(self, prefixCommand=None, targets=None, **kwargs): self.prefixCommand = prefixCommand self.targets = targets command = [] if prefixCommand: command += prefixCommand command += [ "ninja", WithProperties("%(jobs:+-j)s"), WithProperties("%(jobs:-)s"), WithProperties("%(loadaverage:+-l)s"), WithProperties("%(loadaverage:-)s") ] if targets: command += targets # Don't forget to remove all the empty items from the command, # which we could get because of WithProperties rendered as empty strings. kwargs['command'] = command # And upcall to let the base class do its work WarningCountingShellCommand.__init__(self, **kwargs) self.addFactoryArguments(prefixCommand=prefixCommand, targets=targets)
def __init__(self, prefixCommand=None, options=None, targets=None, **kwargs): self.prefixCommand = prefixCommand self.targets = targets command = [] if prefixCommand: command += prefixCommand command += ["ninja"] if options is None: self.options = list() else: self.options = list(options) j_opt = re.compile(r'^-j$|^-j\d+$') l_opt = re.compile(r'^-l$|^-l\d+(\.(\d+)?)?$') # We can get jobs in the options. If so, we would use that. if not any(j_opt.search(opt) for opt in self.options if isinstance(opt, basestring)): # Otherwise let's see if we got it in the kwargs. if kwargs.get('jobs', None): self.options += ["-j", kwargs['jobs']] else: # Use the property if option was not explicitly # specified. command += [ WithProperties("%(jobs:+-j)s"), WithProperties("%(jobs:-)s"), ] # The same logic is for hanling the loadaverage option. if not any(l_opt.search(opt) for opt in self.options if isinstance(opt, basestring)): if kwargs.get('loadaverage', None): self.options += ["-l", kwargs['loadaverage']] else: command += [ WithProperties("%(loadaverage:+-l)s"), WithProperties("%(loadaverage:-)s"), ] if self.options: command += self.options if targets: command += targets # Remove here all the kwargs any of our LLVM buildbot command could consume. # Note: We will remove all the empty items from the command at start, as we # still didn't get yet WithProperties rendered. sanitized_kwargs = self.sanitize_kwargs(kwargs) sanitized_kwargs["command"] = command # And upcall to let the base class do its work WarningCountingShellCommand.__init__(self, **sanitized_kwargs) self.addFactoryArguments(prefixCommand=prefixCommand, options=self.options, targets=targets)
def setupEnvironment(self, cmd): # First upcall to get everything prepared. WarningCountingShellCommand.setupEnvironment(self, cmd) # Set default status format string. if cmd.args['env'] is None: cmd.args['env'] = {} cmd.args['env']['NINJA_STATUS'] = cmd.args['env'].get('NINJA_STATUS', "%e [%u/%r/%f] ")
def setupEnvironment(self, cmd): # First upcall to get everything prepared. WarningCountingShellCommand.setupEnvironment(self, cmd) # Set default status format string. if cmd.args['env'] is None: cmd.args['env'] = {} cmd.args['env']['NINJA_STATUS'] = cmd.args['env'].get( 'NINJA_STATUS', "%e [%u/%r/%f] ")
def __init__(self, prefixCommand=None, options=None, targets=None, **kwargs): self.prefixCommand = prefixCommand self.targets = targets command = [] if prefixCommand: command += prefixCommand command += ["make"] if options is None: self.options = list() else: self.options = list(options) j_opt = re.compile(r'^-j$|^-j\d+$') # We can get jobs in the options. If so, we would use that. if not any( j_opt.search(opt) for opt in self.options if isinstance(opt, basestring)): # Otherwise let's see if we got it in the kwargs. if kwargs.get('jobs', None): self.options += ["-j", kwargs['jobs']] else: # Use the property if option was not explicitly # specified. command += [ WithProperties("%(jobs:+-j)s"), WithProperties("%(jobs:-)s"), ] if self.options: command += self.options if targets: command += targets # Remove here all the kwargs any of our LLVM buildbot command could consume. # Note: We will remove all the empty items from the command at start, as we # still didn't get yet WithProperties rendered. sanitized_kwargs = self.sanitize_kwargs(kwargs) sanitized_kwargs["command"] = command # And upcall to let the base class do its work WarningCountingShellCommand.__init__(self, **sanitized_kwargs) self.addFactoryArguments(prefixCommand=prefixCommand, options=self.options, targets=targets)
def getLLDBCmakeAndCompileSteps(f, build_compiler, build_type, ninja_target, bindir, target_platform, target_arch, env=None): if env is None: env = {} llvm_builddir = 'build' if target_platform is 'android': llvm_builddir = 'build/android-' + target_arch # Configure f = getLLDBCMakeStep(f, build_compiler, build_type, bindir, target_platform, target_arch, env) # Compile f.addStep( WarningCountingShellCommand(name='ninja-%s-%s' % (target_platform, target_arch), command=[ 'nice', '-n', '10', 'ninja', WithProperties('-j%(jobs)s') ] + ninja_target, env=env, haltOnFailure=True, workdir=llvm_builddir)) return f
def addModernClangGDBTests(f, jobs, install_prefix): make_vars = [ WithProperties('RUNTESTFLAGS=CC_FOR_TARGET=\'{0}/bin/clang\' ' 'CXX_FOR_TARGET=\'{0}/bin/clang++\' ' 'CFLAGS_FOR_TARGET=\'-w -fno-limit-debug-info\''.format( install_prefix)) ] f.addStep( SVN(name='svn-clang-modern-gdb-tests', mode='update', svnurl= 'http://llvm.org/svn/llvm-project/clang-tests-external/trunk/gdb/7.5', workdir='clang-tests/src')) f.addStep( Configure(command='../src/configure', workdir='clang-tests/build/')) f.addStep( WarningCountingShellCommand( name='gdb-75-build', command=['make', WithProperties('-j%s' % jobs)], haltOnFailure=True, workdir='clang-tests/build')) f.addStep( commands.DejaGNUCommand.DejaGNUCommand( name='gdb-75-check', command=['make', '-k', WithProperties('-j%s' % jobs), 'check'] + make_vars, workdir='clang-tests/build', logfiles={ 'dg.sum': 'gdb/testsuite/gdb.sum', 'gdb.log': 'gdb/testsuite/gdb.log' }))
def __init__(self, prefixCommand=None, options=None, path=None, **kwargs): self.prefixCommand = prefixCommand self.path = [path] if options is None: self.options = list() else: self.options = list(options) command = [] if prefixCommand: command += prefixCommand command += ["cmake"] # Set some default options. CmakeCommand.applyDefaultOptions(self.options, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DLLVM_ENABLE_WERROR=', 'ON'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ]) if self.options: command += self.options if self.path: command += self.path # Remove here all the kwargs any of our LLVM buildbot command could consume. # Note: We will remove all the empty items from the command at start, as we # still didn't get yet WithProperties rendered. sanitized_kwargs = self.sanitize_kwargs(kwargs) sanitized_kwargs["command"] = command # And upcall to let the base class do its work WarningCountingShellCommand.__init__(self, **sanitized_kwargs) self.addFactoryArguments(prefixCommand=prefixCommand, options=self.options, path=path)
def __init__(self, prefixCommand=None, options=None, path=None, **kwargs): self.prefixCommand = prefixCommand self.path = [path] if options is None: self.options = list() else: self.options = list(options) command = [] if prefixCommand: command += prefixCommand command += ["cmake"] # Set some default options. CmakeCommand.applyDefaultOptions(self.options, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DLLVM_ENABLE_WERROR=', 'ON'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ]) if self.options: command += self.options if self.path: command += self.path # Remove here all the kwargs any of our LLVM buildbot command could consume. # Note: We will remove all the empty items from the command at start, as we # still didn't get yet WithProperties rendered. sanitized_kwargs = self.sanitize_kwargs(kwargs) sanitized_kwargs["command"] = command # And upcall to let the base class do its work WarningCountingShellCommand.__init__(self, **sanitized_kwargs) self.addFactoryArguments(prefixCommand=prefixCommand, options=self.options, path=path)
def testSuppressingLinelessWarningsPossible(self): # Use a warningExtractor that does not provide line # information w = WarningCountingShellCommand( warningExtractor=WarningCountingShellCommand.warnExtractWholeLine, command="echo") # Add suppression manually instead of using suppressionFile fileRe = None warnRe = ".*SUPPRESS.*" start = None end = None suppression = (fileRe, warnRe, start, end) w.addSuppression([suppression]) # Now call maybeAddWarning warnings = [] line = "this warning should be SUPPRESSed" match = re.match(".*warning.*", line) w.maybeAddWarning(warnings, line, match) # Finally make the suppressed warning was *not* added to the # list of warnings expectedWarnings = 0 self.assertEquals(len(warnings), expectedWarnings)
def __init__(self, prefixCommand=None, targets=None, **kwargs): self.prefixCommand = prefixCommand self.targets = targets command = [] if prefixCommand: command += prefixCommand command += ["ninja", WithProperties("%(jobs:+-j)s"), WithProperties("%(jobs:-)s"), WithProperties("%(loadaverage:+-l)s"), WithProperties("%(loadaverage:-)s")] if targets: command += targets # Don't forget to remove all the empty items from the command, # which we could get because of WithProperties rendered as empty strings. kwargs['command'] = command # And upcall to let the base class do its work WarningCountingShellCommand.__init__(self, **kwargs) self.addFactoryArguments(prefixCommand=prefixCommand, targets=targets)
def start(self): # Don't forget to remove all the empty items from the command, # which we could get because of WithProperties rendered as empty strings. self.command = filter(bool, self.command) # Then upcall. WarningCountingShellCommand.start(self)
def start(self): # Don't forget to remove all the empty items from the command, # which we could get because of WithProperties rendered as empty strings. self.command = filter(bool, self.command) # Then upcall. WarningCountingShellCommand.start(self)
def getLibompCMakeBuildFactory(clean=True, env=None, test=True, c_compiler="gcc", cxx_compiler="g++"): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) openmp_srcdir = "openmp.src" openmp_builddir = "openmp.build" llvm_srcdir = "llvm.src" llvm_builddir = "llvm.build" f = buildbot.process.factory.BuildFactory() # Get libomp f.addStep( SVN(name='svn-libomp', mode='update', baseURL='http://llvm.org/svn/llvm-project/openmp/', defaultBranch='trunk', workdir=openmp_srcdir)) # Get llvm to build llvm-lit f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) # Clean directory, if requested. if clean: f.addStep( ShellCommand(name="clean", command=["rm", "-rf", openmp_builddir, llvm_builddir], warnOnFailure=True, description=["clean"], workdir='.', env=merged_env)) # CMake llvm f.addStep( ShellCommand(name='cmake llvm', command=[ "cmake", "../" + llvm_srcdir, "-DCMAKE_C_COMPILER=" + c_compiler, "-DCMAKE_CXX_COMPILER=" + cxx_compiler ], haltOnFailure=True, description='cmake llvm', workdir=llvm_builddir, env=merged_env)) # Make llvm utils f.addStep( WarningCountingShellCommand(name='make llvm utils build', command=['make', 'LLVMX86Utils', '-j8'], haltOnFailure=True, description='make llvm utils build', workdir=llvm_builddir, env=merged_env)) # Add llvm-lit to PATH merged_env.update({ 'PATH': WithProperties("${PATH}:" + "%(workdir)s/" + llvm_builddir + "/bin") }) # CMake libomp f.addStep( ShellCommand(name='cmake libomp', command=[ "cmake", "../" + openmp_srcdir, "-DCMAKE_C_COMPILER=" + c_compiler, "-DCMAKE_CXX_COMPILER=" + cxx_compiler ], haltOnFailure=True, description='cmake libomp', workdir=openmp_builddir, env=merged_env)) # Make libomp f.addStep( WarningCountingShellCommand(name='make libomp build', command=['make'], haltOnFailure=True, description='make libomp build', workdir=openmp_builddir, env=merged_env)) # Test, if requested if test: f.addStep( lit_test_command.LitTestCommand(name='make check-libomp', command=['make', 'check-libomp'], haltOnFailure=True, description=["make check-libomp"], descriptionDone=["build checked"], workdir=openmp_builddir, env=merged_env)) return f
def startBuild(self, dummy): return WarningCountingShellCommand.start(self)
def _getClangCMakeBuildFactory( clean=True, test=True, cmake='cmake', jobs=None, # VS tools environment variable if using MSVC. For example, # %VS120COMNTOOLS% selects the 2013 toolchain. vs=None, vs_target_arch='x86', # Multi-stage compilation useTwoStage=False, testStage1=True, stage1_config='Release', stage2_config='Release', # Test-suite runTestSuite=False, nt_flags=[], submitURL=None, testerName=None, # Environmental variables for all steps. env={}, extra_cmake_args=[], # Extra repositories checkout_clang_tools_extra=True, checkout_compiler_rt=True, # Upload artifacts to Google Cloud Storage (for the llvmbisect tool) stage1_upload_directory=None, # Triggers trigger_after_stage1=[]): ############# PREPARING f = buildbot.process.factory.BuildFactory() addSVNUpdateSteps(f, checkout_clang_tools_extra=checkout_clang_tools_extra, checkout_compiler_rt=checkout_compiler_rt, checkout_test_suite=runTestSuite) # If jobs not defined, Ninja will choose a suitable value jobs_cmd = [] lit_args = "'-v" if jobs is not None: jobs_cmd = ["-j" + str(jobs)] lit_args += " -j" + str(jobs) + "'" else: lit_args += "'" ninja_cmd = ['ninja'] + jobs_cmd ninja_install_cmd = ['ninja', 'install'] + jobs_cmd ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd check_build_cmd = [ "sh", "-c", "test -e build.ninja && echo OK || echo Missing" ] if vs: check_build_cmd = [ "cmd", "/c", "if exist build.ninja (echo OK) " + " else (echo Missing & exit 1)" ] # Global configurations stage1_build = 'stage1' stage1_install = 'stage1.install' stage2_build = 'stage2' stage2_install = 'stage2.install' # Set up VS environment, if appropriate. if vs: f.addStep( SetProperty(command=builders_util.getVisualStudioEnvironment( vs, vs_target_arch), extract_fn=builders_util.extractSlaveEnvironment)) assert not env, "Can't have custom builder env vars with VS" env = Property('slave_env') ############# CLEANING if clean: f.addStep( ShellCommand(name='clean stage 1', command=['rm', '-rf', stage1_build], warnOnFailure=True, description='cleaning stage 1', descriptionDone='clean', workdir='.', env=env)) else: f.addStep( SetProperty(name="check ninja files 1", workdir=stage1_build, command=check_build_cmd, flunkOnFailure=False, property="exists_ninja_1")) ############# STAGE 1 f.addStep( ShellCommand( name='cmake stage 1', command=[ cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + stage1_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args, "-DCMAKE_INSTALL_PREFIX=../" + stage1_install ] + extra_cmake_args, haltOnFailure=True, description='cmake stage 1', workdir=stage1_build, doStepIf=lambda step: step.build.getProperty("exists_ninja_1" ) != "OK", env=env)) f.addStep( WarningCountingShellCommand(name='build stage 1', command=ninja_cmd, haltOnFailure=True, description='ninja all', workdir=stage1_build, env=env)) if test and testStage1: haltOnStage1Check = not useTwoStage and not runTestSuite f.addStep( lit_test_command.LitTestCommand( name='ninja check 1', command=ninja_check_cmd, haltOnFailure=haltOnStage1Check, description=["checking stage 1"], descriptionDone=["stage 1 checked"], workdir=stage1_build, env=env)) if useTwoStage or runTestSuite or stage1_upload_directory: f.addStep( ShellCommand(name='install stage 1', command=ninja_install_cmd, description='ninja install', workdir=stage1_build, env=env)) if stage1_upload_directory: addGCSUploadSteps(f, 'stage 1', stage1_install, stage1_upload_directory, env, gcs_url_property='stage1_package_gcs_url') # Compute the cmake define flag to set the C and C++ compiler to clang. Use # clang-cl if we used MSVC for stage1. if not vs: cc = 'clang' cxx = 'clang++' else: cc = 'clang-cl.exe' cxx = 'clang-cl.exe' ############# STAGE 2 if useTwoStage: # We always cleanly build the stage 2. If the compiler has been # changed on the stage 1, we cannot trust any of the intermediate file # from the old compiler. And if the stage 1 compiler is the same, we # should not build in the first place. f.addStep( ShellCommand(name='clean stage 2', command=['rm', '-rf', stage2_build], warnOnFailure=True, description='cleaning stage 2', descriptionDone='clean', workdir='.', env=env)) # Set the compiler using the CC and CXX environment variables to work around # backslash string escaping bugs somewhere between buildbot and cmake. The # env.exe helper is required to run the tests, so hopefully it's already on # PATH. cmake_cmd2 = [ 'env', WithProperties('CC=%(workdir)s/' + stage1_install + '/bin/' + cc), WithProperties('CXX=%(workdir)s/' + stage1_install + '/bin/' + cxx), cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + stage2_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args, "-DCMAKE_INSTALL_PREFIX=../" + stage2_install ] + extra_cmake_args f.addStep( ShellCommand(name='cmake stage 2', command=cmake_cmd2, haltOnFailure=True, description='cmake stage 2', workdir=stage2_build, doStepIf=lambda step: step.build.getProperty( "exists_ninja_2") != "OK", env=env)) f.addStep( WarningCountingShellCommand(name='build stage 2', command=ninja_cmd, haltOnFailure=True, description='ninja all', workdir=stage2_build, env=env)) if test: f.addStep( lit_test_command.LitTestCommand( name='ninja check 2', command=ninja_check_cmd, haltOnFailure=not runTestSuite, description=["checking stage 2"], descriptionDone=["stage 2 checked"], workdir=stage2_build, env=env)) ############# TEST SUITE ## Test-Suite (stage 2 if built, stage 1 otherwise) if runTestSuite: compiler_path = stage1_install if useTwoStage: compiler_path = stage2_install f.addStep( ShellCommand(name='install stage 2', command=ninja_install_cmd, description='ninja install 2', workdir=stage2_build, env=env)) # Get generated python, lnt python = WithProperties('%(workdir)s/test/sandbox/bin/python') lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt') lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py') # Paths sandbox = WithProperties('%(workdir)s/test/sandbox') test_suite_dir = WithProperties('%(workdir)s/test/test-suite') # Get latest built Clang (stage1 or stage2) cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc) cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx) # LNT Command line (don't pass -jN. Users need to pass both --threads # and --build-threads in nt_flags to get the same effect) test_suite_cmd = [ python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx ] # Append any option provided by the user test_suite_cmd.extend(nt_flags) # Only submit if a URL has been specified if submitURL is not None: if not isinstance(submitURL, list): submitURL = [submitURL] for url in submitURL: test_suite_cmd.extend(['--submit', url]) if testerName: test_suite_cmd.extend(['--no-machdep-info', testerName]) # CC and CXX are needed as env for build-tools test_suite_env = copy.deepcopy(env) test_suite_env['CC'] = cc test_suite_env['CXX'] = cxx # Steps to prepare, build and run LNT f.addStep( ShellCommand(name='clean sandbox', command=['rm', '-rf', 'sandbox'], haltOnFailure=True, description='removing sandbox directory', workdir='test', env=env)) f.addStep( ShellCommand(name='recreate sandbox', command=['virtualenv', 'sandbox'], haltOnFailure=True, description='recreating sandbox', workdir='test', env=env)) f.addStep( ShellCommand(name='setup lit', command=[python, lnt_setup, 'develop'], haltOnFailure=True, description='setting up LNT in sandbox', workdir='test/sandbox', env=env)) f.addStep( commands.LitTestCommand.LitTestCommand( name='test-suite', command=test_suite_cmd, haltOnFailure=True, description=['running the test suite'], workdir='test/sandbox', logfiles={ 'configure.log': 'build/configure.log', 'build-tools.log': 'build/build-tools.log', 'test.log': 'build/test.log', 'report.json': 'build/report.json' }, env=test_suite_env)) return f
def getClangMSVCBuildFactory(update=True, clean=True, vcDrive='c', jobs=1, cmake=r"cmake"): f = buildbot.process.factory.BuildFactory() if update: f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir='llvm')) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='llvm/tools/clang')) f.addStep( SVN(name='svn-clang-tools-extra', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', defaultBranch='trunk', workdir='llvm/tools/clang/tools/extra')) # Full & fast clean. if clean: f.addStep( ShellCommand(name='clean-1', command=['del', '/s/q', 'build'], warnOnFailure=True, description='cleaning', descriptionDone='clean', workdir='llvm')) f.addStep( ShellCommand(name='clean-2', command=['rmdir', '/s/q', 'build'], warnOnFailure=True, description='cleaning', descriptionDone='clean', workdir='llvm')) # Create the project files. # Use batch files instead of ShellCommand directly, Windows quoting is # borked. FIXME: See buildbot ticket #595 and buildbot ticket #377. f.addStep( batch_file_download.BatchFileDownload( name='cmakegen', command=[ cmake, "-DLLVM_TARGETS_TO_BUILD:=X86", "-DLLVM_INCLUDE_EXAMPLES:=OFF", "-DLLVM_INCLUDE_TESTS:=OFF", "-DLLVM_TARGETS_TO_BUILD:=X86", "-G", "Visual Studio 9 2008", ".." ], workdir="llvm\\build")) f.addStep( ShellCommand(name='cmake', command=['cmakegen.bat'], haltOnFailure=True, description='cmake gen', workdir='llvm\\build')) # Build it. f.addStep( batch_file_download.BatchFileDownload( name='vcbuild', command=[ vcDrive + r""":\Program Files\Microsoft Visual Studio 9.0\VC\VCPackages\vcbuild.exe""", "/M%d" % jobs, "LLVM.sln", "Debug|Win32" ], workdir="llvm\\build")) f.addStep( WarningCountingShellCommand(name='vcbuild', command=['vcbuild.bat'], haltOnFailure=True, description='vcbuild', workdir='llvm\\build', warningPattern=" warning C.*:")) # Build clang-test project. f.addStep( batch_file_download.BatchFileDownload( name='vcbuild_test', command=[ vcDrive + r""":\Program Files\Microsoft Visual Studio 9.0\VC\VCPackages\vcbuild.exe""", "clang-test.vcproj", "Debug|Win32" ], workdir="llvm\\build\\tools\\clang\\test")) f.addStep( lit_test_command.LitTestCommand( name='test-clang', command=["vcbuild_test.bat"], workdir="llvm\\build\\tools\\clang\\test")) return f
def getPollyBuildFactory( clean=False, install=False, make='make', jobs=None, checkAll=False, env=None, extraCmakeArgs=None, testsuite=False,extraTestsuiteCmakeArgs=None, **kwargs): if extraCmakeArgs is None: extraCmakeArgs=[] if extraTestsuiteCmakeArgs is None: extraTestsuiteCmakeArgs = [] llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" llvm_instdir = "llvm.inst" testsuite_srcdir = "test-suite.src" testsuite_builddir = "test-suite.build" jobs_cmd = [] if jobs is not None: jobs_cmd = ["-j{}".format(jobs)] build_cmd = [make] + jobs_cmd install_cmd = [make, 'install'] + jobs_cmd check_all_cmd = [make, 'check-all'] + jobs_cmd check_polly_cmd = [make, 'check-polly'] + jobs_cmd cmake_install = [] if install: cmake_install = ["-DCMAKE_INSTALL_PREFIX=../%s" % llvm_instdir] # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env: merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything. depends_on_projects = ['llvm','clang','polly'] if testsuite: # XRay tests in test-suite require compiler-rt depends_on_projects += ['compiler-rt'] cleanBuildRequestedByProperty = lambda step: step.build.getProperty("clean", False) cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=llvm_objdir, install_dir=llvm_instdir, cleanBuildRequested=cleanBuildRequested, **kwargs) # Pass through all the extra arguments. f.addStep(steps.RemoveDirectory(name='clean-src-dir', dir=f.monorepo_dir, warnOnFailure=True, doStepIf=cleanBuildRequestedByProperty)) # Get the source code. f.addGetSourcecodeSteps(**kwargs) # Clean build dir f.addStep(steps.RemoveDirectory(name='clean-build-dir', dir=llvm_objdir, warnOnFailure=True, doStepIf=cleanBuildRequested)) # Create configuration files with cmake cmakeCommand = ["cmake", "../%s/llvm" % llvm_srcdir, "-DCMAKE_COLOR_MAKEFILE=OFF", "-DPOLLY_TEST_DISABLE_BAR=ON", "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_POLLY_LINK_INTO_TOOLS=ON", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), ] + cmake_install + extraCmakeArgs f.addStep(ShellCommand(name="cmake-configure", command=cmakeCommand, haltOnFailure=False, description=["cmake configure"], workdir=llvm_objdir, env=merged_env)) # Build f.addStep(WarningCountingShellCommand(name="build", command=build_cmd, haltOnFailure=True, description=["build"], workdir=llvm_objdir, env=merged_env)) clangexe = "%(builddir)s/" + llvm_objdir + "/bin/clang" clangxxexe = "%(builddir)s/" + llvm_objdir + "/bin/clang++" litexe = "%(builddir)s/" + llvm_objdir + "/bin/llvm-lit" sizeexe = "%(builddir)s/" + llvm_objdir + "/bin/llvm-size" # Clean install dir if install: f.addStep(steps.RemoveDirectory(name='clean-install-dir', dir=llvm_instdir, haltOnFailure=False, doStepIf=cleanBuildRequested)) f.addStep(ShellCommand(name="install", command=install_cmd, haltOnFailure=True, description=["install"], workdir=llvm_objdir, env=merged_env)) # If installing, use the installed version of clang. clangexe = "%(builddir)s/" + llvm_instdir + "/bin/clang" clangxxexe = "%(builddir)s/" + llvm_instdir + "/bin/clang++" sizeexe = "%(builddir)s/" + llvm_instdir + "/bin/llvm-size" # Test if checkAll: f.addStep(LitTestCommand(name="check_all", command=check_all_cmd, haltOnFailure=False, description=["check all"], workdir=llvm_objdir, env=merged_env)) else: f.addStep(LitTestCommand(name="check_polly", command=check_polly_cmd, haltOnFailure=False, description=["check polly"], workdir=llvm_objdir, env=merged_env)) if testsuite: f.addStep(steps.RemoveDirectory(name='test-suite_clean-src-dir', dir=testsuite_srcdir, haltOnFailure=False, warnOnFailure=True, doStepIf=cleanBuildRequestedByProperty)) f.addGetSourcecodeForProject( project='test-suite', src_dir=testsuite_srcdir, alwaysUseLatest=True) f.addStep(steps.RemoveDirectory(name='test-suite_clean-build-dir', dir=testsuite_builddir, haltOnFailure=False, warnOnFailure=True)) # -Wno-unused-command-line-argument is needed because linking will not uses the "-mllvm -polly" argument. f.addStep(ShellCommand(name='test-suite_cmake-configure', description=["Test-Suite: cmake"], command=["cmake", '-B', testsuite_builddir, '-S', testsuite_srcdir, "-DCMAKE_BUILD_TYPE=Release", "-DTEST_SUITE_COLLECT_STATS=ON", "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", "-DTEST_SUITE_EXTRA_CXX_FLAGS=-mllvm -polly", "-DTEST_SUITE_LIT_FLAGS=-vv;-o;report.json", WithProperties("-DCMAKE_C_COMPILER=" + clangexe), WithProperties("-DCMAKE_CXX_COMPILER=" + clangxxexe), WithProperties("-DTEST_SUITE_LLVM_SIZE=" + sizeexe), WithProperties("-DTEST_SUITE_LIT=" + litexe), ] + extraTestsuiteCmakeArgs, haltOnFailure=True, workdir='.', env=merged_env)) f.addStep(WarningCountingShellCommand(name='test-suite_build', description=["Test-Suite: build"], # Continue building; programs that don't compile will fail with NOEXE. command=[make, 'all', '-k0'] + jobs_cmd, haltOnFailure=False, flunkOnFailure=True, workdir=testsuite_builddir, env=merged_env)) f.addStep(LitTestCommand(name='test-suite_run', description=['Test-Suite: run'], command=[WithProperties(litexe), '-vv', '-o', 'report.json', '.'], haltOnFailure=True, workdir=testsuite_builddir, logfiles={ 'test.log' : 'test.log', 'report.json': 'report.json' }, env=merged_env)) return f
def getClangBuildFactory( triple=None, clean=True, test=True, package_dst=None, run_cxx_tests=False, examples=False, valgrind=False, valgrindLeakCheck=False, useTwoStage=False, completely_clean=False, make='make', jobs="%(jobs)s", stage1_config='Debug+Asserts', stage2_config='Release+Asserts', env={}, # Environmental variables for all steps. extra_configure_args=[], stage2_extra_configure_args=[], use_pty_in_tests=False, trunk_revision=None, force_checkout=False, extra_clean_step=None, checkout_compiler_rt=False, run_gdb=False, run_modern_gdb=False, run_gcc=False): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) llvm_srcdir = "llvm.src" llvm_1_objdir = "llvm.obj" llvm_1_installdir = "llvm.install.1" llvm_2_objdir = "llvm.obj.2" llvm_2_installdir = "llvm.install" f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( buildbot.steps.shell.SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".", env=merged_env)) # Blow away completely, if requested. if completely_clean: f.addStep( ShellCommand(name="rm-llvm.src", command=["rm", "-rf", llvm_srcdir], haltOnFailure=True, description=["rm src dir", "llvm"], workdir=".", env=merged_env)) # Checkout sources. if trunk_revision: # The SVN build step provides no mechanism to check out a specific revision # based on a property, so just run the commands directly here. svn_co = ['svn', 'checkout'] if force_checkout: svn_co += ['--force'] svn_co += ['--revision', WithProperties(trunk_revision)] svn_co_llvm = svn_co + \ [WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%s' % trunk_revision), llvm_srcdir] svn_co_clang = svn_co + \ [WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%s' % trunk_revision), '%s/tools/clang' % llvm_srcdir] svn_co_clang_tools_extra = svn_co + \ [WithProperties('http://llvm.org/svn/llvm-project/clang-tools-extra/trunk@%s' % trunk_revision), '%s/tools/clang/tools/extra' % llvm_srcdir] f.addStep( ShellCommand(name='svn-llvm', command=svn_co_llvm, haltOnFailure=True, workdir='.')) f.addStep( ShellCommand(name='svn-clang', command=svn_co_clang, haltOnFailure=True, workdir='.')) f.addStep( ShellCommand(name='svn-clang-tools-extra', command=svn_co_clang_tools_extra, haltOnFailure=True, workdir='.')) else: f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir)) f.addStep( SVN(name='svn-clang-tools-extra', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', defaultBranch='trunk', workdir='%s/tools/clang/tools/extra' % llvm_srcdir)) if checkout_compiler_rt: f.addStep( SVN(name='svn-compiler-rt', mode='update', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', defaultBranch='trunk', workdir='%s/projects/compiler-rt' % llvm_srcdir)) # Clean up llvm (stage 1); unless in-dir. if clean and llvm_srcdir != llvm_1_objdir: f.addStep( ShellCommand(name="rm-llvm.obj.stage1", command=["rm", "-rf", llvm_1_objdir], haltOnFailure=True, description=["rm build dir", "llvm"], workdir=".", env=merged_env)) if not clean: expected_makefile = 'Makefile' f.addStep( SetProperty(name="Makefile_isready", workdir=llvm_1_objdir, command=[ "sh", "-c", "test -e %s && echo OK || echo Missing" % expected_makefile ], flunkOnFailure=False, property="exists_Makefile")) cmake_triple_arg = [] if triple: cmake_triple_arg = ['-DLLVM_HOST_TRIPLE=%s' % triple] f.addStep( ShellCommand(name='cmake', command=[ 'cmake', '-DLLVM_BUILD_TESTS=ON', '-DCMAKE_BUILD_TYPE=%s' % stage1_config ] + cmake_triple_arg + extra_configure_args + ["../" + llvm_srcdir], description='cmake stage1', workdir=llvm_1_objdir, env=merged_env, doStepIf=lambda step: step.build.getProperty( "exists_Makefile") != "OK")) # Make clean if using in-dir builds. if clean and llvm_srcdir == llvm_1_objdir: f.addStep( WarningCountingShellCommand(name="clean-llvm", command=[make, "clean"], haltOnFailure=True, description="cleaning llvm", descriptionDone="clean llvm", workdir=llvm_1_objdir, doStepIf=clean, env=merged_env)) if extra_clean_step: f.addStep(extra_clean_step) f.addStep( WarningCountingShellCommand( name="compile", command=['nice', '-n', '10', make, WithProperties("-j%s" % jobs)], haltOnFailure=True, flunkOnFailure=not run_gdb, description=["compiling", stage1_config], descriptionDone=["compile", stage1_config], workdir=llvm_1_objdir, env=merged_env)) if examples: f.addStep( WarningCountingShellCommand( name="compile.examples", command=[ 'nice', '-n', '10', make, WithProperties("-j%s" % jobs), "BUILD_EXAMPLES=1" ], haltOnFailure=True, description=["compiling", stage1_config, "examples"], descriptionDone=["compile", stage1_config, "examples"], workdir=llvm_1_objdir, env=merged_env)) clangTestArgs = '-v -j %s' % jobs if valgrind: clangTestArgs += ' --vg' if valgrindLeakCheck: clangTestArgs += ' --vg-leak' clangTestArgs += ' --vg-arg --suppressions=%(builddir)s/llvm/tools/clang/utils/valgrind/x86_64-pc-linux-gnu_gcc-4.3.3.supp --vg-arg --suppressions=%(builddir)s/llvm/utils/valgrind/x86_64-pc-linux-gnu.supp' extraTestDirs = '' if run_cxx_tests: extraTestDirs += '%(builddir)s/llvm/tools/clang/utils/C++Tests' if test: f.addStep( lit_test_command.LitTestCommand( name='check-all', command=[ make, "check-all", "VERBOSE=1", WithProperties("LIT_ARGS=%s" % clangTestArgs), WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs) ], flunkOnFailure=not run_gdb, description=["checking"], descriptionDone=["checked"], workdir=llvm_1_objdir, usePTY=use_pty_in_tests, env=merged_env)) # TODO: Install llvm and clang for stage1. if run_gdb or run_gcc or run_modern_gdb: ignores = getClangTestsIgnoresFromPath( os.path.expanduser('~/public/clang-tests'), 'clang-x86_64-darwin10') install_prefix = "%%(builddir)s/%s" % llvm_1_installdir if run_gdb: addClangGDBTests(f, ignores, install_prefix) if run_modern_gdb: addModernClangGDBTests(f, jobs, install_prefix) if run_gcc: addClangGCCTests(f, ignores, install_prefix) if not useTwoStage: if package_dst: name = WithProperties( "%(builddir)s/" + llvm_1_objdir + "/clang-r%(got_revision)s-b%(buildnumber)s.tgz") f.addStep( ShellCommand(name='pkg.tar', description="tar root", command=["tar", "zcvf", name, "./"], workdir=llvm_1_installdir, warnOnFailure=True, flunkOnFailure=False, haltOnFailure=False, env=merged_env)) f.addStep( ShellCommand(name='pkg.upload', description="upload root", command=[ "scp", name, WithProperties(package_dst + "/%(buildername)s") ], workdir=".", warnOnFailure=True, flunkOnFailure=False, haltOnFailure=False, env=merged_env)) return f # Clean up llvm (stage 2). # # We always cleanly build the stage 2. If the compiler has been # changed on the stage 1, we cannot trust any of the intermediate file # from the old compiler. And if the stage 1 compiler is the same, we should # not build in the first place. f.addStep( ShellCommand(name="rm-llvm.obj.stage2", command=["rm", "-rf", llvm_2_objdir], haltOnFailure=True, description=["rm build dir", "llvm", "(stage 2)"], workdir=".", env=merged_env)) # Configure llvm (stage 2). f.addStep( ShellCommand( name='cmake', command=['cmake'] + stage2_extra_configure_args + [ '-DLLVM_BUILD_TESTS=ON', WithProperties('-DCMAKE_C_COMPILER=%%(builddir)s/%s/bin/clang' % llvm_1_objdir), # FIXME use installdir WithProperties( '-DCMAKE_CXX_COMPILER=%%(builddir)s/%s/bin/clang++' % llvm_1_objdir), '-DCMAKE_BUILD_TYPE=%s' % stage2_config, "../" + llvm_srcdir ], description='cmake stage2', workdir=llvm_2_objdir, env=merged_env)) # Build llvm (stage 2). f.addStep( WarningCountingShellCommand( name="compile.llvm.stage2", command=['nice', '-n', '10', make, WithProperties("-j%s" % jobs)], haltOnFailure=True, description=["compiling", "(stage 2)", stage2_config], descriptionDone=["compile", "(stage 2)", stage2_config], workdir=llvm_2_objdir, env=merged_env)) if test: f.addStep( lit_test_command.LitTestCommand( name='check-all', command=[ make, "check-all", "VERBOSE=1", WithProperties("LIT_ARGS=%s" % clangTestArgs), WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs) ], description=["checking"], descriptionDone=["checked"], workdir=llvm_2_objdir, usePTY=use_pty_in_tests, env=merged_env)) # TODO: Install llvm and clang for stage2. if package_dst: name = WithProperties("%(builddir)s/" + llvm_2_objdir + "/clang-r%(got_revision)s-b%(buildnumber)s.tgz") f.addStep( ShellCommand(name='pkg.tar', description="tar root", command=["tar", "zcvf", name, "./"], workdir=llvm_2_installdir, warnOnFailure=True, flunkOnFailure=False, haltOnFailure=False, env=merged_env)) f.addStep( ShellCommand(name='pkg.upload', description="upload root", command=[ "scp", name, WithProperties(package_dst + "/%(buildername)s") ], workdir=".", warnOnFailure=True, flunkOnFailure=False, haltOnFailure=False, env=merged_env)) return f
def startBuild(self, cmd): if cmd.rc != 0: log.msg("Failure when running %s." % cmd) self.finished(FAILURE) else: return WarningCountingShellCommand.start(self)
def get3StageClangLTOBuildFactory( clean=True, env=None, build_gold=False, cmake_cache_file=None, extra_cmake_options=None, ): llvm_srcdir = "llvm.src" llvm_objdir = "llvm-stage1.obj" merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: merged_env.update(env) f = BuildFactory() f.addStep( SVN( name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir ) ) f.addStep( SVN( name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir ) ) # Clean directory, if requested. cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( ShellCommand( doStepIf=cleanBuildRequested, name="rm-llvm_objdir", command=["rm", "-rf", llvm_objdir], haltOnFailure=True, description=["rm build dir", "llvm"], workdir=".", env=merged_env ) ) cmake_command = ["cmake"] if cmake_cache_file: cmake_command += ['-C', cmake_cache_file] if extra_cmake_options: cmake_command += extra_cmake_options cmake_command += [ "../%s" % llvm_srcdir ] # Note: ShellCommand does not pass the params with special symbols right. # The " ".join is a workaround for this bug. f.addStep( WarningCountingShellCommand( name="cmake-configure", description=["cmake configure"], haltOnFailure=True, command=WithProperties(" ".join(cmake_command)), workdir=llvm_objdir, env=merged_env ) ) if build_gold: f.addStep( NinjaCommand(name='build-LLVMgold.so', targets=['lib/LLVMgold.so'], haltOnFailure=True, warnOnWarnings=True, description=["3 Stage Build Clang"], workdir=llvm_objdir, env=merged_env) ) f.addStep( NinjaCommand(name='build-stage3-clang', targets=['stage3-clang'], haltOnFailure=True, warnOnWarnings=True, description=["3 Stage Build Clang"], timeout=3600, # Default value is not enough. workdir=llvm_objdir, env=merged_env) ) f.addStep( NinjaCommand(name='build-stage3-check-clang', targets=['stage3-check-clang'], haltOnFailure=True, warnOnWarnings=True, description=["Check Clang"], timeout=3600, # Default value is not enough. workdir=llvm_objdir, env=merged_env) ) # Compare stage2 & stage3 clang shell_command = [ "diff", "-q", "tools/clang/stage2-bins/bin/clang", "tools/clang/stage2-bins/tools/clang/stage3-bins/bin/clang" ] f.addStep( ShellCommand( name="compare-clang", description=["comapre stage2 & stage3 clang"], haltOnFailure=True, command=WithProperties(" ".join(shell_command)), workdir=llvm_objdir, env=merged_env ) ) return f
def getSanitizerBuildFactoryII( clean=False, sanity_check=True, sanitizers=['sanitizer','asan','lsan','msan','tsan','ubsan','dfsan'], build_type="Release", common_cmake_options=None, support_32_bit=True, env=None, jobs="%(jobs)s", timeout=1200): llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" llvm_objdir64 = "llvm64.obj" # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb', # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".", env=merged_env)) # Clean up the source and build tree, if requested. if clean: f.addStep(ShellCommand(name="rm-llvm_llvm_srcdir", command=["rm", "-rf", llvm_srcdir], haltOnFailure=True, description=["rm src dir", "llvm"], workdir=".", env=merged_env)) f.addStep(ShellCommand(name="rm-llvm_objdir", command=["rm", "-rf", llvm_objdir], haltOnFailure=True, description=["rm build dir", "llvm"], workdir=".", env=merged_env)) f.addStep(ShellCommand(name="rm-llvm_objdir64", command=["rm", "-rf", llvm_objdir64], haltOnFailure=True, description=["rm build64 dir", "llvm"], workdir=".", env=merged_env)) # Get llvm, clang, ompiler-rt, libcxx, libcxxabi, libunwind f.addStep(SVN(name='svn-llvm', mode='update', description='svn-llvm', descriptionDone='svn-llvm', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) f.addStep(SVN(name='svn-clang', mode='update', description='svn-clang', descriptionDone='svn-clang', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir)) f.addStep(SVN(name='svn-compiler-rt', mode='update', description='svn-compiler-rt', descriptionDone='svn--compiler-rt', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', defaultBranch='trunk', workdir='%s/projects/compiler-rt' % llvm_srcdir)) f.addStep(SVN(name='svn-libcxx', mode='update', description='svn-libcxx', descriptionDone='svn-libcxx', baseURL='http://llvm.org/svn/llvm-project/libcxx/', defaultBranch='trunk', workdir='%s/projects/libcxx' % llvm_srcdir)) f.addStep(SVN(name='svn-libcxxabi', mode='update', description='svn-libcxxabi', descriptionDone='svn-libcxxabi', baseURL='http://llvm.org/svn/llvm-project/libcxxabi/', defaultBranch='trunk', workdir='%s/projects/libcxxabi' % llvm_srcdir)) f.addStep(SVN(name='svn-libunwind', mode='update', description='svn-libunwind', descriptionDone='svn-libunwind', baseURL='http://llvm.org/svn/llvm-project/libunwind/', defaultBranch='trunk', workdir='%s/projects/libunwind' % llvm_srcdir)) lint_script = os.path.join("..", llvm_srcdir, "projects", "compiler-rt", "lib", "sanitizer_common", "scripts", "check_lint.sh") # Run annotated command for sanitizer. f.addStep(AnnotatedCommand(name="lint", description="lint", timeout=timeout, haltOnFailure=True, command=lint_script, env=merged_env)) # Create configuration files with cmake. f.addStep(ShellCommand(name="create-build-dir", command=["mkdir", "-p", llvm_objdir], haltOnFailure=True, description=["create build dir"], workdir=".", env=merged_env)) # TODO: make it better way - use list for common_cmake_options and just merge. if common_cmake_options: cmakeCommand = [ "cmake", "-DCMAKE_BUILD_TYPE=%s" % build_type, "-DCMAKE_C_COMPILER=clang", "-DCMAKE_CXX_COMPILER=clang++", "-DCMAKE_CXX_FLAGS='-std=c++11 -stdlib=libc++'", "%s" % common_cmake_options, "../%s" % llvm_srcdir] else: cmakeCommand = [ "cmake", "-DCMAKE_BUILD_TYPE=%s" % build_type, "-DCMAKE_C_COMPILER=clang", "-DCMAKE_CXX_COMPILER=clang++", "-DCMAKE_CXX_FLAGS='-std=c++11 -stdlib=libc++'", "../%s" % llvm_srcdir] # Note: ShellCommand does not pass the params with special symbols correctly. # The " ".join is a workaround for this bug. f.addStep(ShellCommand(name="cmake-configure-1", description=["cmake configure, phase 1"], haltOnFailure=True, command=WithProperties(" ".join(cmakeCommand)), workdir=llvm_objdir, env=merged_env)) # Build everything. f.addStep(WarningCountingShellCommand(name="compile", command=['nice', '-n', '10', 'make', WithProperties("-j%s" % jobs)], haltOnFailure=True, description=["compiling"], descriptionDone=["compile"], workdir=llvm_objdir, env=merged_env)) # Run tests for each of the requested sanitizers. if sanity_check and sanitizers: for item in sanitizers: f.addStep(WarningCountingShellCommand(name="make-check-%s"% item, command=['make', 'check-%s' % item, WithProperties("-j%s" % jobs)], haltOnFailure=False, description=["make check-%s" % item,], descriptionDone=["make check-%s" % item,], workdir=llvm_objdir, env=merged_env)) # build 64-bit llvm using clang # Create configuration files with cmake. f.addStep(ShellCommand(name="create-llvm_build64-dir", command=["mkdir", "-p", llvm_objdir64], haltOnFailure=True, description=["create llvm_build64 dir"], workdir=".", env=merged_env)) # Use just built compiler. clang_path = "%(builddir)s" + "/%s/bin" % llvm_objdir if common_cmake_options: cmakeCommand_llvm64 = [ "cmake", "-DCMAKE_BUILD_TYPE=%s" % build_type, "%s" % common_cmake_options, "-DCMAKE_C_COMPILER=%s/clang" % clang_path, "-DCMAKE_CXX_COMPILER=%s/clang++" % clang_path, "-DCMAKE_CXX_FLAGS='-I" + "%(builddir)s" + "/%s/projects/libcxx/include -std=c++11 -stdlib=libc++'" % llvm_srcdir, "../%s" % llvm_srcdir] else: cmakeCommand_llvm64 = [ "cmake", "-DCMAKE_BUILD_TYPE=%s" % build_type, "-DCMAKE_C_COMPILER=%s/clang" % clang_path, "-DCMAKE_CXX_COMPILER=%s/clang++" % clang_path, "-DCMAKE_CXX_FLAGS='-I" + "%(builddir)s" + "/%s/projects/libcxx/include -std=c++11 -stdlib=libc++'" % llvm_srcdir, "../%s" % llvm_srcdir] # Note: ShellCommand does not pass the params with special symbols correctly. # The " ".join is a workaround for this bug. f.addStep(ShellCommand(name="cmake-configure-2", description=["cmake configure 64-bit llvm"], haltOnFailure=True, command=WithProperties(" ".join(cmakeCommand_llvm64)), workdir=llvm_objdir64, env=merged_env)) f.addStep(WarningCountingShellCommand(name="compile 64-bit llvm", command=['nice', '-n', '10', 'make', WithProperties("-j%s" % jobs)], haltOnFailure=True, description=["compiling"], descriptionDone=["compile"], workdir=llvm_objdir64, env=merged_env)) # Run tests for each of the requested sanitizers. if sanity_check and sanitizers: for item in sanitizers: f.addStep(WarningCountingShellCommand(name="make-check-%s 64-bit"% item, command=['make', 'check-%s' % item, WithProperties("-j%s" % jobs)], haltOnFailure=False, description=["make check-%s 64-bit" % item,], descriptionDone=["make check-%s" % item,], workdir=llvm_objdir64, env=merged_env)) return f
def getLLDBCMakeBuildFactory( clean=False, cmake='cmake', jobs="%(jobs)s", # Source directory containing a built python python_source_dir=None, # Default values for VS devenv and build configuration vs=None, config='Release', target_arch='x86', extra_cmake_args=None, test=False, install=False): ############# PREPARING f = buildbot.process.factory.BuildFactory() # Determine Slave Environment and Set MSVC environment. if vs: f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f = getLLDBSource(f, 'llvm') build_cmd = ['ninja'] install_cmd = ['ninja', 'install'] test_cmd = ['ninja', 'check-lldb'] if jobs: build_cmd.append(WithProperties("-j%s" % jobs)) install_cmd.append(WithProperties("-j%s" % jobs)) test_cmd.append(WithProperties("-j%s" % jobs)) # Global configurations build_dir = 'build' # get full path to build directory f.addStep( SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=build_dir)) ############# CLEANING cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) cmake_cmd = [ "cmake", "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config, "-DCMAKE_INSTALL_PREFIX=../install" ] if python_source_dir: cmake_cmd.append("-DPYTHON_HOME=" + python_source_dir) if extra_cmake_args: cmake_cmd += extra_cmake_args # Note: ShellCommand does not pass the params with special symbols right. # The " ".join is a workaround for this bug. f.addStep( ShellCommand(name="cmake-configure", description=["cmake configure"], command=WithProperties(" ".join(cmake_cmd)), haltOnFailure=True, warnOnWarnings=True, workdir=build_dir, env=Property('slave_env'))) f.addStep( WarningCountingShellCommand(name='build', command=build_cmd, haltOnFailure=True, description='ninja build', workdir=build_dir, env=Property('slave_env'))) ignoreInstallFail = bool(install != 'ignoreFail') f.addStep( ShellCommand(name='install', command=install_cmd, flunkOnFailure=ignoreInstallFail, description='ninja install', workdir=build_dir, doStepIf=bool(install), env=Property('slave_env'))) ignoreTestFail = bool(test != 'ignoreFail') f.addStep( ShellCommand(name='test', command=test_cmd, flunkOnFailure=ignoreTestFail, timeout=2400, description='ninja test', workdir=build_dir, doStepIf=bool(test), env=Property('slave_env'))) return f
def getKLEEBuildFactory(triple, jobs='%(jobs)d', llvm_branch='trunk', config_name='Release+Asserts', clean=True, llvmgccdir=None, *args, **kwargs): if False: f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".")) else: # If we are building from trunk, we need to build Clang as # well so we have access to an LLVM capable compiler. if llvm_branch == 'trunk': f = ClangBuilder.getClangBuildFactory(triple, jobs=jobs, stage1_config=config_name, extra_configure_args=['--with-built-clang', '--enable-targets=host', '--with-llvmcc=clang'], clean=clean, test=False, *args, **kwargs) else: f = LLVMBuilder.getLLVMBuildFactory(triple, jobs=jobs, defaultBranch=llvm_branch, config_name=config_name, llvmgccdir=llvmgccdir, enable_targets='x86', clean=clean, test=False, *args, **kwargs) # Checkout sources. f.addStep(SVN(name='svn-klee', mode='update', baseURL='http://llvm.org/svn/llvm-project/klee/', defaultBranch='trunk', workdir='klee')) # Configure. configure_args = ["./configure", WithProperties("--with-llvm=%(builddir)s/llvm")] configure_args += getConfigArgs(config_name) if triple: configure_args += ['--build=%s' % triple, '--host=%s' % triple, '--target=%s' % triple] f.addStep(Configure(command=configure_args, workdir='klee', description=['configure','klee',config_name])) # Clean, if requested. if clean: f.addStep(WarningCountingShellCommand(name="clean-klee", command=['make', 'clean'], haltOnFailure=True, description="clean klee", workdir='klee')) # Compile. f.addStep(WarningCountingShellCommand(name="compile", command=['nice', '-n', '10', 'make', WithProperties("-j%s" % jobs)], haltOnFailure=True, description="compile klee", workdir='klee')) # Test. f.addStep(DejaGNUCommand(name="test", command=['nice', '-n', '10', 'make', 'check'], haltOnFailure=True, description="test klee", workdir='klee', logfiles={ 'dg.sum' : 'test/testrun.sum' })) return f
def startBuild(self, cmd): if cmd.rc != 0: log.msg("Failure when running %s." % cmd) self.finished(FAILURE) else: return WarningCountingShellCommand.start(self)
def getLLDBWindowsCMakeBuildFactory( clean=False, cmake='cmake', jobs="%(jobs)s", # Source directory containing a built python python_source_dir=r'C:/src/python', # Default values for VS devenv and build configuration vs=r"""%VS120COMNTOOLS%""", config='Release', target_arch='x86', extra_cmake_args=[], test=False, install=False): ############# PREPARING f = buildbot.process.factory.BuildFactory() # Determine Slave Environment and Set MSVC environment. f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f = getLLDBSource(f, 'llvm') build_cmd = ['ninja'] install_cmd = ['ninja', 'install'] test_cmd = ['ninja', 'check-lldb'] if jobs: build_cmd.append(WithProperties("-j%s" % jobs)) install_cmd.append(WithProperties("-j%s" % jobs)) test_cmd.append(WithProperties("-j%s" % jobs)) # Global configurations build_dir = 'build' ############# CLEANING cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) if config.lower() == 'release': python_lib = 'python27.lib' python_exe = 'python.exe' elif config.lower() == 'debug': python_lib = 'python27_d.lib' python_exe = 'python_d.exe' python_lib = os.path.join(python_source_dir, 'PCbuild', python_lib) python_exe = os.path.join(python_source_dir, 'PCbuild', python_exe) python_include = os.path.join(python_source_dir, 'Include') # Use batch files instead of ShellCommand directly, Windows quoting is # borked. FIXME: See buildbot ticket #595 and buildbot ticket #377. f.addStep( batch_file_download.BatchFileDownload( name='cmakegen', command=[ cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config, # Need to use our custom built version of python '-DPYTHON_LIBRARY=' + python_lib, '-DPYTHON_INCLUDE_DIR=' + python_include, '-DPYTHON_EXECUTABLE=' + python_exe, "-DCMAKE_INSTALL_PREFIX=../install" ] + extra_cmake_args, workdir=build_dir)) f.addStep( ShellCommand(name='cmake', command=['cmakegen.bat'], haltOnFailure=True, description='cmake gen', workdir=build_dir, env=Property('slave_env'))) f.addStep( WarningCountingShellCommand(name='build', command=build_cmd, haltOnFailure=True, description='ninja build', workdir=build_dir, env=Property('slave_env'))) ignoreInstallFail = bool(install != 'ignoreFail') f.addStep( ShellCommand(name='install', command=install_cmd, flunkOnFailure=ignoreInstallFail, description='ninja install', workdir=build_dir, doStepIf=bool(install), env=Property('slave_env'))) ignoreTestFail = bool(test != 'ignoreFail') f.addStep( ShellCommand(name='test', command=test_cmd, flunkOnFailure=ignoreTestFail, description='ninja test', workdir=build_dir, doStepIf=bool(test), env=Property('slave_env'))) return f
def __init__(self, architecture=None, distribution=None, basetgz=None, mirror=None, extrapackages=None, keyring=None, components=None, **kwargs): """ Creates the DebPbuilder object. @type architecture: str @param architecture: the name of the architecture to build @type distribution: str @param distribution: the man of the distribution to use @type basetgz: str @param basetgz: the path or path template of the basetgz @type mirror: str @param mirror: the mirror for building basetgz @type extrapackages: list @param extrapackages: adds packages specified to buildroot @type keyring: str @param keyring: keyring file to use for verification @type components: str @param components: components to use for chroot creation @type kwargs: dict @param kwargs: All further keyword arguments. """ WarningCountingShellCommand.__init__(self, **kwargs) if architecture: self.architecture = architecture if distribution: self.distribution = distribution if mirror: self.mirror = mirror if extrapackages: self.extrapackages = extrapackages if keyring: self.keyring = keyring if components: self.components = components if self.architecture: kwargs['architecture'] = self.architecture else: kwargs['architecture'] = 'local' kwargs['distribution'] = self.distribution if basetgz: self.basetgz = basetgz % kwargs else: self.basetgz = self.basetgz % kwargs if not self.distribution: config.error("You must specify a distribution.") self.command = [ 'pdebuild', '--buildresult', '.', '--pbuilder', self.pbuilder ] if self.architecture: self.command += ['--architecture', self.architecture] self.command += [ '--', '--buildresult', '.', self.baseOption, self.basetgz ] if self.extrapackages: self.command += ['--extrapackages', " ".join(self.extrapackages)] self.suppressions.append( (None, re.compile(r"\.pbuilderrc does not exist"), None, None))
def getLLVMGCCBuildFactory(jobs='%(jobs)s', update=True, clean=True, gxxincludedir=None, triple=None, build=None, host=None, target=None, useTwoStage=True, stage1_config='Release+Asserts', stage2_config='Release+Asserts', make='make', extra_configure_args=[], extra_languages=None, verbose=False, env = {}, defaultBranch='trunk', timeout=20, package_dst=None): if build or host or target: if not build or not host or not target: raise ValueError,"Must specify all of 'build', 'host', 'target' if used." if triple: raise ValueError,"Cannot specify 'triple' and 'build', 'host', 'target' options." elif triple: build = host = target = triple # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything. f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir", command=["pwd"], property = "builddir", description = "set build dir", workdir = ".", env = merged_env)) # Get the sources. if update: f.addStep(SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch = defaultBranch, workdir = "llvm.src")) f.addStep(SVN(name='svn-llvm-gcc', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm-gcc-4.2/', defaultBranch = defaultBranch, workdir = "llvm-gcc.src")) # Clean up llvm (stage 1). if clean: f.addStep(ShellCommand(name="rm-llvm.obj.stage1", command=["rm", "-rf", "llvm.obj"], haltOnFailure = True, description = ["rm build dir", "llvm", "(stage 1)"], workdir = ".", env = merged_env)) # Configure llvm (stage 1). base_llvm_configure_args = [WithProperties("%(builddir)s/llvm.src/configure")] if build: base_llvm_configure_args.append('--build=' + build) base_llvm_configure_args.append('--host=' + host) base_llvm_configure_args.append('--target=' + target) stage_configure_args = getConfigArgs(stage1_config) f.addStep(Configure(name='configure.llvm.stage1', command=base_llvm_configure_args + stage_configure_args + ["--without-llvmgcc", "--without-llvmgxx"], description = [ "configure", "llvm", "(stage 1)", stage1_config ], workdir = "llvm.obj", env = merged_env)) # Build llvm (stage 1). base_llvm_make_args = ['nice', '-n', '10', make, WithProperties("-j%s" % jobs)] if verbose: base_llvm_make_args.append('VERBOSE=1') f.addStep(WarningCountingShellCommand(name = "compile.llvm.stage1", command = base_llvm_make_args, haltOnFailure = True, description = ["compile", "llvm", "(stage 1)", stage1_config], workdir = "llvm.obj", env = merged_env, timeout = timeout * 60)) # Run LLVM tests (stage 1). f.addStep(LitTestCommand(name = 'test.llvm.stage1', command = [make, "check-lit", "VERBOSE=1"], description = ["testing", "llvm"], descriptionDone = ["test", "llvm"], workdir = 'llvm.obj', env = merged_env)) # Clean up llvm-gcc. if clean: f.addStep(ShellCommand(name="rm-llvm-gcc.obj.stage1", command=["rm", "-rf", "llvm-gcc.obj"], haltOnFailure = True, description = ["rm build dir", "llvm-gcc"], workdir = ".", env = merged_env)) # Configure llvm-gcc. base_llvmgcc_configure_args = ["../llvm-gcc.src/configure"] llvmgcc_languages = "--enable-languages=c,c++" if extra_languages: llvmgcc_languages = llvmgcc_languages + "," + extra_languages base_llvmgcc_configure_args.append(llvmgcc_languages) if gxxincludedir: base_llvmgcc_configure_args.append('--with-gxx-include-dir=' + gxxincludedir) base_llvmgcc_configure_args.extend(extra_configure_args) if build: base_llvmgcc_configure_args.append('--build=' + build) base_llvmgcc_configure_args.append('--host=' + host) base_llvmgcc_configure_args.append('--target=' + target) f.addStep(Configure(name='configure.llvm-gcc.stage1', command=(base_llvmgcc_configure_args + ["--program-prefix=llvm-", WithProperties("--prefix=%(builddir)s/llvm-gcc.install"), WithProperties("--enable-llvm=%(builddir)s/llvm.obj")]), haltOnFailure = True, description = ["configure", "llvm-gcc", "(stage 1)"], workdir = "llvm-gcc.obj", env = merged_env)) # Build llvm-gcc. f.addStep(WarningCountingShellCommand(name="compile.llvm-gcc.stage1", command=['nice', '-n', '10', make, WithProperties("-j%s" % jobs)], haltOnFailure = True, description = ["compile", "llvm-gcc"], workdir = "llvm-gcc.obj", env = merged_env, timeout = timeout * 60)) # Clean up llvm-gcc install. if clean: f.addStep(ShellCommand(name="rm-llvm-gcc.install.stage1", command=["rm", "-rf", "llvm-gcc.install"], haltOnFailure = True, description = ["rm install dir", "llvm-gcc"], workdir = ".", env = merged_env)) # Install llvm-gcc. f.addStep(WarningCountingShellCommand(name="install.llvm-gcc.stage1", command=['nice', '-n', '10', make, 'install'], haltOnFailure = True, description = ["install", "llvm-gcc"], workdir = "llvm-gcc.obj", env = merged_env)) # We are done if not doing a two-stage build. if not useTwoStage: return f # Clean up llvm (stage 2). if clean: f.addStep(ShellCommand(name="rm-llvm.obj.stage2", command=["rm", "-rf", "llvm.obj.2"], haltOnFailure = True, description = ["rm build dir", "llvm", "(stage 2)"], workdir = ".", env = merged_env)) # Configure llvm (stage 2). stage_configure_args = getConfigArgs(stage2_config) local_env = dict(merged_env) local_env['CC'] = WithProperties("%(builddir)s/llvm-gcc.install/bin/llvm-gcc") local_env['CXX'] = WithProperties("%(builddir)s/llvm-gcc.install/bin/llvm-g++") f.addStep(Configure(name="configure.llvm.stage2", command=base_llvm_configure_args + stage_configure_args + [WithProperties("--with-llvmgcc=%(builddir)s/llvm-gcc.install/bin/llvm-gcc"), WithProperties("--with-llvmgxx=%(builddir)s/llvm-gcc.install/bin/llvm-g++")], haltOnFailure = True, description = ["configure", "llvm", "(stage 2)", stage2_config], workdir = "llvm.obj.2", env = local_env)) # Build LLVM (stage 2). f.addStep(WarningCountingShellCommand(name = "compile.llvm.stage2", command = base_llvm_make_args, haltOnFailure = True, description = ["compile", "llvm", "(stage 2)", stage2_config], workdir = "llvm.obj.2", env = merged_env, timeout = timeout * 60)) # Run LLVM tests (stage 2). f.addStep(LitTestCommand(name = 'test.llvm.stage2', command = [make, "check-lit", "VERBOSE=1"], description = ["testing", "llvm", "(stage 2)"], descriptionDone = ["test", "llvm", "(stage 2)"], workdir = 'llvm.obj.2', env = merged_env)) # Clean up llvm-gcc (stage 2). if clean: f.addStep(ShellCommand(name="rm-llvm-gcc.obj.stage2", command=["rm", "-rf", "llvm-gcc.obj.2"], haltOnFailure = True, description = ["rm build dir", "llvm-gcc", "(stage 2)"], workdir = ".", env = merged_env)) # Configure llvm-gcc (stage 2). local_env = dict(merged_env) local_env['CC'] = WithProperties("%(builddir)s/llvm-gcc.install/bin/llvm-gcc") local_env['CXX'] = WithProperties("%(builddir)s/llvm-gcc.install/bin/llvm-g++") f.addStep(Configure(name = 'configure.llvm-gcc.stage2', command=base_llvmgcc_configure_args + [ "--program-prefix=llvm.2-", WithProperties("--prefix=%(builddir)s/llvm-gcc.install.2"), WithProperties("--enable-llvm=%(builddir)s/llvm.obj.2")], haltOnFailure = True, description = ["configure", "llvm-gcc", "(stage 2)"], workdir = "llvm-gcc.obj.2", env = local_env)) # Build llvm-gcc (stage 2). f.addStep(WarningCountingShellCommand(name="compile.llvm-gcc.stage2", command=['nice', '-n', '10', make, WithProperties("-j%s" % jobs)], haltOnFailure = True, description = ["compile", "llvm-gcc", "(stage 2)"], workdir = "llvm-gcc.obj.2", env = merged_env, timeout = timeout * 60)) # Clean up llvm-gcc install (stage 2). if clean: f.addStep(ShellCommand(name="rm-llvm-gcc.install.stage2", command=["rm", "-rf", "llvm-gcc.install.2"], haltOnFailure = True, description = ["rm install dir", "llvm-gcc", "(stage 2)"], workdir = ".", env = merged_env)) # Install llvm-gcc. f.addStep(WarningCountingShellCommand(name="install.llvm-gcc.stage2", command = ['nice', '-n', '10', make, 'install'], haltOnFailure = True, description = ["install", "llvm-gcc", "(stage 2)"], workdir = "llvm-gcc.obj.2", env = merged_env)) if package_dst: addPackageStep(f, package_dst, obj_path = 'llvm-gcc.install.2', env = merged_env) return f
def _getClangCMakeBuildFactory( clean=True, test=True, cmake='cmake', jobs=None, # VS tools environment variable if using MSVC. For example, # %VS120COMNTOOLS% selects the 2013 toolchain. vs=None, vs_target_arch='x86', # Multi-stage compilation useTwoStage=False, testStage1=True, stage1_config='Release', stage2_config='Release', # Test-suite runTestSuite=False, nt_flags=None, testsuite_flags=None, submitURL=None, testerName=None, # Environmental variables for all steps. env=None, extra_cmake_args=None, # Extra repositories checkout_clang_tools_extra=True, checkout_compiler_rt=True, checkout_lld=True, checkout_libcxx=False, checkout_test_suite=False, checkout_flang=False, # Upload artifacts to Google Cloud Storage (for the llvmbisect tool) stage1_upload_directory=None, # Use a lower compression level to generate the build-cache package faster # default is 6 according to documentation xz_compression_factor=6, use_pixz_compression=False, # Triggers trigger_after_stage1=None): ############# PREPARING if nt_flags is None: nt_flags = [] if testsuite_flags is None: testsuite_flags = [] if env is None: env = {} if extra_cmake_args is None: extra_cmake_args = [] if trigger_after_stage1 is None: trigger_after_stage1 = [] clean_build_requested = lambda step: \ step.build.getProperty( \ "clean", \ default=step.build.getProperty("clean_obj") \ ) or clean # We *must* checkout at least Clang+LLVM depends_on_projects = ['llvm', 'clang'] if checkout_clang_tools_extra: depends_on_projects.append('clang-tools-extra') if checkout_compiler_rt: depends_on_projects.append('compiler-rt') if checkout_lld: depends_on_projects.append('lld') if checkout_libcxx: depends_on_projects.append('libcxx') depends_on_projects.append('libcxxabi') depends_on_projects.append('libunwind') if checkout_flang: depends_on_projects.append('flang') depends_on_projects.append('mlir') f = LLVMBuildFactory(depends_on_projects=depends_on_projects, llvm_srcdir='llvm') # Checkout the latest code for LNT # and the test-suite separately. Le's do this first, # so we wouldn't poison got_revision property. if runTestSuite or checkout_test_suite: f.addGetSourcecodeForProject(project='lnt', src_dir='test/lnt', alwaysUseLatest=True) f.addGetSourcecodeForProject(project='test-suite', src_dir='test/test-suite', alwaysUseLatest=True) # Then get the LLVM source code revision this particular build is for. f.addGetSourcecodeSteps() # If jobs not defined, Ninja will choose a suitable value jobs_cmd = [] lit_args = "'-v" if jobs is not None: jobs_cmd = ["-j" + str(jobs)] lit_args += " -j" + str(jobs) + "'" else: lit_args += "'" ninja_cmd = ['ninja'] + jobs_cmd ninja_install_cmd = ['ninja', 'install'] + jobs_cmd ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd # Global configurations stage1_build = 'stage1' stage1_install = 'stage1.install' stage2_build = 'stage2' stage2_install = 'stage2.install' # Set up VS environment, if appropriate. if vs and vs != "manual": f.addStep( SetProperty(command=builders_util.getVisualStudioEnvironment( vs, vs_target_arch), extract_fn=builders_util.extractSlaveEnvironment)) assert not env, "Can't have custom builder env vars with VS" env = Property('slave_env') ############# CLEANING f.addStep( ShellCommand(name='clean stage 1', command=['rm', '-rf', stage1_build], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='cleaning stage 1', descriptionDone='clean', workdir='.', doStepIf=clean_build_requested)) ############# STAGE 1 CmakeCommand.applyRequiredOptions(extra_cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage1_build) f.addStep( ShellCommand( name='cmake stage 1', command=[ cmake, "-G", "Ninja", rel_src_dir, "-DCMAKE_BUILD_TYPE=" + stage1_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args, "-DCMAKE_INSTALL_PREFIX=../" + stage1_install ] + extra_cmake_args, haltOnFailure=True, description='cmake stage 1', workdir=stage1_build, doStepIf=FileDoesNotExist("build.ninja"), env=env)) f.addStep( WarningCountingShellCommand(name='build stage 1', command=ninja_cmd, haltOnFailure=True, description='ninja all', workdir=stage1_build, env=env)) if test and testStage1: haltOnStage1Check = not useTwoStage and not runTestSuite f.addStep( LitTestCommand(name='ninja check 1', command=ninja_check_cmd, haltOnFailure=haltOnStage1Check, description=["checking stage 1"], descriptionDone=["stage 1 checked"], workdir=stage1_build, env=env)) if useTwoStage or runTestSuite or stage1_upload_directory: f.addStep( ShellCommand(name='clean stage 1 install', command=['rm', '-rf', stage1_install], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='cleaning stage 1 install', descriptionDone='clean', workdir='.')) f.addStep( ShellCommand(name='install stage 1', command=ninja_install_cmd, description='ninja install', workdir=stage1_build, env=env)) if stage1_upload_directory: addGCSUploadSteps(f, 'stage 1', stage1_install, stage1_upload_directory, env, gcs_url_property='stage1_package_gcs_url', use_pixz_compression=use_pixz_compression, xz_compression_factor=xz_compression_factor) # Compute the cmake define flag to set the C and C++ compiler to clang. Use # clang-cl if we used MSVC for stage1. if not vs: cc = 'clang' cxx = 'clang++' else: cc = 'clang-cl.exe' cxx = 'clang-cl.exe' ############# STAGE 2 if useTwoStage: # We always cleanly build the stage 2. If the compiler has been # changed on the stage 1, we cannot trust any of the intermediate file # from the old compiler. And if the stage 1 compiler is the same, we # should not build in the first place. f.addStep( ShellCommand(name='clean stage 2', command=['rm', '-rf', stage2_build], warnOnFailure=True, description='cleaning stage 2', descriptionDone='clean', workdir='.')) # Set the compiler using the CC and CXX environment variables to work around # backslash string escaping bugs somewhere between buildbot and cmake. The # env.exe helper is required to run the tests, so hopefully it's already on # PATH. rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage2_build) cmake_cmd2 = [ cmake, "-G", "Ninja", rel_src_dir, WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + stage1_install + "/bin/" + cc), WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" + stage1_install + "/bin/" + cxx), "-DCMAKE_BUILD_TYPE=" + stage2_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args, "-DCMAKE_INSTALL_PREFIX=../" + stage2_install ] + extra_cmake_args f.addStep( ShellCommand(name='cmake stage 2', command=cmake_cmd2, haltOnFailure=True, description='cmake stage 2', workdir=stage2_build, env=env)) f.addStep( WarningCountingShellCommand(name='build stage 2', command=ninja_cmd, haltOnFailure=True, description='ninja all', workdir=stage2_build, env=env)) if test: f.addStep( LitTestCommand(name='ninja check 2', command=ninja_check_cmd, haltOnFailure=not runTestSuite, description=["checking stage 2"], descriptionDone=["stage 2 checked"], workdir=stage2_build, env=env)) ############# TEST SUITE ## Test-Suite (stage 2 if built, stage 1 otherwise) if runTestSuite: compiler_path = stage1_install if useTwoStage: compiler_path = stage2_install f.addStep( ShellCommand(name='clean stage 2 install', command=['rm', '-rf', stage2_install], warnOnFailure=True, description='cleaning stage 2 install', descriptionDone='clean', workdir='.')) f.addStep( ShellCommand(name='install stage 2', command=ninja_install_cmd, description='ninja install 2', workdir=stage2_build, env=env)) # Get generated python, lnt python = WithProperties('%(workdir)s/test/sandbox/bin/python') lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt') lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py') # Paths sandbox = WithProperties('%(workdir)s/test/sandbox') test_suite_dir = WithProperties('%(workdir)s/test/test-suite') # Get latest built Clang (stage1 or stage2) cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc) cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx) # LNT Command line (don't pass -jN. Users need to pass both --threads # and --build-threads in nt_flags/test_suite_flags to get the same effect) use_runtest_testsuite = len(nt_flags) == 0 if not use_runtest_testsuite: test_suite_cmd = [ python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx ] # Append any option provided by the user test_suite_cmd.extend(nt_flags) else: lit = WithProperties('%(workdir)s/' + stage1_build + '/bin/llvm-lit') test_suite_cmd = [ python, lnt, 'runtest', 'test-suite', '--no-timestamp', '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx, '--use-lit', lit ] # Append any option provided by the user test_suite_cmd.extend(testsuite_flags) # Only submit if a URL has been specified if submitURL is not None: if not isinstance(submitURL, list): submitURL = [submitURL] for url in submitURL: test_suite_cmd.extend(['--submit', url]) # lnt runtest test-suite doesn't understand --no-machdep-info: if testerName and not use_runtest_testsuite: test_suite_cmd.extend(['--no-machdep-info', testerName]) # CC and CXX are needed as env for build-tools test_suite_env = copy.deepcopy(env) test_suite_env['CC'] = cc test_suite_env['CXX'] = cxx # Steps to prepare, build and run LNT f.addStep( ShellCommand(name='clean sandbox', command=['rm', '-rf', 'sandbox'], haltOnFailure=True, description='removing sandbox directory', workdir='test', env=env)) f.addStep( ShellCommand(name='recreate sandbox', command=['virtualenv', 'sandbox'], haltOnFailure=True, description='recreating sandbox', workdir='test', env=env)) f.addStep( ShellCommand(name='setup lit', command=[python, lnt_setup, 'develop'], haltOnFailure=True, description='setting up LNT in sandbox', workdir='test/sandbox', env=env)) f.addStep( LitTestCommand(name='test-suite', command=test_suite_cmd, haltOnFailure=True, description=['running the test suite'], workdir='test/sandbox', logfiles={ 'configure.log': 'build/configure.log', 'build-tools.log': 'build/build-tools.log', 'test.log': 'build/test.log', 'report.json': 'build/report.json' }, env=test_suite_env)) return f
def getLibompCMakeBuildFactory(clean=True, env=None, ompt=False, test=True, c_compiler="gcc", cxx_compiler="g++"): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) openmp_srcdir = "openmp.src" openmp_builddir = "openmp.build" llvm_srcdir = "llvm.src" llvm_builddir = "llvm.build" f = buildbot.process.factory.BuildFactory() # Get libomp f.addStep( SVN(name='svn-libomp', mode='update', baseURL='http://llvm.org/svn/llvm-project/openmp/', defaultBranch='trunk', workdir=openmp_srcdir)) # Get llvm to build llvm-lit f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) # Get clang if c/c++ compilers is clang/clang++ if c_compiler == "clang": f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir)) # Clean directory, if requested. if clean: f.addStep( ShellCommand(name="clean", command=["rm", "-rf", openmp_builddir, llvm_builddir], warnOnFailure=True, description=["clean"], workdir='.', env=merged_env)) # CMake llvm f.addStep( ShellCommand(name='cmake llvm', command=[ "cmake", "../" + llvm_srcdir, "-G", "Ninja", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_ENABLE_ASSERTIONS=ON", "-DCMAKE_C_COMPILER=" + c_compiler, "-DCMAKE_CXX_COMPILER=" + cxx_compiler ], haltOnFailure=True, description='cmake llvm', workdir=llvm_builddir, env=merged_env)) # Make clang build or just llvm utils build if c_compiler == "clang": f.addStep( NinjaCommand(name="make clang build", haltOnFailure=True, description=["make clang build"], workdir=llvm_builddir, env=merged_env)) else: f.addStep( NinjaCommand(name="make llvm utils build", targets=["LLVMX86Utils"], haltOnFailure=True, description=["make llvm utils build"], workdir=llvm_builddir, env=merged_env)) if ompt: f.addStep( NinjaCommand(name="make ompt test utils", targets=["FileCheck"], haltOnFailure=True, description=["make ompt test utils"], workdir=llvm_builddir, env=merged_env)) # Add clang/llvm-lit to PATH merged_env.update({ 'PATH': WithProperties("%(workdir)s/" + llvm_builddir + "/bin:" + "${PATH}") }) # CMake libomp command = [ "cmake", "../" + openmp_srcdir, "-DCMAKE_C_COMPILER=" + c_compiler, "-DCMAKE_CXX_COMPILER=" + cxx_compiler ] if ompt: command.append("-DLIBOMP_OMPT_SUPPORT=ON") f.addStep( ShellCommand(name='cmake libomp', command=command, haltOnFailure=True, description='cmake libomp', workdir=openmp_builddir, env=merged_env)) # Make libomp f.addStep( WarningCountingShellCommand(name='make libomp build', command=['make'], haltOnFailure=True, description='make libomp build', workdir=openmp_builddir, env=merged_env)) # Test, if requested if test: f.addStep( LitTestCommand(name="make check-libomp", command=["make", "check-libomp"], haltOnFailure=True, description=["make check-libomp"], descriptionDone=["build checked"], workdir=openmp_builddir, env=merged_env)) return f
def getLLDBBuildFactory(triple, useTwoStage=False, make='make', jobs='%(jobs)s', extra_configure_args=[], env={}, *args, **kwargs): llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".")) # Find out what version of llvm and clang are needed to build this version # of lldb. Right now we will assume they use the same version. # XXX - could this be done directly on the master instead of the slave? f.addStep( SetProperty( command= 'svn cat http://llvm.org/svn/llvm-project/lldb/trunk/scripts/build-llvm.pl | grep ^our.*llvm_revision | cut -d \\" -f 2', property='llvmrev')) # The SVN build step provides no mechanism to check out a specific revision # based on a property, so just run the commands directly here. svn_co = ['svn', 'checkout', '--force'] svn_co += ['--revision', WithProperties('%(llvmrev)s')] # build llvm svn checkout command svn_co_llvm = svn_co + \ [WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%(llvmrev)s'), llvm_srcdir] # build clang svn checkout command svn_co_clang = svn_co + \ [WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%(llvmrev)s'), '%s/tools/clang' % llvm_srcdir] f.addStep( ShellCommand(name='svn-llvm', command=svn_co_llvm, haltOnFailure=True, workdir='.')) f.addStep( ShellCommand(name='svn-clang', command=svn_co_clang, haltOnFailure=True, workdir='.')) f.addStep( SVN(name='svn-lldb', mode='update', baseURL='http://llvm.org/svn/llvm-project/lldb/', defaultBranch='trunk', always_purge=True, workdir='%s/tools/lldb' % llvm_srcdir)) # Run configure config_args = [ WithProperties("%%(builddir)s/%s/configure" % llvm_srcdir), "--disable-bindings", "--without-llvmgcc", "--without-llvmgxx", ] if triple: config_args += ['--build=%s' % triple] config_args += extra_configure_args f.addStep( Configure(name='configure', command=config_args, env=env, workdir=llvm_objdir)) f.addStep( WarningCountingShellCommand( name="compile", command=['nice', '-n', '10', make, WithProperties("-j%s" % jobs)], env=env, haltOnFailure=True, workdir=llvm_objdir)) # Test. f.addStep( LitTestCommand(name="test lldb", command=['nice', '-n', '10', make], description="test lldb", env=env, workdir='%s/tools/lldb/test' % llvm_objdir)) return f
def getDragonEggBootstrapFactory(gcc_repository, extra_languages=[], extra_gcc_configure_args=[], extra_llvm_configure_args=[], check_llvm=True, check_dragonegg=True, clean=True, env={}, jobs='%(jobs)s', timeout=20): # Add gcc configure arguments required by the plugin. gcc_configure_args = extra_gcc_configure_args + [ '--enable-plugin', '--enable-lto', ','.join(['--enable-languages=c,c++'] + extra_languages) ] llvm_configure_args = extra_llvm_configure_args f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( buildbot.steps.shell.SetProperty(name='get_builddir', command=['pwd'], property='builddir', description='set build dir', workdir='.', env=env)) # Checkout LLVM sources. llvm_src_dir = 'llvm.src' f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_src_dir, env=env)) # Checkout DragonEgg sources. dragonegg_src_dir = 'dragonegg.src' f.addStep( SVN(name='svn-dragonegg', mode='update', baseURL='http://llvm.org/svn/llvm-project/dragonegg/', defaultBranch='trunk', workdir=dragonegg_src_dir, env=env)) # Checkout GCC. This is usually a specific known good revision (supplied by # appending @revision to the URL). The SVN step can't handle that. As it # provides no mechanism at all for checking out a specific revision, just # run the command directly here. gcc_src_dir = 'gcc.src' svn_co = ['svn', 'checkout', gcc_repository, gcc_src_dir] f.addStep( ShellCommand(name='svn-gcc', command=svn_co, haltOnFailure=True, workdir='.', env=env)) gcc_install_dir = 'gcc.install' # Names are embedded in object files, so llvm_install_dir = 'llvm.install' # would cause bootstrap comparison fails # if they were different for each stage. # Remove any install directories hanging over from a previous run. if clean: f.addStep( ShellCommand(name='rm-%s' % gcc_install_dir, command=['rm', '-rf', gcc_install_dir], haltOnFailure=True, description=['rm install dir', 'gcc'], workdir='.', env=env)) f.addStep( ShellCommand(name='rm-%s' % llvm_install_dir, command=['rm', '-rf', llvm_install_dir], haltOnFailure=True, description=['rm install dir', 'llvm'], workdir='.', env=env)) # Do the boostrap. cur_env = env prev_gcc = None # C compiler built during the previous stage. prev_gxx = None # C++ compiler built during the previous stage. prev_plugin = None # Plugin built during the previous stage. for stage in 'stage1', 'stage2', 'stage3': # Build and install GCC. gcc_obj_dir = 'gcc.obj.%s' % stage if clean: f.addStep( ShellCommand(name='rm-%s' % gcc_obj_dir, command=['rm', '-rf', gcc_obj_dir], haltOnFailure=True, description=['rm build dir', 'gcc', stage], workdir='.', env=cur_env)) f.addStep( Configure(name='configure.gcc.%s' % stage, command=([ '../' + gcc_src_dir + '/configure', WithProperties( '--prefix=%%(builddir)s/%s' % gcc_install_dir) ] + gcc_configure_args + getCCSetting(prev_gcc, prev_gxx)), haltOnFailure=True, description=['configure', 'gcc', stage], workdir=gcc_obj_dir, env=cur_env)) f.addStep( WarningCountingShellCommand(name='compile.gcc.%s' % stage, command=[ 'nice', '-n', '10', 'make', WithProperties('-j%s' % jobs) ], haltOnFailure=True, description=['compile', 'gcc', stage], workdir=gcc_obj_dir, env=cur_env, timeout=timeout * 60)) f.addStep( WarningCountingShellCommand( name='install.gcc.%s' % stage, command=['nice', '-n', '10', 'make', 'install'], haltOnFailure=True, description=['install', 'gcc', stage], workdir=gcc_obj_dir, env=cur_env)) # From this point on build everything using the just built GCC. prev_gcc = '%(builddir)s/' + gcc_install_dir + '/bin/gcc' prev_gxx = '%(builddir)s/' + gcc_install_dir + '/bin/g++' # The built libstdc++ and libgcc may well be more recent than the system # versions. Set the library path so that programs compiled with the just # built GCC will start successfully, rather than failing due to missing # shared library dependencies. f.addStep( buildbot.steps.shell.SetProperty( name='gcc.search.paths.%s' % stage, command=[WithProperties(prev_gcc), '-print-search-dirs'], extract_fn=extractSearchPaths, haltOnFailure=True, description=['gcc', 'search paths', stage], env=cur_env)) cur_env = cur_env.copy() if 'LIBRARY_PATH' in env: cur_env['LIBRARY_PATH'] = WithProperties('%(gcc_libraries)s' + ':' + env['LIBRARY_PATH']) else: cur_env['LIBRARY_PATH'] = WithProperties('%(gcc_libraries)s') if 'LD_LIBRARY_PATH' in env: cur_env['LD_LIBRARY_PATH'] = WithProperties('%(gcc_libraries)s' + ':' + env['LD_LIBRARY_PATH']) else: cur_env['LD_LIBRARY_PATH'] = WithProperties('%(gcc_libraries)s') # Build everything using the DragonEgg plugin from the previous stage. if prev_plugin is not None: prev_gcc += ' -fplugin=' + prev_plugin prev_gxx += ' -fplugin=' + prev_plugin # Build LLVM with the just built GCC and install it. llvm_obj_dir = 'llvm.obj.%s' % stage if clean: f.addStep( ShellCommand(name='rm-%s' % llvm_obj_dir, command=['rm', '-rf', llvm_obj_dir], haltOnFailure=True, description=['rm build dir', 'llvm', stage], workdir='.', env=cur_env)) f.addStep( Configure(name='configure.llvm.%s' % stage, command=([ '../' + llvm_src_dir + '/configure', WithProperties( '--prefix=%%(builddir)s/%s' % llvm_install_dir) ] + llvm_configure_args + getCCSetting(prev_gcc, prev_gxx)), haltOnFailure=True, description=['configure', 'llvm', stage], workdir=llvm_obj_dir, env=cur_env)) f.addStep( WarningCountingShellCommand(name='compile.llvm.%s' % stage, command=[ 'nice', '-n', '10', 'make', WithProperties('-j%s' % jobs) ], haltOnFailure=True, description=['compile', 'llvm', stage], workdir=llvm_obj_dir, env=cur_env, timeout=timeout * 60)) # Optionally run the LLVM testsuite. if check_llvm: f.addStep( LitTestCommand(name='check.llvm.%s' % stage, command=[ 'nice', '-n', '10', 'make', WithProperties('LIT_ARGS=-v -j%s' % jobs), 'check-all' ], description=['test', 'llvm', stage], haltOnFailure=True, workdir=llvm_obj_dir, env=cur_env, timeout=timeout * 60)) f.addStep( WarningCountingShellCommand( name='install.llvm.%s' % stage, command=['nice', '-n', '10', 'make', 'install'], haltOnFailure=True, description=['install', 'llvm', stage], workdir=llvm_obj_dir, env=cur_env)) # Build dragonegg with the just built LLVM and GCC. dragonegg_pre_obj_dir = 'dragonegg.obj.pre.%s' % stage if clean: f.addStep( ShellCommand( name='rm-%s' % dragonegg_pre_obj_dir, command=['rm', '-rf', dragonegg_pre_obj_dir], description=['rm build dir', 'dragonegg pre', stage], haltOnFailure=True, workdir='.', env=cur_env)) f.addStep( WarningCountingShellCommand( name='compile.dragonegg.pre.%s' % stage, command=[ 'nice', '-n', '10', 'make', '-f', '../' + dragonegg_src_dir + '/Makefile', WithProperties('-j%s' % jobs), WithProperties('GCC=%(builddir)s/' + gcc_install_dir + '/bin/gcc'), WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir + '/bin/llvm-config'), WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir) ] + getCCSetting(prev_gcc, prev_gxx), haltOnFailure=True, description=['compile', 'dragonegg pre', stage], workdir=dragonegg_pre_obj_dir, env=cur_env, timeout=timeout * 60)) prev_gcc = '%(builddir)s/' + gcc_install_dir + '/bin/gcc -fplugin=%(builddir)s/' + dragonegg_pre_obj_dir + '/dragonegg.so' prev_gxx = '%(builddir)s/' + gcc_install_dir + '/bin/g++ -fplugin=%(builddir)s/' + dragonegg_pre_obj_dir + '/dragonegg.so' # Now build dragonegg again using the just built dragonegg. The build is # always done in the same directory to ensure that object files built by # the different stages should be the same (the build directory name ends # up in object files via debug info), then moved to a per-stage directory. dragonegg_bld_dir = 'dragonegg.obj' # The common build directory. dragonegg_obj_dir = 'dragonegg.obj.%s' % stage # The per-stage directory. f.addStep( ShellCommand( name='rm-%s' % dragonegg_obj_dir, command=['rm', '-rf', dragonegg_bld_dir, dragonegg_obj_dir], description=['rm build dirs', 'dragonegg', stage], haltOnFailure=True, workdir='.', env=cur_env)) f.addStep( WarningCountingShellCommand( name='compile.dragonegg.%s' % stage, command=[ 'nice', '-n', '10', 'make', '-f', '../' + dragonegg_src_dir + '/Makefile', 'DISABLE_VERSION_CHECK=1', WithProperties('-j%s' % jobs), WithProperties('GCC=%(builddir)s/' + gcc_install_dir + '/bin/gcc'), WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir + '/bin/llvm-config'), WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir) ] + getCCSetting(prev_gcc, prev_gxx), description=['compile', 'dragonegg', stage], haltOnFailure=True, workdir=dragonegg_bld_dir, env=cur_env, timeout=timeout * 60)) f.addStep( ShellCommand(name='mv-%s' % dragonegg_obj_dir, command=['mv', dragonegg_bld_dir, dragonegg_obj_dir], description=['mv build dir', 'dragonegg', stage], haltOnFailure=True, workdir='.', env=cur_env)) # Optionally run the dragonegg testsuite. if check_dragonegg: f.addStep( LitTestCommand( name='check.dragonegg.%s' % stage, command=[ 'nice', '-n', '10', 'make', '-f', '../' + dragonegg_src_dir + '/Makefile', WithProperties('GCC=%(builddir)s/' + gcc_install_dir + '/bin/gcc'), WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir + '/bin/llvm-config'), WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir), WithProperties('LIT_ARGS=-v -j%s' % jobs), 'check' ], description=['test', 'dragonegg', stage], haltOnFailure=True, workdir=dragonegg_obj_dir, env=cur_env, timeout=timeout * 60)) # Ensure that the following stages use the just built plugin. prev_plugin = '%(builddir)s/' + dragonegg_obj_dir + '/dragonegg.so' prev_gcc = '%(builddir)s/' + gcc_install_dir + '/bin/gcc -fplugin=' + prev_plugin prev_gxx = '%(builddir)s/' + gcc_install_dir + '/bin/g++ -fplugin=' + prev_plugin # Check that the dragonegg objects didn't change between stages 2 and 3. f.addStep( ShellCommand(name='compare.stages', command=[ 'sh', '-c', 'for O in *.o ; do cmp ' + '../dragonegg.obj.stage2/$O ' + '../dragonegg.obj.stage3/$O || exit 1 ; ' + 'done' ], haltOnFailure=True, description='compare stages 2 and 3', workdir='dragonegg.obj.stage3', env=cur_env)) return f
def getLLDBCMakeBuildFactory( clean=False, jobs="%(jobs)s", # Source directory containing a built python python_source_dir=None, # Default values for VS devenv and build configuration vs=None, config='Release', target_arch='x86', extra_cmake_args=None, test=False, testTimeout=2400, install=False): ############# PREPARING # Global configurations build_dir = 'build' f = LLVMBuildFactory(is_legacy_mode=False, depends_on_projects=["llvm", "clang", "lldb", "lld"], obj_dir=build_dir) # Determine Slave Environment and Set MSVC environment. if vs: f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f.addGetSourcecodeSteps() build_cmd = ['ninja'] install_cmd = ['ninja', 'install'] test_cmd = ['ninja', 'check-lldb'] if jobs: build_cmd.append(WithProperties("-j%s" % jobs)) install_cmd.append(WithProperties("-j%s" % jobs)) test_cmd.append(WithProperties("-j%s" % jobs)) ############# CLEANING cleanBuildRequested = lambda step: clean or step.build.getProperty( "clean", default=step.build.getProperty("clean_obj")) f.addStep( RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, f.obj_dir) cmake_options = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=" + config, "-DCMAKE_INSTALL_PREFIX=../install", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), ] if python_source_dir: cmake_options.append("-DPYTHON_HOME=" + python_source_dir) if extra_cmake_args: cmake_options += extra_cmake_args f.addStep( CmakeCommand(name="cmake-configure", description=["cmake configure"], haltOnFailure=True, options=cmake_options, path=rel_src_dir, env=Property('slave_env'), workdir=build_dir)) f.addStep( WarningCountingShellCommand(name='build', command=build_cmd, haltOnFailure=True, description='ninja build', workdir=build_dir, env=Property('slave_env'))) ignoreInstallFail = bool(install != 'ignoreFail') f.addStep( ShellCommand(name='install', command=install_cmd, flunkOnFailure=ignoreInstallFail, description='ninja install', workdir=build_dir, doStepIf=bool(install), env=Property('slave_env'))) ignoreTestFail = bool(test != 'ignoreFail') f.addStep( ShellCommand(name='test', command=test_cmd, flunkOnFailure=ignoreTestFail, timeout=testTimeout, description='ninja test', workdir=build_dir, doStepIf=bool(test), env=Property('slave_env'))) return f
def getDragonEggNightlyTestBuildFactory(gcc='gcc', gxx='g++', llvm_configure_args=[], testsuite_configure_args=[], xfails=[], clean=True, env={}, jobs='%(jobs)s', timeout=20): f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( buildbot.steps.shell.SetProperty(name='get_builddir', command=['pwd'], property='builddir', description='set build dir', workdir='.', env=env)) # Checkout LLVM sources. llvm_src_dir = 'llvm.src' f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_src_dir, env=env)) # Checkout DragonEgg sources. dragonegg_src_dir = 'dragonegg.src' f.addStep( SVN(name='svn-dragonegg', mode='update', baseURL='http://llvm.org/svn/llvm-project/dragonegg/', defaultBranch='trunk', workdir=dragonegg_src_dir, env=env)) # Checkout the test-suite sources. testsuite_src_dir = 'test-suite.src' f.addStep( SVN(name='svn-test-suite', mode='update', baseURL='http://llvm.org/svn/llvm-project/test-suite/', defaultBranch='trunk', workdir=testsuite_src_dir, env=env)) # Build and install LLVM. llvm_obj_dir = 'llvm.obj' llvm_install_dir = 'llvm.install' if clean: f.addStep( ShellCommand(name='rm-%s' % llvm_obj_dir, command=['rm', '-rf', llvm_obj_dir, llvm_install_dir], description='rm build dir llvm', haltOnFailure=True, workdir='.', env=env)) f.addStep( Configure( name='configure.llvm', command=([ '../' + llvm_src_dir + '/configure', WithProperties('--prefix=%%(builddir)s/%s' % llvm_install_dir) ] + llvm_configure_args), description='configuring llvm', descriptionDone='configure llvm', haltOnFailure=True, workdir=llvm_obj_dir, env=env)) f.addStep( WarningCountingShellCommand(name='compile.llvm', command=[ 'nice', '-n', '10', 'make', WithProperties('-j%s' % jobs) ], haltOnFailure=True, description='compiling llvm', descriptionDone='compile llvm', workdir=llvm_obj_dir, env=env, timeout=timeout * 60)) f.addStep( WarningCountingShellCommand( name='install.llvm', command=['nice', '-n', '10', 'make', 'install'], haltOnFailure=True, description='installing llvm', descriptionDone='install llvm', workdir=llvm_obj_dir, env=env)) # Build dragonegg with the just built LLVM. dragonegg_obj_dir = 'dragonegg.obj' if clean: f.addStep( ShellCommand(name='rm-%s' % dragonegg_obj_dir, command=['rm', '-rf', dragonegg_obj_dir], description='rm build dir dragonegg', haltOnFailure=True, workdir='.', env=env)) f.addStep( WarningCountingShellCommand( name='compile.dragonegg', command=[ 'nice', '-n', '10', 'make', '-f', '../' + dragonegg_src_dir + '/Makefile', WithProperties('-j%s' % jobs), WithProperties('GCC=' + gcc), WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir + '/bin/llvm-config'), WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir) ], haltOnFailure=True, description='compiling dragonegg', descriptionDone='compile dragonegg', workdir=dragonegg_obj_dir, env=env, timeout=timeout * 60)) # Pretend that DragonEgg is llvm-gcc by creating llvm-gcc and llvm-g++ # scripts that dispatch to gcc with dragonegg and g++ with dragonegg. if clean: f.addStep( ShellCommand(name='rm-bin', command=['rm', '-rf', 'bin'], description='rm bin dir', haltOnFailure=True, workdir='.', env=env)) f.addStep( ShellCommand( name='create.llvm-gcc.script', command=WithProperties('echo "#!/bin/sh" > llvm-gcc' '; echo "exec ' + gcc + ' -fplugin=%(builddir)s/' + dragonegg_obj_dir + '/dragonegg.so \\"\$@\\"" >> llvm-gcc' '; chmod a+x llvm-gcc'), description='create llvm-gcc script', haltOnFailure=True, workdir='bin', env=env)) f.addStep( ShellCommand( name='create.llvm-g++.script', command=WithProperties('echo "#!/bin/sh" > llvm-g++' '; echo "exec ' + gxx + ' -fplugin=%(builddir)s/' + dragonegg_obj_dir + '/dragonegg.so \\"\$@\\"" >> llvm-g++' '; chmod a+x llvm-g++'), description='create llvm-g++ script', haltOnFailure=True, workdir='bin', env=env)) # Configure the test-suite. testsuite_obj_dir = 'test-suite.obj' if clean: f.addStep( ShellCommand(name='rm-%s' % testsuite_obj_dir, command=['rm', '-rf', testsuite_obj_dir], description='rm test-suite build dir', haltOnFailure=True, workdir='.', env=env)) f.addStep( Configure( name='configure.test-suite', command=[ '../' + testsuite_src_dir + '/configure', WithProperties('--with-llvmsrc=%(builddir)s/' + llvm_src_dir), WithProperties('--with-llvmobj=%(builddir)s/' + llvm_obj_dir), WithProperties('--with-llvmgccdir=%(builddir)s/'), '--with-llvmcc=llvm-gcc', 'CC=' + gcc, 'CXX=' + gxx ] + testsuite_configure_args, description='configuring nightly test-suite', descriptionDone='configure nightly test-suite', haltOnFailure=True, workdir=testsuite_obj_dir, env=env)) # Build and test. f.addStep( ShellCommand(name='rm.test-suite.report', command=[ 'rm', '-rf', testsuite_obj_dir + '/report', testsuite_obj_dir + '/report.nightly.raw.out', testsuite_obj_dir + '/report.nightly.txt' ], description='rm test-suite report', haltOnFailure=True, workdir='.', env=env)) f.addStep( NightlyTestCommand(name='make.test-suite', command=[ 'make', WithProperties('-j%s' % jobs), 'ENABLE_PARALLEL_REPORT=1', 'DISABLE_CBE=1', 'DISABLE_JIT=1', 'RUNTIMELIMIT=%s' % (timeout * 60), 'TEST=nightly', 'report' ], logfiles={'report': 'report.nightly.txt'}, description='running nightly test-suite', descriptionDone='run nightly test-suite', haltOnFailure=True, xfails=xfails, timeout=timeout * 60, workdir=testsuite_obj_dir, env=env)) return f
def __init__(self, architecture=None, distribution=None, basetgz=None, mirror=None, extrapackages=None, keyring=None, components=None, **kwargs): """ Creates the DebPbuilder object. @type architecture: str @param architecture: the name of the architecture to build @type distribution: str @param distribution: the man of the distribution to use @type basetgz: str @param basetgz: the path or path template of the basetgz @type mirror: str @param mirror: the mirror for building basetgz @type extrapackages: list @param extrapackages: adds packages specified to buildroot @type keyring: str @param keyring: keyring file to use for verification @type components: str @param components: components to use for chroot creation @type kwargs: dict @param kwargs: All further keyword arguments. """ WarningCountingShellCommand.__init__(self, **kwargs) if architecture: self.architecture = architecture if distribution: self.distribution = distribution if mirror: self.mirror = mirror if extrapackages: self.extrapackages = extrapackages if keyring: self.keyring = keyring if components: self.components = components if self.architecture: kwargs['architecture'] = self.architecture else: kwargs['architecture'] = 'local' kwargs['distribution'] = self.distribution if basetgz: self.basetgz = basetgz % kwargs else: self.basetgz = self.basetgz % kwargs if not self.distribution: config.error("You must specify a distribution.") self.command = ['pdebuild', '--buildresult', '.', '--pbuilder', self.pbuilder] if self.architecture: self.command += ['--architecture', self.architecture] self.command += ['--', '--buildresult', '.', self.baseOption, self.basetgz] if self.extrapackages: self.command += ['--extrapackages', " ".join(self.extrapackages)] self.suppressions.append((None, re.compile(r"\.pbuilderrc does not exist"), None, None)) self.addLogObserver( 'stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
def getDragonEggTestBuildFactory(gcc='gcc', svn_testsuites=[], llvm_configure_args=[], xfails=[], clean=True, env={}, jobs='%(jobs)s', timeout=20): f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( buildbot.steps.shell.SetProperty(name='get_builddir', command=['pwd'], property='builddir', description='set build dir', workdir='.', env=env)) # Checkout LLVM sources. llvm_src_dir = 'llvm.src' f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_src_dir, env=env)) # Checkout DragonEgg sources. dragonegg_src_dir = 'dragonegg.src' f.addStep( SVN(name='svn-dragonegg', mode='update', baseURL='http://llvm.org/svn/llvm-project/dragonegg/', defaultBranch='trunk', workdir=dragonegg_src_dir, env=env)) # Checkout victims for the compilator. compilator_dir = dragonegg_src_dir + '/test/compilator' # Checkout testsuites held in subversion repositories. These are usually # specified using a known good revision (supplied by appending @revision to # the URL). The SVN step can't handle that. As it provides no mechanism at # all for checking out a specific revision, run the command directly here. for svn_testsuite in svn_testsuites: url = svn_testsuite[0] name = svn_testsuite[1] if url is None: f.addStep( ShellCommand(name='rm-%s' % name, command=['rm', '-rf', name], description=['rm', name], haltOnFailure=True, workdir=compilator_dir, env=env)) else: svn_co = ['svn', 'checkout', url, name] f.addStep( ShellCommand(name=name, command=svn_co, haltOnFailure=True, workdir=compilator_dir, env=env)) # Checkout miscellaneous testsuites. # LAPACK. f.addStep( ShellCommand( name='wget.lapack', command='wget -N http://www.netlib.org/lapack/lapack-3.4.0.tgz', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand(name='unpack.lapack', command='tar xzf lapack-3.4.0.tgz', haltOnFailure=True, workdir=compilator_dir, env=env)) # Nist Fortran 77 test suite. f.addStep( ShellCommand( name='wget.nist', command= 'wget -N http://www.itl.nist.gov/div897/ctg/suites/fcvs21.tar.Z', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand(name='unpack.nist', command='tar xzf ../fcvs21.tar.Z', haltOnFailure=True, workdir=compilator_dir + '/fcvs21/', env=env)) # Polyhedron. f.addStep( ShellCommand( name='wget.polyhedron', command= 'wget -N http://www.polyhedron.com/web_images/documents/pb11.zip', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand(name='unpack.polyhedron', command='unzip -o pb11.zip', haltOnFailure=True, workdir=compilator_dir, env=env)) # Large single compilation-unit C programs. f.addStep( ShellCommand( name='wget.bzip2', command= 'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/bzip2.c', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand( name='wget.gzip', command= 'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/gzip.c', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand(name='fix.gzip', command='sed -i "s/^static char \*$/char */" gzip.c', haltOnFailure=True, workdir=compilator_dir, env=env)) f.addStep( ShellCommand( name='wget.oggenc', command= 'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/oggenc.c', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand( name='wget.gcc', command= 'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/gcc.c.bz2', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand(name='unpack.gcc', command='bunzip2 -f -k gcc.c.bz2', haltOnFailure=True, workdir=compilator_dir, env=env)) # Build and install LLVM. llvm_obj_dir = 'llvm.obj' llvm_install_dir = 'llvm.install' if clean: f.addStep( ShellCommand(name='rm-%s' % llvm_obj_dir, command=['rm', '-rf', llvm_obj_dir, llvm_install_dir], description='rm build dir llvm', haltOnFailure=True, workdir='.', env=env)) f.addStep( Configure( name='configure.llvm', command=([ '../' + llvm_src_dir + '/configure', WithProperties('--prefix=%%(builddir)s/%s' % llvm_install_dir) ] + llvm_configure_args), description='configuring llvm', descriptionDone='configure llvm', haltOnFailure=True, workdir=llvm_obj_dir, env=env)) f.addStep( WarningCountingShellCommand(name='compile.llvm', command=[ 'nice', '-n', '10', 'make', WithProperties('-j%s' % jobs) ], haltOnFailure=True, description='compiling llvm', descriptionDone='compile llvm', workdir=llvm_obj_dir, env=env, timeout=timeout * 60)) f.addStep( WarningCountingShellCommand( name='install.llvm', command=['nice', '-n', '10', 'make', 'install'], haltOnFailure=True, description='installing llvm', descriptionDone='install llvm', workdir=llvm_obj_dir, env=env)) # Build dragonegg with the just built LLVM. dragonegg_obj_dir = 'dragonegg.obj' if clean: f.addStep( ShellCommand(name='rm-%s' % dragonegg_obj_dir, command=['rm', '-rf', dragonegg_obj_dir], description='rm build dir dragonegg', haltOnFailure=True, workdir='.', env=env)) f.addStep( WarningCountingShellCommand( name='compile.dragonegg', command=[ 'nice', '-n', '10', 'make', '-f', '../' + dragonegg_src_dir + '/Makefile', WithProperties('-j%s' % jobs), WithProperties('GCC=' + gcc), WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir + '/bin/llvm-config'), WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir) ], haltOnFailure=True, description='compiling dragonegg', descriptionDone='compile dragonegg', workdir=dragonegg_obj_dir, env=env, timeout=timeout * 60)) # Run the dragonegg testsuite. testsuite_output_dir = dragonegg_obj_dir + '/test/Output' if clean: f.addStep( ShellCommand(name='rm-%s' % testsuite_output_dir, command=['rm', '-rf', testsuite_output_dir], description='rm test-suite output directory', haltOnFailure=True, workdir='.', env=env)) f.addStep( LitTestCommand( name='make.check', command=[ 'nice', '-n', '10', 'make', '-f', '../' + dragonegg_src_dir + '/Makefile', WithProperties('GCC=' + gcc), WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir + '/bin/llvm-config'), WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir), WithProperties('LIT_ARGS=-v -j%s' % jobs), 'check' ], description='running test-suite', descriptionDone='run test-suite', haltOnFailure=True, workdir=dragonegg_obj_dir, env=env, timeout=timeout * 60)) return f