def getLLDBSource(f, llvmTopDir='llvm'): f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvmTopDir)) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvmTopDir)) f.addStep( SVN(name='svn-lld', mode='update', baseURL='http://llvm.org/svn/llvm-project/lld/', defaultBranch='trunk', workdir='%s/tools/lld' % llvmTopDir)) f.addStep( SVN(name='svn-lldb', mode='update', baseURL='http://llvm.org/svn/llvm-project/lldb/', defaultBranch='trunk', workdir='%s/tools/lldb' % llvmTopDir)) return f
def createLLVMFactory(srcdir, builddir, installdir, test_install_dir): f = factory.BuildFactory() f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=srcdir)) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % srcdir)) f.addStep( ShellCommand(command=[ '%s/configure' % srcdir, '--prefix=' + installdir, '--enable-optimized', '--enable-targets=host', '--enable-shared' ], workdir=builddir, haltOnFailure=True, name="configure", descriptionDone='configure', description='configuring')) f.addStep( ShellCommand(command=['make', '-j', '4'], workdir=builddir, haltOnFailure=True, name="compile", descriptionDone='compile', description='compiling')) f.addStep( ShellCommand(command=['make', 'check'], workdir=builddir, name='check', descriptionDone='check', haltOnFailure=True, description='checking')) f.addStep( ShellCommand(command=['make', 'install'], env={'DESTDIR': test_install_dir}, workdir=builddir, haltOnFailure=True, name='install', descriptionDone='install', description='installing')) f = createPoclFactory(llvm_dir=test_install_dir + installdir, f=f) f.addStep( ShellCommand(command=['make', 'install'], workdir=builddir, haltOnFailure=True, name='install', descriptionDone='install', description='installing')) return f
def __init__(self, svnurl=None, baseURL=None, defaultBranch=None, directory=None, workdir=None, mode='update',alwaysUseLatest=True,timeout=20*60, retry=None,**kwargs): LoggingBuildStep.__init__(self, **kwargs) SVN.__init__(self, svnurl=svnurl, baseURL=baseURL, defaultBranch=defaultBranch, directory=directory, workdir=workdir, mode=mode, alwaysUseLatest=alwaysUseLatest, timeout=timeout, retry=retry,**kwargs) self.name = 'svn-update' self.description = ["updating", "branch %s%s"%(baseURL,defaultBranch)] self.descriptionDone = ["updated", "branch %s%s"%(baseURL,defaultBranch)]
def startVC(self, branch, revision, patch): SVN.startVC(self, branch, revision, patch) for d in [self.description, self.descriptionDone]: try: d.remove("[branch]") d.append(branch) except ValueError: pass
def addSVNUpdateSteps(f, checkout_clang_tools_extra, checkout_compiler_rt, checkout_test_suite, checkout_lld, checkout_libcxx): # We *must* checkout at least Clang+LLVM f.addStep(SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir='llvm')) f.addStep(SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='llvm/tools/clang')) # Extra stuff that will be built/tested if checkout_clang_tools_extra: f.addStep(SVN(name='svn-clang-tools-extra', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', defaultBranch='trunk', workdir='llvm/tools/clang/tools/extra')) if checkout_compiler_rt: f.addStep(SVN(name='svn-compiler-rt', mode='update', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', defaultBranch='trunk', workdir='llvm/projects/compiler-rt')) if checkout_test_suite: f.addStep(SVN(name='svn-lnt', mode='update', baseURL='http://llvm.org/svn/llvm-project/lnt/', defaultBranch='trunk', workdir='test/lnt')) f.addStep(SVN(name='svn-test-suite', mode='update', baseURL='https://llvm.org/svn/llvm-project/test-suite/', defaultBranch='trunk', workdir='test/test-suite')) if checkout_lld: f.addStep(SVN(name='svn-lld', mode='update', baseURL='http://llvm.org/svn/llvm-project/lld/', defaultBranch='trunk', workdir='llvm/tools/lld')) if checkout_libcxx: f.addStep(SVN(name='svn-libcxx', mode='update', baseURL='http://llvm.org/svn/llvm-project/libcxx/', defaultBranch='trunk', workdir='llvm/projects/libcxx')) f.addStep(SVN(name='svn-libcxxabi', mode='update', baseURL='http://llvm.org/svn/llvm-project/libcxxabi/', defaultBranch='trunk', workdir='llvm/projects/libcxxabi')) f.addStep(SVN(name='svn-libunwind', mode='update', baseURL='http://llvm.org/svn/llvm-project/libunwind/', defaultBranch='trunk', workdir='llvm/projects/libunwind'))
def startVC(self, branch, revision, patch): svnurl = self.baseURL + self.branch if svnurl == branch: pass else: revision= None patch=None branch = self.branch SVN.startVC(self,self.branch, revision, patch)
def __init__(self, branch=None, **kwargs): if branch is None or branch == "trunk": svnurl = "http://code.djangoproject.com/svn/django/trunk" else: svnurl = "http://code.djangoproject.com/svn/django/branches/releases/%s" % branch kwargs["svnurl"] = svnurl kwargs["mode"] = "clobber" SVN.__init__(self, **kwargs) self.addFactoryArguments(branch=branch)
def __init__(self, repository, vcs_type=None, branch=None, username=None, password=None): factory.BuildFactory.__init__(self, []) if not repository.endswith("/"): repository += "/" if not vcs_type: if repository.startswith("https://svn."): vcs_type = "svn" elif repository.startswith("git://github.com/"): vcs_type = "git" if not branch: branch = dict(svn="trunk", git="master")[vcs_type] if vcs_type == "svn": self.addStep( SVN( baseURL=repository, defaultBranch=branch, username=username, password=password, )) elif vcs_type == "git": self.addStep(Git( repourl=repository, branch=branch, ))
def addClangGCCTests(f, ignores={}, install_prefix="%(builddir)s/llvm.install", languages=('gcc', 'g++', 'objc', 'obj-c++')): make_vars = [ WithProperties('CC_UNDER_TEST=%s/bin/clang' % install_prefix), WithProperties('CXX_UNDER_TEST=%s/bin/clang++' % install_prefix) ] f.addStep( SVN(name='svn-clang-gcc-tests', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tests/', defaultBranch='trunk', workdir='clang-tests')) gcc_dg_ignores = ignores.get('gcc-4_2-testsuite', {}) for lang in languages: f.addStep( commands.SuppressionDejaGNUCommand.SuppressionDejaGNUCommand( name='test-gcc-4_2-testsuite-%s' % lang, command=["make", "-k", "check-%s" % lang] + make_vars, description="gcc-4_2-testsuite (%s)" % lang, workdir='clang-tests/gcc-4_2-testsuite', logfiles={ 'dg.sum': 'obj/%s/%s.sum' % (lang, lang), '%s.log' % lang: 'obj/%s/%s.log' % (lang, lang) }, ignore=gcc_dg_ignores.get(lang, [])))
def addModernClangGDBTests(f, jobs, install_prefix): make_vars = [ WithProperties('RUNTESTFLAGS=CC_FOR_TARGET=\'{0}/bin/clang\' ' 'CXX_FOR_TARGET=\'{0}/bin/clang++\' ' 'CFLAGS_FOR_TARGET=\'-w -fno-limit-debug-info\''.format( install_prefix)) ] f.addStep( SVN(name='svn-clang-modern-gdb-tests', mode='update', svnurl= 'http://llvm.org/svn/llvm-project/clang-tests-external/trunk/gdb/7.5', workdir='clang-tests/src')) f.addStep( Configure(command='../src/configure', workdir='clang-tests/build/')) f.addStep( WarningCountingShellCommand( name='gdb-75-build', command=['make', WithProperties('-j%s' % jobs)], haltOnFailure=True, workdir='clang-tests/build')) f.addStep( commands.DejaGNUCommand.DejaGNUCommand( name='gdb-75-check', command=['make', '-k', WithProperties('-j%s' % jobs), 'check'] + make_vars, workdir='clang-tests/build', logfiles={ 'dg.sum': 'gdb/testsuite/gdb.sum', 'gdb.log': 'gdb/testsuite/gdb.log' }))
def AddExternalPollyBuildFactory(f, llvm_installdir, build_type="Release"): polly_srcdir = 'polly.src' polly_objdir = 'polly.obj' polly_installdir = 'polly.install' # Determine the build directory. f.addStep( buildbot.steps.shell.SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".")) f.addStep( SVN(name='svn-polly', mode='update', baseURL='http://llvm.org/svn/llvm-project/polly/', defaultBranch='trunk', workdir=polly_srcdir)) # Create configuration files with cmake f.addStep( ShellCommand(name="create-build-dir", command=["mkdir", "-p", polly_objdir], haltOnFailure=False, description=["create build dir"], workdir=".")) cmakeCommand = ["cmake", "../%s" % polly_srcdir] cmakeCommand.append('-DCMAKE_PREFIX_PATH=../%s/lib/cmake/llvm' % llvm_installdir) cmakeCommand.append('-DCMAKE_BUILD_TYPE=' + build_type) cmakeCommand.append('-DCMAKE_INSTALL_PREFIX=../' + polly_installdir) cmakeCommand.append('-DCMAKE_COLOR_MAKEFILE=OFF') f.addStep( ShellCommand(name="cmake-configure", command=cmakeCommand, haltOnFailure=False, description=["cmake configure"], workdir=polly_objdir)) # Build Polly f.addStep( ShellCommand(name="build-polly", command=["make"], haltOnFailure=True, description=["build polly"], workdir=polly_objdir)) f.addStep( ShellCommand(name="remove-polly-install", command=["rm", "-rf", polly_installdir], haltOnFailure=True, description=["remove polly install"], workdir=".")) f.addStep( ShellCommand(name="install-polly", command=["make", "install"], haltOnFailure=True, description=["install polly"], workdir=polly_objdir))
def initFactory(self,arch): f = GNUAutoconf(SVN("http://svn.cri.ensmp.fr/svn/linear/trunk"), test=None, configureFlags=self.configure_flags) if arch == "linux-64": f.addStep(FileUpload(slavesrc=os.path.join("BUILD",self.targz), masterdest=self.www+self.targz, mode=0644)) return f
def getAnnotatedBuildFactory(script, clean=False, depends_on_projects=None, env=None, timeout=1200): """ Returns a new build factory that uses AnnotatedCommand, which allows the build to be run by version-controlled scripts that do not require a buildmaster restart to update. """ f = LLVMBuildFactory(depends_on_projects=depends_on_projects, llvm_srcdir='llvm.src') if clean: f.addStep(SetProperty(property='clean', command='echo 1')) # We normally use the clean property to indicate that we want a # clean build, but AnnotatedCommand uses the clobber property # instead. Therefore, set clobber if clean is set to a truthy # value. This will cause AnnotatedCommand to set # BUILDBOT_CLOBBER=1 in the environment, which is how we # communicate to the script that we need a clean build. f.addStep( SetProperty( property='clobber', command='echo 1', doStepIf=lambda step: step.build.getProperty('clean', False))) merged_env = { 'TERM': 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set # anything. merged_env.update(env) scripts_dir = "annotated" f.addStep( SVN(name='update-annotate-scripts', mode='update', svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/' 'zorg/buildbot/builders/annotated', workdir=scripts_dir, alwaysUseLatest=True)) # Explicitly use '/' as separator, because it works on *nix and Windows. script_path = "../%s/%s" % (scripts_dir, script) f.addStep( AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, command=WithProperties( "python %(script)s --jobs=%(jobs:-)s", script=lambda _: script_path), env=merged_env)) return f
def addGetSourcecodeForProject(self, project, name=None, src_dir=None, **kwargs): # Remove 'is_legacy_mode' if it leaked in to kwargs. kwargs.pop('is_legacy_mode', None) # Bail out if we are in the legacy mode and SVN checkout is required. if self.is_legacy_mode: workdir, baseURL = svn_repos[project] if not name: name = 'svn-%s' % project # Check out to the given directory if any. # Otherwise this is a part of the unified source tree. if src_dir is None: src_dir = workdir % {'llvm_srcdir': self.llvm_srcdir} if not kwargs.get('mode', None): kwargs['mode'] = 'update' if not kwargs.get('defaultBranch', None): kwargs['defaultBranch'] = 'trunk' self.addStep( SVN(name=name, workdir=src_dir, baseURL=WithProperties(baseURL), **kwargs)) else: # project contains a repo name which is not a part of the monorepo. # We do not enforce it here, though. _repourl = kwargs.pop('repourl', None) if not _repourl: _repourl = self.repourl_prefix + "llvm-%s.git" % project if not name: name = 'Checkout %s' % project # Check out to the given directory if any. # Otherwise this is a part of the unified source tree. if src_dir is None: src_dir = 'llvm-%s' % project # Ignore workdir if given. We check out to src_dir. kwargs.pop('workdir', None) self.addStep( Git(name=name, repourl=_repourl, progress=True, workdir=WithProperties(src_dir), **kwargs))
def __init__(self, svnurl, configure=None, configureEnv={}, compile="make all", test="make check"): source = SVN(svnurl=svnurl, mode="update") GNUAutoconf.__init__(self, source, configure=configure, configureEnv=configureEnv, compile=compile, test=test)
def initFactory(self,arch): self.make = "gmake" if arch == "bsd-64" else "make" f = GNUAutoconf( SVN("http://svn.cri.ensmp.fr/svn/pips/trunk"), configureFlags=self.configure_flags, compile=[self.make, "all"], test=None, install=[self.make, "install"], distcheck=None #[self.make, "distcheck"] ) if arch == "linux-64": f.addStep(FileUpload(slavesrc=os.path.join("BUILD",self.targz), masterdest=self.www+self.targz, mode=0644)) return f
def addClangGDBTests(f, ignores={}, install_prefix="%(builddir)s/llvm.install"): make_vars = [WithProperties( 'CC_UNDER_TEST=%s/bin/clang' % install_prefix), WithProperties( 'CXX_UNDER_TEST=%s/bin/clang++' % install_prefix)] f.addStep(SVN(name='svn-clang-gdb-tests', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tests/', defaultBranch='trunk', workdir='clang-tests')) f.addStep(commands.SuppressionDejaGNUCommand.SuppressionDejaGNUCommand( name='test-gdb-1472-testsuite', command=["make", "-k", "check"] + make_vars, description="gdb-1472-testsuite", workdir='clang-tests/gdb-1472-testsuite', logfiles={ 'dg.sum' : 'obj/filtered.gdb.sum', 'gdb.log' : 'obj/gdb.log' }))
def getSanitizerBuildFactory( clean=False, env=None, timeout=1200): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { "TERM" : "dumb", # Make sure Clang doesn't use color escape sequences. } # Use env variables defined in the system. merged_env.update(os.environ) # Clobber bot if we need a clean build. if clean: merged_env["BUILDBOT_CLOBBER"] = "1" # Overwrite pre-set items with the given ones, so user can set anything. if env is not None: merged_env.update(env) f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".", env=merged_env)) sanitizer_buildbot = "sanitizer_buildbot" sanitizer_script_dir = os.path.join(sanitizer_buildbot, "sanitizers") # Get sanitizer buildbot scripts. f.addStep(SVN(name='svn-sanitizer-buildbot', mode='update', svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/' 'zorg/buildbot/builders/sanitizers', workdir=sanitizer_script_dir, alwaysUseLatest=True)) sanitizer_script = os.path.join("..", sanitizer_script_dir, "buildbot_selector.py") # Run annotated command for sanitizer. f.addStep(AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, command="python " + sanitizer_script, env=merged_env)) return f
def getGlibcAnnotatedFactory( clean=False, env=None, timeout=1200): merged_env = {} # Use env variables defined in the system. merged_env.update(os.environ) # Clobber bot if we need a clean build. if clean: merged_env['BUILDBOT_CLOBBER'] = '1' # Overwrite pre-set items with the given ones, so user can set anything. if env is not None: merged_env.update(env) f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep(buildbot.steps.shell.SetProperty(name='get_builddir', command=['pwd'], property='builddir', description='set build dir', workdir='.', env=merged_env)) # Get buildbot scripts. f.addStep(SVN(name='update scripts', mode='update', svnurl='http://address-sanitizer.googlecode.com/svn/trunk/' 'glibc_buildbot/scripts', workdir='scripts', alwaysUseLatest=True)) selector_script = os.path.join('..', 'scripts', 'slave', 'buildbot_selector.py') # Run annotated command. f.addStep(AnnotatedCommand(name='annotate', description='annotate', timeout=timeout, haltOnFailure=True, command='python ' + selector_script, env=merged_env)) return f
def addSVNSteps(self, llvm_srcdir=None, **kwargs): if llvm_srcdir is None: llvm_srcdir = self.llvm_srcdir if not kwargs.get('mode', None): kwargs['mode'] = 'update' if not kwargs.get('defaultBranch', None): kwargs['defaultBranch'] = 'trunk' # Add a SVM step for each project this builder depends on. # We want the projects be always checked out in a certain order. for project in svn_repos.keys(): if project in self.depends_on_projects: workdir, baseURL = svn_repos[project] self.addStep( SVN(name='svn-%s' % project, workdir=workdir % {'llvm_srcdir' : llvm_srcdir}, baseURL=WithProperties(baseURL), **kwargs))
def addRepository(self, project=None, repository=None, vcs_type=None, branch=None, username=None, password=None, **kwargs): kwargs = dict(kwargs) if not repository.endswith("/"): repository += "/" if not vcs_type: if repository.startswith("https://svn."): vcs_type = "svn" elif repository.startswith("git://github.com/"): vcs_type = "git" if not branch: branch = dict(svn="trunk", git="master")[vcs_type] if vcs_type == "svn": kwargs.update( dict( baseURL=repository, defaultBranch=branch, username=username, password=password, codebase=project, )) self.addStep(SVN(**kwargs)) elif vcs_type == "git": kwargs.update( dict( repourl=repository, branch=branch, codebase=project, )) self.addStep(Git(**kwargs))
def initFactory(self,arch): if arch != "linux-64": self.make="gmake" else: self.make="make" f = factory.BuildFactory() f.addStep(SVN("http://svn.cri.ensmp.fr/svn/validation/trunk")) s=PipsValid(command=[self.make, 'validate', # 'TARGET=TutorialPPoPP2010', ], logfiles={'summary':'SUMMARY.short'}, env={ 'PATH':'{0}:{1}'.format(os.environ['PATH'],os.path.join(self.install_prefix,"bin")), 'PYTHONPATH':'{0}'.format(os.path.join(self.install_prefix,"lib","python2.6","site-packages","pips")), 'LD_LIBRARY_PATH':'{0}'.format(os.path.join(self.install_prefix,"lib")), 'PIPS_F77':'gfortran', 'PIPS_F90':'gfortran', }) f.addStep(s) return f
def getPollyBuildFactory(): llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" polly_srcdir = '%s/tools/polly' % llvm_srcdir clang_srcdir = '%s/tools/clang' % llvm_srcdir f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( buildbot.steps.shell.SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".")) # Get LLVM, clang and Polly f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir=clang_srcdir)) f.addStep( SVN(name='svn-polly', mode='update', baseURL='http://llvm.org/svn/llvm-project/polly/', defaultBranch='trunk', workdir=polly_srcdir)) # Create configuration files with cmake f.addStep( ShellCommand(name="create-build-dir", command=["mkdir", "-p", llvm_objdir], haltOnFailure=False, description=["create build dir"], workdir=".")) cmakeCommand = [ "cmake", "../%s" % llvm_srcdir, "-DCMAKE_COLOR_MAKEFILE=OFF", "-DPOLLY_TEST_DISABLE_BAR=ON", "-DCMAKE_BUILD_TYPE=Release" ] f.addStep( ShellCommand(name="cmake-configure", command=cmakeCommand, haltOnFailure=False, description=["cmake configure"], workdir=llvm_objdir)) # Build Polly f.addStep( ShellCommand(name="build_polly", command=["make"], haltOnFailure=True, description=["build polly"], workdir=llvm_objdir)) # Test Polly f.addStep( ShellCommand(name="test_polly", command=["make", "polly-test"], haltOnFailure=True, description=["test polly"], workdir=llvm_objdir)) # Check formatting f.addStep( ShellCommand(name="test_polly_format", command=["make", "polly-check-format"], haltOnFailure=False, description=["Check formatting"], workdir=llvm_objdir)) return f
def getLLDBxcodebuildFactory(use_cc=None, build_type='Debug', remote_configs=None, env=None): if env is None: env = {} f = buildbot.process.factory.BuildFactory() f.addStep( SetProperty(name='get_builddir', command=['pwd'], property='builddir', description='set build dir', workdir='.')) lldb_srcdir = 'lldb' OBJROOT = '%(builddir)s/' + lldb_srcdir + '/build' f.addStep( SetProperty(name='get_bindir', command=[ 'echo', WithProperties('%(builddir)s/' + lldb_srcdir + '/build/' + build_type) ], property='lldb_bindir', description='set bin dir', workdir='.')) # cleaning out the build directory is vital for codesigning. f.addStep( ShellCommand(name='clean.lldb-buid', command=['rm', '-rf', WithProperties(OBJROOT)], haltOnFailure=True, workdir=WithProperties('%(builddir)s'))) f.addStep( ShellCommand(name='clean.llvm-buid', command=['rm', '-rf', '%s/llvm-build' % lldb_srcdir], haltOnFailure=True, workdir=WithProperties('%(builddir)s'))) f.addStep( ShellCommand(name='clean.test trace', command='rm -rf build/*', haltOnFailure=False, flunkOnFailure=False, workdir='.')) # Remove symbolic link to lldb, otherwise xcodebuild will have circular dependency f.addStep( ShellCommand(name='remove symbolic link lldb', command=['rm', lldb_srcdir + '/llvm/tools/lldb'], haltOnFailure=False, flunkOnFailure=False, workdir=WithProperties('%(builddir)s'))) f.addStep( SVN(name='svn-lldb', mode='update', baseURL='http://llvm.org/svn/llvm-project/lldb/', defaultBranch='trunk', workdir=lldb_srcdir)) f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir='%s/llvm' % lldb_srcdir)) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/llvm/tools/clang' % lldb_srcdir)) # setup keychain for codesign # In order for the codesigning to work inside of buildbot, security must be # called to unlock the keychain, which requires a password. # I've set up a special keychain for this purpose, so as to not compromise # the login password of the buildslave. # This means I have to set the special keychain as the default and unlock it # prior to building the sources. f.addStep( ShellCommand(name='check.keychain', command=['security', 'default-keychain'], haltOnFailure=True, workdir=WithProperties('%(builddir)s'))) f.addStep( ShellCommand( name='find.certificate', command=['security', 'find-certificate', '-c', 'lldb_codesign'], haltOnFailure=True, workdir=WithProperties('%(builddir)s'))) # Building the sources # buildcmd = ' '.join([ 'xcrun', 'xcodebuild', '-target', 'desktop', '-configuration', build_type, 'SYMROOT=' + OBJROOT, 'OBJROOT=' + OBJROOT ]) f.addStep( ShellCommand(name='lldb-build', command=WithProperties(buildcmd + " || " + buildcmd), haltOnFailure=True, workdir=lldb_srcdir)) # Testing # if not use_cc: use_cc = '/Applications/Xcode.app/Contents/Developer/Toolchains/' use_cc += 'XcodeDefault.xctoolchain/usr/bin/clang' f.addStep( SetProperty(name='set.cc', command=['xcrun', '-find', 'clang'], property='use_cc', description='set cc', workdir=lldb_srcdir)) else: f.addStep( SetProperty(name='set.cc', command=['echo', use_cc], property='use_cc', description='set cc', workdir=lldb_srcdir)) DOTEST_OPTS = ' '.join([ '--executable', '%(lldb_bindir)s/lldb', '--filecheck', '%(lldb_bindir)s/FileCheck', '--framework', '%(lldb_bindir)s/LLDB.framework', '-A', 'x86_64', '-C', 'clang', '-s', '../../build/lldb-test-traces' ]) f.addStep( LitTestCommand( name='lldb-test', command=['./dosep.py', '--options', WithProperties(DOTEST_OPTS)], haltOnFailure=False, workdir='%s/test' % lldb_srcdir, env={'DYLD_FRAMEWORK_PATH': WithProperties('%(lldb_bindir)s')})) # Remote test steps if remote_configs is not None: # Source structure to use cmake command f.addStep( SetProperty( name='get tblgen bindir', command=[ 'echo', WithProperties( '%(builddir)s/' + lldb_srcdir + '/llvm-build/Release+Asserts/x86_64/Release+Asserts/bin' ) ], property='tblgen_bindir', description='set tblgen binaries dir', workdir='.')) f = getSymbLinkSteps(f, lldb_srcdir) for config in remote_configs: f = getLLDBRemoteTestSteps( f, '%(lldb_bindir)s', build_type, config, env={'DYLD_FRAMEWORK_PATH': WithProperties('%(lldb_bindir)s')}) # Compress and upload test log f = archiveLLDBTestTraces(f, "build/lldb-test-traces*") # Results go in a directory coded named according to the date and time of the test run, e.g.: # # 2012-10-16-11_26_48/Failure-x86_64-_Applications_Xcode.app_Contents_Developer_Toolchains_XcodeDefault.xctoolchain_usr_bin_clang-TestLogging.LogTestCase.test_with_dsym.log # # Possible results are ExpectedFailure, Failure, SkippedTest, UnexpectedSuccess, and Error. return f return f
def getDragonEggNightlyTestBuildFactory(gcc='gcc', gxx='g++', llvm_configure_args=[], testsuite_configure_args=[], xfails=[], clean=True, env={}, jobs='%(jobs)s', timeout=20): f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( buildbot.steps.shell.SetProperty(name='get_builddir', command=['pwd'], property='builddir', description='set build dir', workdir='.', env=env)) # Checkout LLVM sources. llvm_src_dir = 'llvm.src' f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_src_dir, env=env)) # Checkout DragonEgg sources. dragonegg_src_dir = 'dragonegg.src' f.addStep( SVN(name='svn-dragonegg', mode='update', baseURL='http://llvm.org/svn/llvm-project/dragonegg/', defaultBranch='trunk', workdir=dragonegg_src_dir, env=env)) # Checkout the test-suite sources. testsuite_src_dir = 'test-suite.src' f.addStep( SVN(name='svn-test-suite', mode='update', baseURL='http://llvm.org/svn/llvm-project/test-suite/', defaultBranch='trunk', workdir=testsuite_src_dir, env=env)) # Build and install LLVM. llvm_obj_dir = 'llvm.obj' llvm_install_dir = 'llvm.install' if clean: f.addStep( ShellCommand(name='rm-%s' % llvm_obj_dir, command=['rm', '-rf', llvm_obj_dir, llvm_install_dir], description='rm build dir llvm', haltOnFailure=True, workdir='.', env=env)) f.addStep( Configure( name='configure.llvm', command=([ '../' + llvm_src_dir + '/configure', WithProperties('--prefix=%%(builddir)s/%s' % llvm_install_dir) ] + llvm_configure_args), description='configuring llvm', descriptionDone='configure llvm', haltOnFailure=True, workdir=llvm_obj_dir, env=env)) f.addStep( WarningCountingShellCommand(name='compile.llvm', command=[ 'nice', '-n', '10', 'make', WithProperties('-j%s' % jobs) ], haltOnFailure=True, description='compiling llvm', descriptionDone='compile llvm', workdir=llvm_obj_dir, env=env, timeout=timeout * 60)) f.addStep( WarningCountingShellCommand( name='install.llvm', command=['nice', '-n', '10', 'make', 'install'], haltOnFailure=True, description='installing llvm', descriptionDone='install llvm', workdir=llvm_obj_dir, env=env)) # Build dragonegg with the just built LLVM. dragonegg_obj_dir = 'dragonegg.obj' if clean: f.addStep( ShellCommand(name='rm-%s' % dragonegg_obj_dir, command=['rm', '-rf', dragonegg_obj_dir], description='rm build dir dragonegg', haltOnFailure=True, workdir='.', env=env)) f.addStep( WarningCountingShellCommand( name='compile.dragonegg', command=[ 'nice', '-n', '10', 'make', '-f', '../' + dragonegg_src_dir + '/Makefile', WithProperties('-j%s' % jobs), WithProperties('GCC=' + gcc), WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir + '/bin/llvm-config'), WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir) ], haltOnFailure=True, description='compiling dragonegg', descriptionDone='compile dragonegg', workdir=dragonegg_obj_dir, env=env, timeout=timeout * 60)) # Pretend that DragonEgg is llvm-gcc by creating llvm-gcc and llvm-g++ # scripts that dispatch to gcc with dragonegg and g++ with dragonegg. if clean: f.addStep( ShellCommand(name='rm-bin', command=['rm', '-rf', 'bin'], description='rm bin dir', haltOnFailure=True, workdir='.', env=env)) f.addStep( ShellCommand( name='create.llvm-gcc.script', command=WithProperties('echo "#!/bin/sh" > llvm-gcc' '; echo "exec ' + gcc + ' -fplugin=%(builddir)s/' + dragonegg_obj_dir + '/dragonegg.so \\"\$@\\"" >> llvm-gcc' '; chmod a+x llvm-gcc'), description='create llvm-gcc script', haltOnFailure=True, workdir='bin', env=env)) f.addStep( ShellCommand( name='create.llvm-g++.script', command=WithProperties('echo "#!/bin/sh" > llvm-g++' '; echo "exec ' + gxx + ' -fplugin=%(builddir)s/' + dragonegg_obj_dir + '/dragonegg.so \\"\$@\\"" >> llvm-g++' '; chmod a+x llvm-g++'), description='create llvm-g++ script', haltOnFailure=True, workdir='bin', env=env)) # Configure the test-suite. testsuite_obj_dir = 'test-suite.obj' if clean: f.addStep( ShellCommand(name='rm-%s' % testsuite_obj_dir, command=['rm', '-rf', testsuite_obj_dir], description='rm test-suite build dir', haltOnFailure=True, workdir='.', env=env)) f.addStep( Configure( name='configure.test-suite', command=[ '../' + testsuite_src_dir + '/configure', WithProperties('--with-llvmsrc=%(builddir)s/' + llvm_src_dir), WithProperties('--with-llvmobj=%(builddir)s/' + llvm_obj_dir), WithProperties('--with-llvmgccdir=%(builddir)s/'), '--with-llvmcc=llvm-gcc', 'CC=' + gcc, 'CXX=' + gxx ] + testsuite_configure_args, description='configuring nightly test-suite', descriptionDone='configure nightly test-suite', haltOnFailure=True, workdir=testsuite_obj_dir, env=env)) # Build and test. f.addStep( ShellCommand(name='rm.test-suite.report', command=[ 'rm', '-rf', testsuite_obj_dir + '/report', testsuite_obj_dir + '/report.nightly.raw.out', testsuite_obj_dir + '/report.nightly.txt' ], description='rm test-suite report', haltOnFailure=True, workdir='.', env=env)) f.addStep( NightlyTestCommand(name='make.test-suite', command=[ 'make', WithProperties('-j%s' % jobs), 'ENABLE_PARALLEL_REPORT=1', 'DISABLE_CBE=1', 'DISABLE_JIT=1', 'RUNTIMELIMIT=%s' % (timeout * 60), 'TEST=nightly', 'report' ], logfiles={'report': 'report.nightly.txt'}, description='running nightly test-suite', descriptionDone='run nightly test-suite', haltOnFailure=True, xfails=xfails, timeout=timeout * 60, workdir=testsuite_obj_dir, env=env)) return f
def getLLDBBuildFactory(triple, useTwoStage=False, make='make', jobs='%(jobs)s', extra_configure_args=[], env={}, *args, **kwargs): llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".")) # Find out what version of llvm and clang are needed to build this version # of lldb. Right now we will assume they use the same version. # XXX - could this be done directly on the master instead of the slave? f.addStep( SetProperty( command= 'svn cat http://llvm.org/svn/llvm-project/lldb/trunk/scripts/build-llvm.pl | grep ^our.*llvm_revision | cut -d \\" -f 2', property='llvmrev')) # The SVN build step provides no mechanism to check out a specific revision # based on a property, so just run the commands directly here. svn_co = ['svn', 'checkout', '--force'] svn_co += ['--revision', WithProperties('%(llvmrev)s')] # build llvm svn checkout command svn_co_llvm = svn_co + \ [WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%(llvmrev)s'), llvm_srcdir] # build clang svn checkout command svn_co_clang = svn_co + \ [WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%(llvmrev)s'), '%s/tools/clang' % llvm_srcdir] f.addStep( ShellCommand(name='svn-llvm', command=svn_co_llvm, haltOnFailure=True, workdir='.')) f.addStep( ShellCommand(name='svn-clang', command=svn_co_clang, haltOnFailure=True, workdir='.')) f.addStep( SVN(name='svn-lldb', mode='update', baseURL='http://llvm.org/svn/llvm-project/lldb/', defaultBranch='trunk', always_purge=True, workdir='%s/tools/lldb' % llvm_srcdir)) # Run configure config_args = [ WithProperties("%%(builddir)s/%s/configure" % llvm_srcdir), "--disable-bindings", "--without-llvmgcc", "--without-llvmgxx", ] if triple: config_args += ['--build=%s' % triple] config_args += extra_configure_args f.addStep( Configure(name='configure', command=config_args, env=env, workdir=llvm_objdir)) f.addStep( WarningCountingShellCommand( name="compile", command=['nice', '-n', '10', make, WithProperties("-j%s" % jobs)], env=env, haltOnFailure=True, workdir=llvm_objdir)) # Test. f.addStep( LitTestCommand(name="test lldb", command=['nice', '-n', '10', make], description="test lldb", env=env, workdir='%s/tools/lldb/test' % llvm_objdir)) return f
def getToolchainBuildFactory( clean=False, test=True, env=None, # Environmental variables for all steps. extra_configure_args=None): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { "TERM" : "dumb", # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) src_dir = "llvm.src" obj_dir = "llvm.obj" install_dir = "llvm.install" f = LLVMBuildFactory( depends_on_projects=[ "llvm", "clang", "clang-tools-extra", "compiler-rt", "libcxx", "libcxxabi", "libunwind", "lld" ]) # Get Fuchsia SDK. sdk_dir = "fuchsia.sdk" sdk_platform = { "Linux": "linux-amd64", "Darwin": "mac-amd64", }[platform.system()] sdk_version = "latest" sdk_url = WithProperties( "https://chrome-infra-packages.appspot.com/dl/" "fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s", sdk_platform=lambda _: sdk_platform, sdk_version=lambda _: sdk_version) f.addStep(RemoveDirectory(name="clean-sdk", dir=sdk_dir, haltOnFailure=True)) f.addStep(ShellCommand(name="fetch-sdk", command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url], description=["download", "fuchsia sdk"], workdir=sdk_dir)) f.addStep(ShellCommand(name="extract-sdk", command=["unzip", "-fo", "sdk.cipd"], description=["extract", "fuchsia sdk"], workdir=sdk_dir)) cleanCheckoutRequested = lambda step: step.build.getProperty("clean", default=False) or clean # Clean up llvm sources. f.addStep(RemoveDirectory(name="clean-llvm.src", dir=src_dir, haltOnFailure=True, doStepIf=cleanCheckoutRequested)) # Get sources. for project in f.depends_on_projects: _, baseURL = svn_repos[project] f.addStep(SVN(name="svn-%s" % project, workdir=src_dir + "/" + project, baseURL=baseURL)) cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean # Clean up llvm build. f.addStep(RemoveDirectory(name="clean-llvm.obj", dir=obj_dir, haltOnFailure=True, doStepIf=cleanBuildRequested)) # Configure. if extra_configure_args is None: cmake_options = [] else: cmake_options = extra_configure_args[:] # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_options, [ ("-G", "Ninja"), ("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"), ("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"), ]) # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_options, [ ("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"), ("-DLLVM_ENABLE_LTO=", "OFF"), ]) cmake_options.append( WithProperties( "-DCMAKE_INSTALL_PREFIX=%(workdir)s/" + install_dir )) cmake_options.append( WithProperties( "-DFUCHSIA_SDK=%(workdir)s/" + sdk_dir )) CmakeCommand.applyRequiredOptions(cmake_options, [ ("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"), ]) f.addStep(CmakeCommand(name="cmake-configure", options=cmake_options, path='../' + src_dir + '/llvm', haltOnFailure=True, description=["configure"], workdir=obj_dir, env=merged_env, doStepIf=FileDoesNotExist("CMakeCache.txt"))) # Build distribution. f.addStep(NinjaCommand(name="ninja-build", targets=["stage2-distribution"], haltOnFailure=True, description=["build"], workdir=obj_dir, env=merged_env)) # Test llvm, clang and lld. f.addStep(NinjaCommand(name="check", targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")], haltOnFailure=True, description=["check"], workdir=obj_dir, env=merged_env, doStepIf=test)) # Install distribution. f.addStep(NinjaCommand(name="install", targets=["stage2-install-distribution"], haltOnFailure=True, description=["install"], workdir=obj_dir, env=merged_env)) return f
def getDragonEggTestBuildFactory(gcc='gcc', svn_testsuites=[], llvm_configure_args=[], xfails=[], clean=True, env={}, jobs='%(jobs)s', timeout=20): f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( buildbot.steps.shell.SetProperty(name='get_builddir', command=['pwd'], property='builddir', description='set build dir', workdir='.', env=env)) # Checkout LLVM sources. llvm_src_dir = 'llvm.src' f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_src_dir, env=env)) # Checkout DragonEgg sources. dragonegg_src_dir = 'dragonegg.src' f.addStep( SVN(name='svn-dragonegg', mode='update', baseURL='http://llvm.org/svn/llvm-project/dragonegg/', defaultBranch='trunk', workdir=dragonegg_src_dir, env=env)) # Checkout victims for the compilator. compilator_dir = dragonegg_src_dir + '/test/compilator' # Checkout testsuites held in subversion repositories. These are usually # specified using a known good revision (supplied by appending @revision to # the URL). The SVN step can't handle that. As it provides no mechanism at # all for checking out a specific revision, run the command directly here. for svn_testsuite in svn_testsuites: url = svn_testsuite[0] name = svn_testsuite[1] if url is None: f.addStep( ShellCommand(name='rm-%s' % name, command=['rm', '-rf', name], description=['rm', name], haltOnFailure=True, workdir=compilator_dir, env=env)) else: svn_co = ['svn', 'checkout', url, name] f.addStep( ShellCommand(name=name, command=svn_co, haltOnFailure=True, workdir=compilator_dir, env=env)) # Checkout miscellaneous testsuites. # LAPACK. f.addStep( ShellCommand( name='wget.lapack', command='wget -N http://www.netlib.org/lapack/lapack-3.4.0.tgz', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand(name='unpack.lapack', command='tar xzf lapack-3.4.0.tgz', haltOnFailure=True, workdir=compilator_dir, env=env)) # Nist Fortran 77 test suite. f.addStep( ShellCommand( name='wget.nist', command= 'wget -N http://www.itl.nist.gov/div897/ctg/suites/fcvs21.tar.Z', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand(name='unpack.nist', command='tar xzf ../fcvs21.tar.Z', haltOnFailure=True, workdir=compilator_dir + '/fcvs21/', env=env)) # Polyhedron. f.addStep( ShellCommand( name='wget.polyhedron', command= 'wget -N http://www.polyhedron.com/web_images/documents/pb11.zip', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand(name='unpack.polyhedron', command='unzip -o pb11.zip', haltOnFailure=True, workdir=compilator_dir, env=env)) # Large single compilation-unit C programs. f.addStep( ShellCommand( name='wget.bzip2', command= 'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/bzip2.c', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand( name='wget.gzip', command= 'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/gzip.c', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand(name='fix.gzip', command='sed -i "s/^static char \*$/char */" gzip.c', haltOnFailure=True, workdir=compilator_dir, env=env)) f.addStep( ShellCommand( name='wget.oggenc', command= 'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/oggenc.c', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand( name='wget.gcc', command= 'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/gcc.c.bz2', haltOnFailure=False, workdir=compilator_dir, env=env)) f.addStep( ShellCommand(name='unpack.gcc', command='bunzip2 -f -k gcc.c.bz2', haltOnFailure=True, workdir=compilator_dir, env=env)) # Build and install LLVM. llvm_obj_dir = 'llvm.obj' llvm_install_dir = 'llvm.install' if clean: f.addStep( ShellCommand(name='rm-%s' % llvm_obj_dir, command=['rm', '-rf', llvm_obj_dir, llvm_install_dir], description='rm build dir llvm', haltOnFailure=True, workdir='.', env=env)) f.addStep( Configure( name='configure.llvm', command=([ '../' + llvm_src_dir + '/configure', WithProperties('--prefix=%%(builddir)s/%s' % llvm_install_dir) ] + llvm_configure_args), description='configuring llvm', descriptionDone='configure llvm', haltOnFailure=True, workdir=llvm_obj_dir, env=env)) f.addStep( WarningCountingShellCommand(name='compile.llvm', command=[ 'nice', '-n', '10', 'make', WithProperties('-j%s' % jobs) ], haltOnFailure=True, description='compiling llvm', descriptionDone='compile llvm', workdir=llvm_obj_dir, env=env, timeout=timeout * 60)) f.addStep( WarningCountingShellCommand( name='install.llvm', command=['nice', '-n', '10', 'make', 'install'], haltOnFailure=True, description='installing llvm', descriptionDone='install llvm', workdir=llvm_obj_dir, env=env)) # Build dragonegg with the just built LLVM. dragonegg_obj_dir = 'dragonegg.obj' if clean: f.addStep( ShellCommand(name='rm-%s' % dragonegg_obj_dir, command=['rm', '-rf', dragonegg_obj_dir], description='rm build dir dragonegg', haltOnFailure=True, workdir='.', env=env)) f.addStep( WarningCountingShellCommand( name='compile.dragonegg', command=[ 'nice', '-n', '10', 'make', '-f', '../' + dragonegg_src_dir + '/Makefile', WithProperties('-j%s' % jobs), WithProperties('GCC=' + gcc), WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir + '/bin/llvm-config'), WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir) ], haltOnFailure=True, description='compiling dragonegg', descriptionDone='compile dragonegg', workdir=dragonegg_obj_dir, env=env, timeout=timeout * 60)) # Run the dragonegg testsuite. testsuite_output_dir = dragonegg_obj_dir + '/test/Output' if clean: f.addStep( ShellCommand(name='rm-%s' % testsuite_output_dir, command=['rm', '-rf', testsuite_output_dir], description='rm test-suite output directory', haltOnFailure=True, workdir='.', env=env)) f.addStep( LitTestCommand( name='make.check', command=[ 'nice', '-n', '10', 'make', '-f', '../' + dragonegg_src_dir + '/Makefile', WithProperties('GCC=' + gcc), WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir + '/bin/llvm-config'), WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir), WithProperties('LIT_ARGS=-v -j%s' % jobs), 'check' ], description='running test-suite', descriptionDone='run test-suite', haltOnFailure=True, workdir=dragonegg_obj_dir, env=env, timeout=timeout * 60)) return f
def getClangBuildFactory( triple=None, clean=True, test=True, package_dst=None, run_cxx_tests=False, examples=False, valgrind=False, valgrindLeakCheck=False, useTwoStage=False, completely_clean=False, make='make', jobs="%(jobs)s", stage1_config='Debug+Asserts', stage2_config='Release+Asserts', env={}, # Environmental variables for all steps. extra_configure_args=[], stage2_extra_configure_args=[], use_pty_in_tests=False, trunk_revision=None, force_checkout=False, extra_clean_step=None, checkout_compiler_rt=False, run_gdb=False, run_modern_gdb=False, run_gcc=False): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) llvm_srcdir = "llvm.src" llvm_1_objdir = "llvm.obj" llvm_1_installdir = "llvm.install.1" llvm_2_objdir = "llvm.obj.2" llvm_2_installdir = "llvm.install" f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( buildbot.steps.shell.SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".", env=merged_env)) # Blow away completely, if requested. if completely_clean: f.addStep( ShellCommand(name="rm-llvm.src", command=["rm", "-rf", llvm_srcdir], haltOnFailure=True, description=["rm src dir", "llvm"], workdir=".", env=merged_env)) # Checkout sources. if trunk_revision: # The SVN build step provides no mechanism to check out a specific revision # based on a property, so just run the commands directly here. svn_co = ['svn', 'checkout'] if force_checkout: svn_co += ['--force'] svn_co += ['--revision', WithProperties(trunk_revision)] svn_co_llvm = svn_co + \ [WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%s' % trunk_revision), llvm_srcdir] svn_co_clang = svn_co + \ [WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%s' % trunk_revision), '%s/tools/clang' % llvm_srcdir] svn_co_clang_tools_extra = svn_co + \ [WithProperties('http://llvm.org/svn/llvm-project/clang-tools-extra/trunk@%s' % trunk_revision), '%s/tools/clang/tools/extra' % llvm_srcdir] f.addStep( ShellCommand(name='svn-llvm', command=svn_co_llvm, haltOnFailure=True, workdir='.')) f.addStep( ShellCommand(name='svn-clang', command=svn_co_clang, haltOnFailure=True, workdir='.')) f.addStep( ShellCommand(name='svn-clang-tools-extra', command=svn_co_clang_tools_extra, haltOnFailure=True, workdir='.')) else: f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir)) f.addStep( SVN(name='svn-clang-tools-extra', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', defaultBranch='trunk', workdir='%s/tools/clang/tools/extra' % llvm_srcdir)) if checkout_compiler_rt: f.addStep( SVN(name='svn-compiler-rt', mode='update', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', defaultBranch='trunk', workdir='%s/projects/compiler-rt' % llvm_srcdir)) # Clean up llvm (stage 1); unless in-dir. if clean and llvm_srcdir != llvm_1_objdir: f.addStep( ShellCommand(name="rm-llvm.obj.stage1", command=["rm", "-rf", llvm_1_objdir], haltOnFailure=True, description=["rm build dir", "llvm"], workdir=".", env=merged_env)) if not clean: expected_makefile = 'Makefile' f.addStep( SetProperty(name="Makefile_isready", workdir=llvm_1_objdir, command=[ "sh", "-c", "test -e %s && echo OK || echo Missing" % expected_makefile ], flunkOnFailure=False, property="exists_Makefile")) cmake_triple_arg = [] if triple: cmake_triple_arg = ['-DLLVM_HOST_TRIPLE=%s' % triple] f.addStep( ShellCommand(name='cmake', command=[ 'cmake', '-DLLVM_BUILD_TESTS=ON', '-DCMAKE_BUILD_TYPE=%s' % stage1_config ] + cmake_triple_arg + extra_configure_args + ["../" + llvm_srcdir], description='cmake stage1', workdir=llvm_1_objdir, env=merged_env, doStepIf=lambda step: step.build.getProperty( "exists_Makefile") != "OK")) # Make clean if using in-dir builds. if clean and llvm_srcdir == llvm_1_objdir: f.addStep( WarningCountingShellCommand(name="clean-llvm", command=[make, "clean"], haltOnFailure=True, description="cleaning llvm", descriptionDone="clean llvm", workdir=llvm_1_objdir, doStepIf=clean, env=merged_env)) if extra_clean_step: f.addStep(extra_clean_step) f.addStep( WarningCountingShellCommand( name="compile", command=['nice', '-n', '10', make, WithProperties("-j%s" % jobs)], haltOnFailure=True, flunkOnFailure=not run_gdb, description=["compiling", stage1_config], descriptionDone=["compile", stage1_config], workdir=llvm_1_objdir, env=merged_env)) if examples: f.addStep( WarningCountingShellCommand( name="compile.examples", command=[ 'nice', '-n', '10', make, WithProperties("-j%s" % jobs), "BUILD_EXAMPLES=1" ], haltOnFailure=True, description=["compiling", stage1_config, "examples"], descriptionDone=["compile", stage1_config, "examples"], workdir=llvm_1_objdir, env=merged_env)) clangTestArgs = '-v -j %s' % jobs if valgrind: clangTestArgs += ' --vg' if valgrindLeakCheck: clangTestArgs += ' --vg-leak' clangTestArgs += ' --vg-arg --suppressions=%(builddir)s/llvm/tools/clang/utils/valgrind/x86_64-pc-linux-gnu_gcc-4.3.3.supp --vg-arg --suppressions=%(builddir)s/llvm/utils/valgrind/x86_64-pc-linux-gnu.supp' extraTestDirs = '' if run_cxx_tests: extraTestDirs += '%(builddir)s/llvm/tools/clang/utils/C++Tests' if test: f.addStep( lit_test_command.LitTestCommand( name='check-all', command=[ make, "check-all", "VERBOSE=1", WithProperties("LIT_ARGS=%s" % clangTestArgs), WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs) ], flunkOnFailure=not run_gdb, description=["checking"], descriptionDone=["checked"], workdir=llvm_1_objdir, usePTY=use_pty_in_tests, env=merged_env)) # TODO: Install llvm and clang for stage1. if run_gdb or run_gcc or run_modern_gdb: ignores = getClangTestsIgnoresFromPath( os.path.expanduser('~/public/clang-tests'), 'clang-x86_64-darwin10') install_prefix = "%%(builddir)s/%s" % llvm_1_installdir if run_gdb: addClangGDBTests(f, ignores, install_prefix) if run_modern_gdb: addModernClangGDBTests(f, jobs, install_prefix) if run_gcc: addClangGCCTests(f, ignores, install_prefix) if not useTwoStage: if package_dst: name = WithProperties( "%(builddir)s/" + llvm_1_objdir + "/clang-r%(got_revision)s-b%(buildnumber)s.tgz") f.addStep( ShellCommand(name='pkg.tar', description="tar root", command=["tar", "zcvf", name, "./"], workdir=llvm_1_installdir, warnOnFailure=True, flunkOnFailure=False, haltOnFailure=False, env=merged_env)) f.addStep( ShellCommand(name='pkg.upload', description="upload root", command=[ "scp", name, WithProperties(package_dst + "/%(buildername)s") ], workdir=".", warnOnFailure=True, flunkOnFailure=False, haltOnFailure=False, env=merged_env)) return f # Clean up llvm (stage 2). # # We always cleanly build the stage 2. If the compiler has been # changed on the stage 1, we cannot trust any of the intermediate file # from the old compiler. And if the stage 1 compiler is the same, we should # not build in the first place. f.addStep( ShellCommand(name="rm-llvm.obj.stage2", command=["rm", "-rf", llvm_2_objdir], haltOnFailure=True, description=["rm build dir", "llvm", "(stage 2)"], workdir=".", env=merged_env)) # Configure llvm (stage 2). f.addStep( ShellCommand( name='cmake', command=['cmake'] + stage2_extra_configure_args + [ '-DLLVM_BUILD_TESTS=ON', WithProperties('-DCMAKE_C_COMPILER=%%(builddir)s/%s/bin/clang' % llvm_1_objdir), # FIXME use installdir WithProperties( '-DCMAKE_CXX_COMPILER=%%(builddir)s/%s/bin/clang++' % llvm_1_objdir), '-DCMAKE_BUILD_TYPE=%s' % stage2_config, "../" + llvm_srcdir ], description='cmake stage2', workdir=llvm_2_objdir, env=merged_env)) # Build llvm (stage 2). f.addStep( WarningCountingShellCommand( name="compile.llvm.stage2", command=['nice', '-n', '10', make, WithProperties("-j%s" % jobs)], haltOnFailure=True, description=["compiling", "(stage 2)", stage2_config], descriptionDone=["compile", "(stage 2)", stage2_config], workdir=llvm_2_objdir, env=merged_env)) if test: f.addStep( lit_test_command.LitTestCommand( name='check-all', command=[ make, "check-all", "VERBOSE=1", WithProperties("LIT_ARGS=%s" % clangTestArgs), WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs) ], description=["checking"], descriptionDone=["checked"], workdir=llvm_2_objdir, usePTY=use_pty_in_tests, env=merged_env)) # TODO: Install llvm and clang for stage2. if package_dst: name = WithProperties("%(builddir)s/" + llvm_2_objdir + "/clang-r%(got_revision)s-b%(buildnumber)s.tgz") f.addStep( ShellCommand(name='pkg.tar', description="tar root", command=["tar", "zcvf", name, "./"], workdir=llvm_2_installdir, warnOnFailure=True, flunkOnFailure=False, haltOnFailure=False, env=merged_env)) f.addStep( ShellCommand(name='pkg.upload', description="upload root", command=[ "scp", name, WithProperties(package_dst + "/%(buildername)s") ], workdir=".", warnOnFailure=True, flunkOnFailure=False, haltOnFailure=False, env=merged_env)) return f
def getClangMSVCBuildFactory(update=True, clean=True, vcDrive='c', jobs=1, cmake=r"cmake"): f = buildbot.process.factory.BuildFactory() if update: f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir='llvm')) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='llvm/tools/clang')) f.addStep( SVN(name='svn-clang-tools-extra', mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', defaultBranch='trunk', workdir='llvm/tools/clang/tools/extra')) # Full & fast clean. if clean: f.addStep( ShellCommand(name='clean-1', command=['del', '/s/q', 'build'], warnOnFailure=True, description='cleaning', descriptionDone='clean', workdir='llvm')) f.addStep( ShellCommand(name='clean-2', command=['rmdir', '/s/q', 'build'], warnOnFailure=True, description='cleaning', descriptionDone='clean', workdir='llvm')) # Create the project files. # Use batch files instead of ShellCommand directly, Windows quoting is # borked. FIXME: See buildbot ticket #595 and buildbot ticket #377. f.addStep( batch_file_download.BatchFileDownload( name='cmakegen', command=[ cmake, "-DLLVM_TARGETS_TO_BUILD:=X86", "-DLLVM_INCLUDE_EXAMPLES:=OFF", "-DLLVM_INCLUDE_TESTS:=OFF", "-DLLVM_TARGETS_TO_BUILD:=X86", "-G", "Visual Studio 9 2008", ".." ], workdir="llvm\\build")) f.addStep( ShellCommand(name='cmake', command=['cmakegen.bat'], haltOnFailure=True, description='cmake gen', workdir='llvm\\build')) # Build it. f.addStep( batch_file_download.BatchFileDownload( name='vcbuild', command=[ vcDrive + r""":\Program Files\Microsoft Visual Studio 9.0\VC\VCPackages\vcbuild.exe""", "/M%d" % jobs, "LLVM.sln", "Debug|Win32" ], workdir="llvm\\build")) f.addStep( WarningCountingShellCommand(name='vcbuild', command=['vcbuild.bat'], haltOnFailure=True, description='vcbuild', workdir='llvm\\build', warningPattern=" warning C.*:")) # Build clang-test project. f.addStep( batch_file_download.BatchFileDownload( name='vcbuild_test', command=[ vcDrive + r""":\Program Files\Microsoft Visual Studio 9.0\VC\VCPackages\vcbuild.exe""", "clang-test.vcproj", "Debug|Win32" ], workdir="llvm\\build\\tools\\clang\\test")) f.addStep( lit_test_command.LitTestCommand( name='test-clang', command=["vcbuild_test.bat"], workdir="llvm\\build\\tools\\clang\\test")) return f
def getPollyBuildFactory(clean=False, install=False, make='make', jobs=None, checkAll=False, env=None, extraCmakeArgs=[]): llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" llvm_instdir = "llvm.inst" polly_srcdir = '%s/tools/polly' % llvm_srcdir clang_srcdir = '%s/tools/clang' % llvm_srcdir jobs_cmd = [] if jobs is not None: jobs_cmd = ["-j" + str(jobs)] build_cmd = [make] + jobs_cmd install_cmd = [make, 'install'] + jobs_cmd check_all_cmd = [make, 'check-all'] + jobs_cmd check_polly_cmd = [make, 'check-polly'] + jobs_cmd cmake_install = [] if install: cmake_install = ["-DCMAKE_INSTALL_PREFIX=../%s" % llvm_instdir] # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env: merged_env.update( env ) # Overwrite pre-set items with the given ones, so user can set anything. f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( buildbot.steps.shell.SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".")) # Get LLVM, Clang and Polly f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir=clang_srcdir)) f.addStep( SVN(name='svn-polly', mode='update', baseURL='http://llvm.org/svn/llvm-project/polly/', defaultBranch='trunk', workdir=polly_srcdir)) # Clean build dir if clean: f.addStep( ShellCommand(name='clean-build-dir', command=['rm', '-rf', llvm_objdir], warnOnFailure=True, description=["clean build dir"], workdir='.', env=merged_env)) # Create configuration files with cmake f.addStep( ShellCommand(name="create-build-dir", command=["mkdir", "-p", llvm_objdir], haltOnFailure=False, description=["create build dir"], workdir=".", env=merged_env)) cmakeCommand = [ "cmake", "../%s" % llvm_srcdir, "-DCMAKE_COLOR_MAKEFILE=OFF", "-DPOLLY_TEST_DISABLE_BAR=ON", "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON", "-DCMAKE_BUILD_TYPE=Release" ] + cmake_install + extraCmakeArgs f.addStep( ShellCommand(name="cmake-configure", command=cmakeCommand, haltOnFailure=False, description=["cmake configure"], workdir=llvm_objdir, env=merged_env)) # Build f.addStep( ShellCommand(name="build", command=build_cmd, haltOnFailure=True, description=["build"], workdir=llvm_objdir, env=merged_env)) # Clean install dir if install and clean: f.addStep( ShellCommand(name='clean-install-dir', command=['rm', '-rf', llvm_instdir], haltOnFailure=False, description=["clean install dir"], workdir='.', env=merged_env)) # Install if install: f.addStep( ShellCommand(name="install", command=install_cmd, haltOnFailure=False, description=["install"], workdir=llvm_objdir, env=merged_env)) # Test if checkAll: f.addStep( ShellCommand(name="check_all", command=check_all_cmd, haltOnFailure=False, description=["check all"], workdir=llvm_objdir, env=merged_env)) else: f.addStep( ShellCommand(name="check_polly", command=check_polly_cmd, haltOnFailure=False, description=["check polly"], workdir=llvm_objdir, env=merged_env)) return f
def getLibompCMakeBuildFactory(clean=True, env=None, test=True, c_compiler="gcc", cxx_compiler="g++"): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) openmp_srcdir = "openmp.src" openmp_builddir = "openmp.build" llvm_srcdir = "llvm.src" llvm_builddir = "llvm.build" f = buildbot.process.factory.BuildFactory() # Get libomp f.addStep( SVN(name='svn-libomp', mode='update', baseURL='http://llvm.org/svn/llvm-project/openmp/', defaultBranch='trunk', workdir=openmp_srcdir)) # Get llvm to build llvm-lit f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) # Clean directory, if requested. if clean: f.addStep( ShellCommand(name="clean", command=["rm", "-rf", openmp_builddir, llvm_builddir], warnOnFailure=True, description=["clean"], workdir='.', env=merged_env)) # CMake llvm f.addStep( ShellCommand(name='cmake llvm', command=[ "cmake", "../" + llvm_srcdir, "-DCMAKE_C_COMPILER=" + c_compiler, "-DCMAKE_CXX_COMPILER=" + cxx_compiler ], haltOnFailure=True, description='cmake llvm', workdir=llvm_builddir, env=merged_env)) # Make llvm utils f.addStep( WarningCountingShellCommand(name='make llvm utils build', command=['make', 'LLVMX86Utils', '-j8'], haltOnFailure=True, description='make llvm utils build', workdir=llvm_builddir, env=merged_env)) # Add llvm-lit to PATH merged_env.update({ 'PATH': WithProperties("${PATH}:" + "%(workdir)s/" + llvm_builddir + "/bin") }) # CMake libomp f.addStep( ShellCommand(name='cmake libomp', command=[ "cmake", "../" + openmp_srcdir, "-DCMAKE_C_COMPILER=" + c_compiler, "-DCMAKE_CXX_COMPILER=" + cxx_compiler ], haltOnFailure=True, description='cmake libomp', workdir=openmp_builddir, env=merged_env)) # Make libomp f.addStep( WarningCountingShellCommand(name='make libomp build', command=['make'], haltOnFailure=True, description='make libomp build', workdir=openmp_builddir, env=merged_env)) # Test, if requested if test: f.addStep( lit_test_command.LitTestCommand(name='make check-libomp', command=['make', 'check-libomp'], haltOnFailure=True, description=["make check-libomp"], descriptionDone=["build checked"], workdir=openmp_builddir, env=merged_env)) return f