def _step_AdditionalProperties(self): self.addStep( SetProperty(name="modifying configures", doStepIf=lambda step: (step.getProperty("slavename") in self.wineslaves), property="gcc_config_args", command=[ "echo", Property("gcc_config_args", default=""), "--build=i686-w64-mingw32" ])) self.addStep( SetProperty(name="modifying configures", doStepIf=lambda step: (step.getProperty("slavename") in self.wineslaves), property="binutils_config_args", command=[ "echo", Property("binutils_config_args", default=""), "--build=i686-w64-mingw32" ])) self.addStep( SetProperty(name="modifying configures", doStepIf=lambda step: (step.getProperty("slavename") in self.wineslaves), property="mingw_config_args", command=[ "echo", Property("mingw_config_args", default=""), "--build=i686-w64-mingw32" ]))
def getAnnotatedBuildFactory(script, clean=False, depends_on_projects=None, env=None, timeout=1200): """ Returns a new build factory that uses AnnotatedCommand, which allows the build to be run by version-controlled scripts that do not require a buildmaster restart to update. """ f = LLVMBuildFactory(depends_on_projects=depends_on_projects, llvm_srcdir='llvm.src') if clean: f.addStep(SetProperty(property='clean', command='echo 1')) # We normally use the clean property to indicate that we want a # clean build, but AnnotatedCommand uses the clobber property # instead. Therefore, set clobber if clean is set to a truthy # value. This will cause AnnotatedCommand to set # BUILDBOT_CLOBBER=1 in the environment, which is how we # communicate to the script that we need a clean build. f.addStep( SetProperty( property='clobber', command='echo 1', doStepIf=lambda step: step.build.getProperty('clean', False))) merged_env = { 'TERM': 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set # anything. merged_env.update(env) scripts_dir = "annotated" f.addStep( SVN(name='update-annotate-scripts', mode='update', svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/' 'zorg/buildbot/builders/annotated', workdir=scripts_dir, alwaysUseLatest=True)) # Explicitly use '/' as separator, because it works on *nix and Windows. script_path = "../%s/%s" % (scripts_dir, script) f.addStep( AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, command=WithProperties( "python %(script)s --jobs=%(jobs:-)s", script=lambda _: script_path), env=merged_env)) return f
def __init__(self, python, package, **kw): SetProperty.__init__( self, name="check-%s-version" % package, description="checking %s version" % package, command=[python, '-c', self.src % dict(package=package)], extract_fn=self._extractVersion, **kw) self.package = package
def __init__(self, **kwargs): factory.BuildFactory.__init__(self, **kwargs) self.addStep( SetProperty( property="optionTargetOs", doStepIf=lambda step: step.build.getProperty("target-os"), command=["echo", "--os"])) self.addStep( SetProperty(property="optionPath", doStepIf=lambda step: step.build.getProperty("path"), command=["echo", "--path"])) self.addStep( SetProperty( property="optionDatestamp", doStepIf=lambda step: step.build.getProperty("datestamp"), command=["echo", "--datestamp"])) # this assumes the following properties have been set: # masterdir - the path to the master configuration (has tarballs + scripts + passwords) # filename - the name of the file to upload (relative to masterdir) # destname - how to name the file on the server self.addStep( ShellCommand(name="sfupload", description=["uploading"], descriptionDone=["upload"], doStepIf=lambda step: (step.build.getProperty("is_nightly")), workdir=Property("masterdir"), command=[ "scripts/upload.py", Property("filename"), Property("destname"), Property("optionTargetOs", default=""), Property("target-os", default=""), Property("optionPath", default=""), Property("path", default=""), Property("optionDatestamp", default=""), Property("datestamp", default="") ], haltOnFailure=True)) self.addStep( ShellCommand(name="uploadComplete", description=["removing", "temporary", "file"], descriptionDone=["temporary", "file", "removed"], doStepIf=lambda step: (step.build.getProperty("is_nightly")), workdir=Property("masterdir"), command=["rm", "-fv", Property("filename")]))
def __init__(self, condprop=None, condinvert=False, condvalue=None, conddefault=None, **kwargs): SetProperty.__init__(self, **kwargs) self.addFactoryArguments(condprop=condprop, condinvert=condinvert, condvalue=condvalue, conddefault=conddefault) self.condProp = condprop self.condInvert = condinvert self.condValue = condvalue self.condDefault = conddefault
def setProperty(f, new_property, new_value): f.addStep(SetProperty(name = 'set.' + new_property, command=['echo', new_value], property=new_property, description=['set property', new_property], workdir='.')) return f
def getUserDir(f): f.addStep(SetProperty(command=['sh', '-c', 'cd ~;pwd'], haltOnFailure=True, property='user_dir', description=['set property', 'user_dir'], workdir='.')) return f
def getBuildDir(f): f.addStep(SetProperty(name='get.build.dir', command=['pwd'], property='builddir', description='set build dir', workdir='.')) return f
def update_repo(self, repo, branch='default', clobber=False): workdir = repo.split("/")[-1] repourl = 'http://%s/%s' % (self.hgHost, repo) if clobber: self.addStep(ShellCommand( name='%s_clobber' % workdir, command=['rm', '-rf', workdir], workdir='.', )) self.addStep(ShellCommand( name='%s_update' % workdir, command=['bash', '-c', 'if test -d %(workdir)s; then hg -R %(workdir)s pull; \ else hg clone --noupdate %(repourl)s %(workdir)s; fi && \ hg -R %(workdir)s up -C --rev %(branch)s' % locals()], timeout=3*60, descriptionDone="%s source" % workdir, workdir='.', )) self.addStep(SetProperty( name='set_%s_revision' % workdir, command=['hg', 'identify', '-i'], property='%s_revision' % workdir, workdir=workdir, ))
def _step_AdditionalProperties(self): self.addStep( SetProperty( property="host-extra", command=[ "bash", "-c", """cygcheck -c -d cygwin | perl -ne 'm/^cygwin\s+(\S+)/ and print "-$1"'""" ]))
def getTestConfig(f): def getRemoteCfg(rc, stdout, stderr): return json.loads(stdout) f.addStep(SetProperty(name="get test config", command="cat test_cfg.json", extract_fn=getRemoteCfg, description="get remote target", workdir="scripts")) return f
def getSanitizerBuildFactory(clean=False, depends_on_projects=None, env=None, timeout=1200, is_legacy_mode=False): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { "TERM": "dumb", # Make sure Clang doesn't use color escape sequences. } # Clobber bot if we need a clean build. if clean: merged_env["BUILDBOT_CLOBBER"] = "1" # Overwrite pre-set items with the given ones, so user can set anything. if env is not None: merged_env.update(env) if depends_on_projects is None: depends_on_projects = [ "llvm", "clang", "compiler-rt", "libcxx", "libcxxabi", "libunwind", "lld" ] # Explicitly use '/' as separator, because it works on *nix and Windows. sanitizer_script_dir = "sanitizer_buildbot/sanitizers" sanitizer_script = "../%s/zorg/buildbot/builders/sanitizers/%s" % ( sanitizer_script_dir, "buildbot_selector.py") f = LLVMBuildFactory(is_legacy_mode=is_legacy_mode, clean=clean, depends_on_projects=depends_on_projects, llvm_srcdir=sanitizer_script_dir) # Determine the build directory. f.addStep( SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".", env=merged_env)) # Get sanitizer buildbot scripts. f.addGetSourcecodeForProject(name='update-annotate-scripts', project='zorg', src_dir=sanitizer_script_dir, alwaysUseLatest=True) # Run annotated command for sanitizer. f.addStep( AnnotatedCommand(name="annotate", description="annotate", timeout=timeout, haltOnFailure=True, command="python " + sanitizer_script, env=merged_env)) return f
def addGCSUploadSteps(f, package_name, install_prefix, gcs_directory, env, gcs_url_property=None, use_pixz_compression=False, xz_compression_factor=6): """ Add steps to upload to the Google Cloud Storage bucket. f - The BuildFactory to modify. package_name - The name of this package for the descriptions (e.g. 'stage 1') install_prefix - The directory the build has been installed to. gcs_directory - The subdirectory of the bucket root to upload to. This should match the builder name. env - The environment to use. Set BOTO_CONFIG to use a configuration file in a non-standard location, and BUCKET to use a different GCS bucket. gcs_url_property - Property to assign the GCS url to. """ gcs_url_fmt = ('gs://%(gcs_bucket)s/%(gcs_directory)s/' 'clang-r%(got_revision)s-t%(now)s-b%(buildnumber)s.tar.xz') time_fmt = '%Y-%m-%d_%H-%M-%S' output_file_name = '../install.tar.xz' gcs_url = \ WithProperties( gcs_url_fmt, gcs_bucket=lambda _: env.get('BUCKET', 'llvm-build-artifacts'), gcs_directory=lambda _: gcs_directory, now=lambda _: datetime.utcnow().strftime(time_fmt)) if gcs_url_property: f.addStep(SetProperty( name="record GCS url for " + package_name, command=['echo', gcs_url], property=gcs_url_property)) if use_pixz_compression: # tweak the xz compression level to generate packages faster tar_command = ['tar', '-Ipixz', '-cvf', output_file_name, '.'] else: xz_command = 'xz -{0}'.format(xz_compression_factor) tar_command = ['tar', '-I', xz_command, '-cvf', output_file_name, '.'] f.addStep(ShellCommand(name='package ' + package_name, command=tar_command, description='packaging ' + package_name + '...', workdir=install_prefix, env=env)) f.addStep(ShellCommand( name='upload ' + package_name + ' to storage bucket', command=['gsutil', 'cp', '../install.tar.xz', gcs_url], description=('uploading ' + package_name + 'to storage bucket ...'), workdir=install_prefix, env=env))
def start(self): if self.condProp is not None: value = self.condDefault matched = False if self.build.getProperties().has_key(self.condProp): value = self.build.getProperty(self.condProp) elif self.condDefault is None: # no value and no default, assume we want to execute matched = True if not matched: if self.condValue is None: matched = bool(value) else: matched = (value == self.condValue) if self.condInvert: matched = not matched if not matched: return SKIPPED SetProperty.start(self)
def __init__(self, repo, *testnames): BuildProcedure.__init__(self, 'elisp') self.addSteps( Git(repourl='git://github.com/%s.git' % repo), SetPropertiesFromEnv(variables=['EMACS']), SetProperty( command=[ Emacs(), '--batch', '--eval', '(princ (make-temp-file "home" t ".bbot"))' ], extract_fn=lambda rc, stdout, stderr: dict(FakeHome=stdout))) for t in testnames or ['test/test']: self.addStep(EmacsTest(load=t + '.el'))
def testGoodStep(self): f = BuildFactory() f.addStep(SetProperty(command=["echo", "value"], property="propname")) ss = SourceStamp() req = FakeBuildRequest("Testing", ss, None) b = f.newBuild([req]) b.build_status = FakeBuildStatus() b.slavebuilder = FakeSlaveBuilder() # This shouldn't raise an exception b.setupBuild(None)
def __init__(self, hgHost, **kwargs): self.parent_class = BuildFactory self.parent_class.__init__(self, **kwargs) self.hgHost = hgHost self.addStep(SetProperty( name='set_topdir', command=['pwd'], property='topdir', workdir='.', )) self.addStep(ShellCommand( name='rm_pyc', command=['find', '.', '-name', '*.pyc', '-exec', 'rm', '-fv', '{}', ';'], workdir=".", ))
def testCommand(self): f = BuildFactory() f.addStep(SetProperty(command=["echo", "value"], property="propname")) f.addStep( ShellCommand( command=["echo", WithProperties("%(propname)s")])) ss = SourceStamp() req = FakeBuildRequest("Testing", {ss.repository: ss}, None) b = f.newBuild([req]) b.build_status = FakeBuildStatus() b.slavebuilder = FakeSlaveBuilder() # This shouldn't raise an exception b.setupBuild(None)
def xmippBundleFactory(groupId): xmippTestSteps = util.BuildFactory() xmippTestSteps.workdir = settings.SDEVEL_XMIPP_HOME env = { "SCIPION_HOME": util.Property("SCIPION_HOME"), "SCIPION_LOCAL_CONFIG": util.Property("SCIPION_LOCAL_CONFIG"), "LD_LIBRARY_PATH": LD_LIBRARY_PATH, "EM_ROOT": settings.EM_ROOT } xmippTestSteps.addStep( SetProperty(command=["bash", "-c", "source build/xmipp.bashrc; env"], extract_fn=glob2list, env=env)) xmippTestSteps.addStep( GenerateStagesCommand( command=["./xmipp", "test", "--show"], name="Generate test stages for Xmipp programs", description="Generating test stages for Xmipp programs", descriptionDone="Generate test stages for Xmipp programs", haltOnFailure=False, pattern='./xmipp test (.*)', rootName=settings.XMIPP_CMD, timeout=settings.timeOutExecute, blacklist=settings.SCIPION_TESTS_BLACKLIST, env=util.Property('env'))) xmippTestSteps.addStep( GenerateStagesCommand( command=["./xmipp", "test", "--show"], name="Generate test stages for Xmipp functions", description="Generating test stages for Xmipp functions", descriptionDone="Generate test stages for Xmipp functions", haltOnFailure=False, timeout=settings.timeOutExecute, pattern='xmipp_test_(.*)', rootName=settings.XMIPP_CMD, blacklist=settings.SCIPION_TESTS_BLACKLIST, env=util.Property('env'))) return xmippTestSteps
def getLLDBCMakeBuildFactory( clean=False, jobs="%(jobs)s", # Source directory containing a built python python_source_dir=None, # Default values for VS devenv and build configuration vs=None, config='Release', target_arch='x86', extra_cmake_args=None, test=False, testTimeout=2400, install=False): ############# PREPARING # Global configurations build_dir = 'build' f = LLVMBuildFactory(is_legacy_mode=False, depends_on_projects=["llvm", "clang", "lldb", "lld"], obj_dir=build_dir) # Determine Slave Environment and Set MSVC environment. if vs: f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f.addGetSourcecodeSteps() build_cmd = ['ninja'] install_cmd = ['ninja', 'install'] test_cmd = ['ninja', 'check-lldb'] if jobs: build_cmd.append(WithProperties("-j%s" % jobs)) install_cmd.append(WithProperties("-j%s" % jobs)) test_cmd.append(WithProperties("-j%s" % jobs)) ############# CLEANING cleanBuildRequested = lambda step: clean or step.build.getProperty( "clean", default=step.build.getProperty("clean_obj")) f.addStep( RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, f.obj_dir) cmake_options = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=" + config, "-DCMAKE_INSTALL_PREFIX=../install", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), ] if python_source_dir: cmake_options.append("-DPYTHON_HOME=" + python_source_dir) if extra_cmake_args: cmake_options += extra_cmake_args f.addStep( CmakeCommand(name="cmake-configure", description=["cmake configure"], haltOnFailure=True, options=cmake_options, path=rel_src_dir, env=Property('slave_env'), workdir=build_dir)) f.addStep( WarningCountingShellCommand(name='build', command=build_cmd, haltOnFailure=True, description='ninja build', workdir=build_dir, env=Property('slave_env'))) ignoreInstallFail = bool(install != 'ignoreFail') f.addStep( ShellCommand(name='install', command=install_cmd, flunkOnFailure=ignoreInstallFail, description='ninja install', workdir=build_dir, doStepIf=bool(install), env=Property('slave_env'))) ignoreTestFail = bool(test != 'ignoreFail') f.addStep( ShellCommand(name='test', command=test_cmd, flunkOnFailure=ignoreTestFail, timeout=testTimeout, description='ninja test', workdir=build_dir, doStepIf=bool(test), env=Property('slave_env'))) return f
def getLLDBUbuntuCMakeBuildFactory(build_compiler, build_type, local_test_archs=None, local_test_compilers=None, remote_configs=None, jobs='%(jobs)s', env=None): """Generate factory steps for ubuntu cmake builder Arguments: build_compiler -- string of compile name, example 'clang', the compiler will be used to build binaries for host platform build_type -- 'Debug' or 'Release', used to define build type for host platform as well as remote platform if any local_test_archs -- list of architectures, example ['i386','x86_64'], defines architectures to run local tests against, if None, local tests won't be executed local_test_compiler -- list of compilers, example ['clang','gcc4.8.2'], definds compilers to run local tests with, if None, local tests won't be executed remote_configs -- list of RemoteConfig objects, example [RemoteConfig(...)], if None, remote tests won't be executed jobs -- number of threads for compilation step, example 40 default value is jobs number defined during slave creation env -- environment variables passed to shell commands """ if env is None: env = {} llvm_srcdir = "llvm" llvm_builddir = "build" bindir = '%(builddir)s/' + llvm_builddir + '/bin' f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".")) # Determine the binary directory of *-tblgen. f.addStep( SetProperty(name="get tblgen dir", command=["echo", WithProperties(bindir)], property="tblgen_bindir", description="set tblgen dir", workdir=".")) # Get source code f = getLLDBSource(f, llvm_srcdir) # Clean Build Folder f.addStep( ShellCommand(name="clean", command="rm -rf *", description="clear build folder", env=env, workdir='%s' % llvm_builddir)) f = getLLDBCmakeAndCompileSteps(f, build_compiler, build_type, [], bindir, 'linux', 'x86_64', env) # TODO: it will be good to check that architectures listed in test_archs are compatible with host architecture # For now, the caller of this function should make sure that each target architecture is supported by builder machine # Add local test steps f = getLLDBTestSteps(f, bindir, local_test_archs, local_test_compilers) # Remote test steps if remote_configs is not None: for config in remote_configs: f = getLLDBRemoteTestSteps(f, bindir, build_type, config, env) # archive test traces f = archiveLLDBTestTraces(f, "build/lldb-test-traces-*") return f
def getLLDBRemoteTestSteps(f, bindir, build_type, remote_config, env): if None in (remote_config.test_archs, remote_config.test_compilers): return f # only supports linux and android as remote target at this time if remote_config.platform not in ('linux', 'android'): return f llvm_srcdir = "llvm" llvm_builddir = "build" stepDesc = remote_config.platform + "-" + remote_config.host_arch # get hostname slave_hostname = None f.addStep( SetProperty(name="get hostname", command=["hostname"], property="slave_hostname", description="set slave hostname", workdir=".")) # get configuration of remote target # config file should be placed under builddir on builder machine # file name: remote_cfg.json # content: json format with keys [remote_platform]-[remote_arch] # the value for each key defines "remote_host", "remote_port", "remote_dir", "toolchain", "deviceId" # example: {"android-i386": {"remote_host":"localhost", # "remote_port":"5430", # "remote_dir":"/data/local/tmp/lldb", # "toolchain_build":"/home/lldb_build/Toolchains/i386-android-toolchain-21", # "toolchain_test":"/home/lldb_build/Toolchains/i386-android-toolchain-16", # "deviceid":"XXXXXXX"}, def getRemoteCfg(rc, stdout, stderr): return json.loads(stdout)[stepDesc] f.addStep( SetProperty(name="get remote target " + stepDesc, command="cat remote_cfg.json", extract_fn=getRemoteCfg, description="get remote target", workdir=".")) # rsync if remote_config.platform is 'linux': shellcmd = ['ssh', WithProperties('%(remote_host)s')] hostname = '%(slave_hostname)s' launchcmd = shellcmd + ['screen', '-d', '-m'] terminatecmd = shellcmd + ['pkill', 'lldb-server'] cleandircmd = WithProperties( 'ssh %(remote_host)s rm -r %(remote_dir)s/*') f.addStep( ShellCommand(name="rsync lldb-server", command=[ 'rsync', '-havL', 'bin/lldb-server', WithProperties('%(remote_host)s:%(remote_dir)s') ], description="rsync lldb-server " + stepDesc, haltOnFailure=True, env=env, workdir='%s' % llvm_builddir)) f.addStep( ShellCommand(name="rsync python2.7", command=[ 'rsync', '-havL', 'lib/python2.7', WithProperties('%(remote_host)s:%(remote_dir)s') ], description="rsync python2.7 " + stepDesc, haltOnFailure=True, env=env, workdir='%s' % llvm_builddir)) elif remote_config.platform is 'android': shellcmd = ['adb', '-s', WithProperties('%(deviceid)s'), 'shell'] hostname = '127.0.0.1' launchcmd = ['screen', '-d', '-m'] + shellcmd + [ WithProperties("TMPDIR=%(remote_dir)s/tmp") ] terminatecmd = 'ps | grep lldb-server | awk \'{print $2}\' | xargs' terminatecmd = WithProperties('adb -s %(deviceid)s shell ' + terminatecmd + ' adb -s %(deviceid)s shell kill') cleandircmd = WithProperties( 'adb -s %(deviceid)s shell rm -rf %(remote_dir)s/*') # compile lldb-server for target platform f = getLLDBCmakeAndCompileSteps(f, 'gcc', build_type, ['lldb-server'], bindir, remote_config.platform, remote_config.host_arch, env) f.addStep( ShellCommand(name="adb push lldb-server " + stepDesc, command=[ 'adb', '-s', WithProperties('%(deviceid)s'), 'push', remote_config.platform + '-' + remote_config.host_arch + '/bin/lldb-server', WithProperties('%(remote_dir)s/') ], description="lldb-server", env=env, haltOnFailure=True, workdir='%s' % llvm_builddir)) f.addStep( ShellCommand(name="Build fingerprint " + stepDesc, command=[ 'adb', '-s', WithProperties('%(deviceid)s'), 'shell', 'getprop', 'ro.build.fingerprint' ], description="get build fingerprint", env=env, haltOnFailure=False, workdir='%s' % llvm_builddir)) # launch lldb-server f.addStep( ShellCommand( name="launch lldb-server " + stepDesc, command=launchcmd + [ WithProperties('%(remote_dir)s/lldb-server'), 'platform', '--listen', WithProperties(hostname + ':%(remote_port)s'), '--server' ], description="launch lldb-server on remote host", env=env, haltOnFailure=True, workdir='%s' % llvm_builddir)) # test steps f = getLLDBTestSteps(f, bindir, remote_config.test_archs, remote_config.test_compilers, remote_config.platform, '%(remote_host)s', '%(remote_port)s', '%(remote_dir)s', env) # terminate lldb-server on remote host f.addStep( ShellCommand(name="terminate lldb-server " + stepDesc, command=terminatecmd, description="terminate lldb-server", env=env, workdir='%s' % llvm_builddir)) # clean remote test directory f.addStep( ShellCommand(name="clean remote dir " + stepDesc, command=cleandircmd, description="clean remote dir", env=env)) return f
def getLLDBBuildFactory(triple, useTwoStage=False, make='make', jobs='%(jobs)s', extra_configure_args=[], env={}, *args, **kwargs): llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" f = buildbot.process.factory.BuildFactory() # Determine the build directory. f.addStep( SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=".")) # Find out what version of llvm and clang are needed to build this version # of lldb. Right now we will assume they use the same version. # XXX - could this be done directly on the master instead of the slave? f.addStep( SetProperty( command= 'svn cat http://llvm.org/svn/llvm-project/lldb/trunk/scripts/build-llvm.pl | grep ^our.*llvm_revision | cut -d \\" -f 2', property='llvmrev')) # The SVN build step provides no mechanism to check out a specific revision # based on a property, so just run the commands directly here. svn_co = ['svn', 'checkout', '--force'] svn_co += ['--revision', WithProperties('%(llvmrev)s')] # build llvm svn checkout command svn_co_llvm = svn_co + \ [WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%(llvmrev)s'), llvm_srcdir] # build clang svn checkout command svn_co_clang = svn_co + \ [WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%(llvmrev)s'), '%s/tools/clang' % llvm_srcdir] f.addStep( ShellCommand(name='svn-llvm', command=svn_co_llvm, haltOnFailure=True, workdir='.')) f.addStep( ShellCommand(name='svn-clang', command=svn_co_clang, haltOnFailure=True, workdir='.')) f.addStep( SVN(name='svn-lldb', mode='update', baseURL='http://llvm.org/svn/llvm-project/lldb/', defaultBranch='trunk', always_purge=True, workdir='%s/tools/lldb' % llvm_srcdir)) # Run configure config_args = [ WithProperties("%%(builddir)s/%s/configure" % llvm_srcdir), "--disable-bindings", "--without-llvmgcc", "--without-llvmgxx", ] if triple: config_args += ['--build=%s' % triple] config_args += extra_configure_args f.addStep( Configure(name='configure', command=config_args, env=env, workdir=llvm_objdir)) f.addStep( WarningCountingShellCommand( name="compile", command=['nice', '-n', '10', make, WithProperties("-j%s" % jobs)], env=env, haltOnFailure=True, workdir=llvm_objdir)) # Test. f.addStep( LitTestCommand(name="test lldb", command=['nice', '-n', '10', make], description="test lldb", env=env, workdir='%s/tools/lldb/test' % llvm_objdir)) return f
def getCmakeWithNinjaWithMSVCBuildFactory( depends_on_projects=None, llvm_srcdir=None, obj_dir=None, checks=None, install_dir=None, clean=False, extra_configure_args=None, # VS tools environment variable if using MSVC. For example, # %VS140COMNTOOLS% selects the 2015 toolchain. vs=None, target_arch=None, env=None, **kwargs): assert not env, "Can't have custom builder env vars with MSVC build" # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() if checks is None: checks = ['check-all'] # Set up VS environment, if appropriate. if not vs: # We build by Visual Studio 2015, unless otherwise is requested. vs = r"""%VS140COMNTOOLS%""" f = getLLVMBuildFactoryAndSVNSteps( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, **kwargs) # Pass through all the extra arguments. f.addStep( SetProperty(command=builders_util.getVisualStudioEnvironment( vs, target_arch), extract_fn=builders_util.extractSlaveEnvironment)) env = Property('slave_env') # Some options are required for this build no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ]) cleanBuildRequested = lambda step: step.build.getProperty( "clean", default=step.build.getProperty("clean_obj")) or clean addCmakeSteps(f, cleanBuildRequested=cleanBuildRequested, obj_dir=f.obj_dir, install_dir=f.install_dir, extra_configure_args=cmake_args, env=env, **kwargs) addNinjaSteps(f, obj_dir=obj_dir, checks=checks, install_dir=f.install_dir, env=env, **kwargs) return f
def getLLDBCMakeBuildFactory( clean=False, cmake='cmake', jobs="%(jobs)s", # Source directory containing a built python python_source_dir=None, # Default values for VS devenv and build configuration vs=None, config='Release', target_arch='x86', extra_cmake_args=None, test=False, install=False): ############# PREPARING f = buildbot.process.factory.BuildFactory() # Determine Slave Environment and Set MSVC environment. if vs: f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f = getLLDBSource(f, 'llvm') build_cmd = ['ninja'] install_cmd = ['ninja', 'install'] test_cmd = ['ninja', 'check-lldb'] if jobs: build_cmd.append(WithProperties("-j%s" % jobs)) install_cmd.append(WithProperties("-j%s" % jobs)) test_cmd.append(WithProperties("-j%s" % jobs)) # Global configurations build_dir = 'build' # get full path to build directory f.addStep( SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=build_dir)) ############# CLEANING cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) cmake_cmd = [ "cmake", "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config, "-DCMAKE_INSTALL_PREFIX=../install" ] if python_source_dir: cmake_cmd.append("-DPYTHON_HOME=" + python_source_dir) if extra_cmake_args: cmake_cmd += extra_cmake_args # Note: ShellCommand does not pass the params with special symbols right. # The " ".join is a workaround for this bug. f.addStep( ShellCommand(name="cmake-configure", description=["cmake configure"], command=WithProperties(" ".join(cmake_cmd)), haltOnFailure=True, warnOnWarnings=True, workdir=build_dir, env=Property('slave_env'))) f.addStep( WarningCountingShellCommand(name='build', command=build_cmd, haltOnFailure=True, description='ninja build', workdir=build_dir, env=Property('slave_env'))) ignoreInstallFail = bool(install != 'ignoreFail') f.addStep( ShellCommand(name='install', command=install_cmd, flunkOnFailure=ignoreInstallFail, description='ninja install', workdir=build_dir, doStepIf=bool(install), env=Property('slave_env'))) ignoreTestFail = bool(test != 'ignoreFail') f.addStep( ShellCommand(name='test', command=test_cmd, flunkOnFailure=ignoreTestFail, timeout=2400, description='ninja test', workdir=build_dir, doStepIf=bool(test), env=Property('slave_env'))) return f
def getText(self, cmd, results): if 'version' in self.property_changes: return [ "%s version %s" % (self.package, self.property_changes['version']) ] else: SetProperty.getText(self, cmd, results)
def _getClangCMakeBuildFactory( clean=True, test=True, cmake='cmake', jobs=None, # VS tools environment variable if using MSVC. For example, # %VS120COMNTOOLS% selects the 2013 toolchain. vs=None, vs_target_arch='x86', # Multi-stage compilation useTwoStage=False, testStage1=True, stage1_config='Release', stage2_config='Release', # Test-suite runTestSuite=False, nt_flags=None, testsuite_flags=None, submitURL=None, testerName=None, # Environmental variables for all steps. env=None, extra_cmake_args=None, # Extra repositories checkout_clang_tools_extra=True, checkout_compiler_rt=True, checkout_lld=True, checkout_libcxx=False, checkout_test_suite=False, checkout_flang=False, # Upload artifacts to Google Cloud Storage (for the llvmbisect tool) stage1_upload_directory=None, # Use a lower compression level to generate the build-cache package faster # default is 6 according to documentation xz_compression_factor=6, use_pixz_compression=False, # Triggers trigger_after_stage1=None): ############# PREPARING if nt_flags is None: nt_flags = [] if testsuite_flags is None: testsuite_flags = [] if env is None: env = {} if extra_cmake_args is None: extra_cmake_args = [] if trigger_after_stage1 is None: trigger_after_stage1 = [] clean_build_requested = lambda step: \ step.build.getProperty( \ "clean", \ default=step.build.getProperty("clean_obj") \ ) or clean # We *must* checkout at least Clang+LLVM depends_on_projects = ['llvm', 'clang'] if checkout_clang_tools_extra: depends_on_projects.append('clang-tools-extra') if checkout_compiler_rt: depends_on_projects.append('compiler-rt') if checkout_lld: depends_on_projects.append('lld') if checkout_libcxx: depends_on_projects.append('libcxx') depends_on_projects.append('libcxxabi') depends_on_projects.append('libunwind') if checkout_flang: depends_on_projects.append('flang') depends_on_projects.append('mlir') f = LLVMBuildFactory(depends_on_projects=depends_on_projects, llvm_srcdir='llvm') # Checkout the latest code for LNT # and the test-suite separately. Le's do this first, # so we wouldn't poison got_revision property. if runTestSuite or checkout_test_suite: f.addGetSourcecodeForProject(project='lnt', src_dir='test/lnt', alwaysUseLatest=True) f.addGetSourcecodeForProject(project='test-suite', src_dir='test/test-suite', alwaysUseLatest=True) # Then get the LLVM source code revision this particular build is for. f.addGetSourcecodeSteps() # If jobs not defined, Ninja will choose a suitable value jobs_cmd = [] lit_args = "'-v" if jobs is not None: jobs_cmd = ["-j" + str(jobs)] lit_args += " -j" + str(jobs) + "'" else: lit_args += "'" ninja_cmd = ['ninja'] + jobs_cmd ninja_install_cmd = ['ninja', 'install'] + jobs_cmd ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd # Global configurations stage1_build = 'stage1' stage1_install = 'stage1.install' stage2_build = 'stage2' stage2_install = 'stage2.install' # Set up VS environment, if appropriate. if vs and vs != "manual": f.addStep( SetProperty(command=builders_util.getVisualStudioEnvironment( vs, vs_target_arch), extract_fn=builders_util.extractSlaveEnvironment)) assert not env, "Can't have custom builder env vars with VS" env = Property('slave_env') ############# CLEANING f.addStep( ShellCommand(name='clean stage 1', command=['rm', '-rf', stage1_build], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='cleaning stage 1', descriptionDone='clean', workdir='.', doStepIf=clean_build_requested)) ############# STAGE 1 CmakeCommand.applyRequiredOptions(extra_cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage1_build) f.addStep( ShellCommand( name='cmake stage 1', command=[ cmake, "-G", "Ninja", rel_src_dir, "-DCMAKE_BUILD_TYPE=" + stage1_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args, "-DCMAKE_INSTALL_PREFIX=../" + stage1_install ] + extra_cmake_args, haltOnFailure=True, description='cmake stage 1', workdir=stage1_build, doStepIf=FileDoesNotExist("build.ninja"), env=env)) f.addStep( WarningCountingShellCommand(name='build stage 1', command=ninja_cmd, haltOnFailure=True, description='ninja all', workdir=stage1_build, env=env)) if test and testStage1: haltOnStage1Check = not useTwoStage and not runTestSuite f.addStep( LitTestCommand(name='ninja check 1', command=ninja_check_cmd, haltOnFailure=haltOnStage1Check, description=["checking stage 1"], descriptionDone=["stage 1 checked"], workdir=stage1_build, env=env)) if useTwoStage or runTestSuite or stage1_upload_directory: f.addStep( ShellCommand(name='clean stage 1 install', command=['rm', '-rf', stage1_install], warnOnFailure=True, haltOnFailure=False, flunkOnFailure=False, description='cleaning stage 1 install', descriptionDone='clean', workdir='.')) f.addStep( ShellCommand(name='install stage 1', command=ninja_install_cmd, description='ninja install', workdir=stage1_build, env=env)) if stage1_upload_directory: addGCSUploadSteps(f, 'stage 1', stage1_install, stage1_upload_directory, env, gcs_url_property='stage1_package_gcs_url', use_pixz_compression=use_pixz_compression, xz_compression_factor=xz_compression_factor) # Compute the cmake define flag to set the C and C++ compiler to clang. Use # clang-cl if we used MSVC for stage1. if not vs: cc = 'clang' cxx = 'clang++' else: cc = 'clang-cl.exe' cxx = 'clang-cl.exe' ############# STAGE 2 if useTwoStage: # We always cleanly build the stage 2. If the compiler has been # changed on the stage 1, we cannot trust any of the intermediate file # from the old compiler. And if the stage 1 compiler is the same, we # should not build in the first place. f.addStep( ShellCommand(name='clean stage 2', command=['rm', '-rf', stage2_build], warnOnFailure=True, description='cleaning stage 2', descriptionDone='clean', workdir='.')) # Set the compiler using the CC and CXX environment variables to work around # backslash string escaping bugs somewhere between buildbot and cmake. The # env.exe helper is required to run the tests, so hopefully it's already on # PATH. rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage2_build) cmake_cmd2 = [ cmake, "-G", "Ninja", rel_src_dir, WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + stage1_install + "/bin/" + cc), WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" + stage1_install + "/bin/" + cxx), "-DCMAKE_BUILD_TYPE=" + stage2_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args, "-DCMAKE_INSTALL_PREFIX=../" + stage2_install ] + extra_cmake_args f.addStep( ShellCommand(name='cmake stage 2', command=cmake_cmd2, haltOnFailure=True, description='cmake stage 2', workdir=stage2_build, env=env)) f.addStep( WarningCountingShellCommand(name='build stage 2', command=ninja_cmd, haltOnFailure=True, description='ninja all', workdir=stage2_build, env=env)) if test: f.addStep( LitTestCommand(name='ninja check 2', command=ninja_check_cmd, haltOnFailure=not runTestSuite, description=["checking stage 2"], descriptionDone=["stage 2 checked"], workdir=stage2_build, env=env)) ############# TEST SUITE ## Test-Suite (stage 2 if built, stage 1 otherwise) if runTestSuite: compiler_path = stage1_install if useTwoStage: compiler_path = stage2_install f.addStep( ShellCommand(name='clean stage 2 install', command=['rm', '-rf', stage2_install], warnOnFailure=True, description='cleaning stage 2 install', descriptionDone='clean', workdir='.')) f.addStep( ShellCommand(name='install stage 2', command=ninja_install_cmd, description='ninja install 2', workdir=stage2_build, env=env)) # Get generated python, lnt python = WithProperties('%(workdir)s/test/sandbox/bin/python') lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt') lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py') # Paths sandbox = WithProperties('%(workdir)s/test/sandbox') test_suite_dir = WithProperties('%(workdir)s/test/test-suite') # Get latest built Clang (stage1 or stage2) cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc) cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx) # LNT Command line (don't pass -jN. Users need to pass both --threads # and --build-threads in nt_flags/test_suite_flags to get the same effect) use_runtest_testsuite = len(nt_flags) == 0 if not use_runtest_testsuite: test_suite_cmd = [ python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx ] # Append any option provided by the user test_suite_cmd.extend(nt_flags) else: lit = WithProperties('%(workdir)s/' + stage1_build + '/bin/llvm-lit') test_suite_cmd = [ python, lnt, 'runtest', 'test-suite', '--no-timestamp', '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx, '--use-lit', lit ] # Append any option provided by the user test_suite_cmd.extend(testsuite_flags) # Only submit if a URL has been specified if submitURL is not None: if not isinstance(submitURL, list): submitURL = [submitURL] for url in submitURL: test_suite_cmd.extend(['--submit', url]) # lnt runtest test-suite doesn't understand --no-machdep-info: if testerName and not use_runtest_testsuite: test_suite_cmd.extend(['--no-machdep-info', testerName]) # CC and CXX are needed as env for build-tools test_suite_env = copy.deepcopy(env) test_suite_env['CC'] = cc test_suite_env['CXX'] = cxx # Steps to prepare, build and run LNT f.addStep( ShellCommand(name='clean sandbox', command=['rm', '-rf', 'sandbox'], haltOnFailure=True, description='removing sandbox directory', workdir='test', env=env)) f.addStep( ShellCommand(name='recreate sandbox', command=['virtualenv', 'sandbox'], haltOnFailure=True, description='recreating sandbox', workdir='test', env=env)) f.addStep( ShellCommand(name='setup lit', command=[python, lnt_setup, 'develop'], haltOnFailure=True, description='setting up LNT in sandbox', workdir='test/sandbox', env=env)) f.addStep( LitTestCommand(name='test-suite', command=test_suite_cmd, haltOnFailure=True, description=['running the test suite'], workdir='test/sandbox', logfiles={ 'configure.log': 'build/configure.log', 'build-tools.log': 'build/build-tools.log', 'test.log': 'build/test.log', 'report.json': 'build/report.json' }, env=test_suite_env)) return f
def getSanitizerWindowsBuildFactory( clean=False, cmake='cmake', # Default values for VS devenv and build configuration vs=r"""%VS120COMNTOOLS%""", config='Release', target_arch='x86', extra_cmake_args=[]): ############# PREPARING f = buildbot.process.factory.BuildFactory() # Kill any stale symbolizer processes for the last run. If there are any # stale processes, the build will fail during linking. This can happen to # any process, but it is most likely to happen to llvm-symbolizer if its # pipe isn't closed. taskkill_cmd = 'taskkill /f /im llvm-symbolizer.exe || exit /b 0' f.addStep( ShellCommand(name='taskkill', description='kill stale processes', command=['cmd', '/c', taskkill_cmd], haltOnFailure=False)) # Determine Slave Environment and Set MSVC environment. f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f = getSource(f, 'llvm') # Global configurations build_dir = 'build' ############# CLEANING cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) f.addStep( ShellCommand( name='cmake', command=[ cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config, "-DLLVM_ENABLE_ASSERTIONS=ON" ] + extra_cmake_args, haltOnFailure=True, workdir=build_dir, env=Property('slave_env'))) # Build compiler-rt first to speed up detection of Windows-specific # compiler-time errors in the sanitizer runtimes. f.addStep( NinjaCommand(name='build compiler-rt', targets=['compiler-rt'], haltOnFailure=True, description='ninja compiler-rt', workdir=build_dir, env=Property('slave_env'))) # Build Clang and LLD next so that most compilation errors occur in a build # step. f.addStep( NinjaCommand(name='build clang lld', targets=['clang', 'lld'], haltOnFailure=True, description='ninja clang lld', workdir=build_dir, env=Property('slave_env'))) # Only run sanitizer tests. # Don't build targets that are not required in order to speed up the cycle. for target in [ 'check-asan', 'check-asan-dynamic', 'check-sanitizer', 'check-cfi' ]: f.addStep( NinjaCommand(name='run %s' % target, targets=[target], haltOnFailure=False, description='ninja %s' % target, workdir=build_dir, env=Property('slave_env'))) return f
def getLLDBxcodebuildFactory(use_cc=None, build_type='Debug', remote_configs=None, env=None): if env is None: env = {} f = buildbot.process.factory.BuildFactory() f.addStep( SetProperty(name='get_builddir', command=['pwd'], property='builddir', description='set build dir', workdir='.')) lldb_srcdir = 'lldb' OBJROOT = '%(builddir)s/' + lldb_srcdir + '/build' f.addStep( SetProperty(name='get_bindir', command=[ 'echo', WithProperties('%(builddir)s/' + lldb_srcdir + '/build/' + build_type) ], property='lldb_bindir', description='set bin dir', workdir='.')) # cleaning out the build directory is vital for codesigning. f.addStep( ShellCommand(name='clean.lldb-buid', command=['rm', '-rf', WithProperties(OBJROOT)], haltOnFailure=True, workdir=WithProperties('%(builddir)s'))) f.addStep( ShellCommand(name='clean.llvm-buid', command=['rm', '-rf', '%s/llvm-build' % lldb_srcdir], haltOnFailure=True, workdir=WithProperties('%(builddir)s'))) f.addStep( ShellCommand(name='clean.test trace', command='rm -rf build/*', haltOnFailure=False, flunkOnFailure=False, workdir='.')) # Remove symbolic link to lldb, otherwise xcodebuild will have circular dependency f.addStep( ShellCommand(name='remove symbolic link lldb', command=['rm', lldb_srcdir + '/llvm/tools/lldb'], haltOnFailure=False, flunkOnFailure=False, workdir=WithProperties('%(builddir)s'))) f.addStep( SVN(name='svn-lldb', mode='update', baseURL='http://llvm.org/svn/llvm-project/lldb/', defaultBranch='trunk', workdir=lldb_srcdir)) f.addStep( SVN(name='svn-llvm', mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir='%s/llvm' % lldb_srcdir)) f.addStep( SVN(name='svn-clang', mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/llvm/tools/clang' % lldb_srcdir)) # setup keychain for codesign # In order for the codesigning to work inside of buildbot, security must be # called to unlock the keychain, which requires a password. # I've set up a special keychain for this purpose, so as to not compromise # the login password of the buildslave. # This means I have to set the special keychain as the default and unlock it # prior to building the sources. f.addStep( ShellCommand(name='check.keychain', command=['security', 'default-keychain'], haltOnFailure=True, workdir=WithProperties('%(builddir)s'))) f.addStep( ShellCommand( name='find.certificate', command=['security', 'find-certificate', '-c', 'lldb_codesign'], haltOnFailure=True, workdir=WithProperties('%(builddir)s'))) # Building the sources # buildcmd = ' '.join([ 'xcrun', 'xcodebuild', '-target', 'desktop', '-configuration', build_type, 'SYMROOT=' + OBJROOT, 'OBJROOT=' + OBJROOT ]) f.addStep( ShellCommand(name='lldb-build', command=WithProperties(buildcmd + " || " + buildcmd), haltOnFailure=True, workdir=lldb_srcdir)) # Testing # if not use_cc: use_cc = '/Applications/Xcode.app/Contents/Developer/Toolchains/' use_cc += 'XcodeDefault.xctoolchain/usr/bin/clang' f.addStep( SetProperty(name='set.cc', command=['xcrun', '-find', 'clang'], property='use_cc', description='set cc', workdir=lldb_srcdir)) else: f.addStep( SetProperty(name='set.cc', command=['echo', use_cc], property='use_cc', description='set cc', workdir=lldb_srcdir)) DOTEST_OPTS = ' '.join([ '--executable', '%(lldb_bindir)s/lldb', '--filecheck', '%(lldb_bindir)s/FileCheck', '--framework', '%(lldb_bindir)s/LLDB.framework', '-A', 'x86_64', '-C', 'clang', '-s', '../../build/lldb-test-traces' ]) f.addStep( LitTestCommand( name='lldb-test', command=['./dosep.py', '--options', WithProperties(DOTEST_OPTS)], haltOnFailure=False, workdir='%s/test' % lldb_srcdir, env={'DYLD_FRAMEWORK_PATH': WithProperties('%(lldb_bindir)s')})) # Remote test steps if remote_configs is not None: # Source structure to use cmake command f.addStep( SetProperty( name='get tblgen bindir', command=[ 'echo', WithProperties( '%(builddir)s/' + lldb_srcdir + '/llvm-build/Release+Asserts/x86_64/Release+Asserts/bin' ) ], property='tblgen_bindir', description='set tblgen binaries dir', workdir='.')) f = getSymbLinkSteps(f, lldb_srcdir) for config in remote_configs: f = getLLDBRemoteTestSteps( f, '%(lldb_bindir)s', build_type, config, env={'DYLD_FRAMEWORK_PATH': WithProperties('%(lldb_bindir)s')}) # Compress and upload test log f = archiveLLDBTestTraces(f, "build/lldb-test-traces*") # Results go in a directory coded named according to the date and time of the test run, e.g.: # # 2012-10-16-11_26_48/Failure-x86_64-_Applications_Xcode.app_Contents_Developer_Toolchains_XcodeDefault.xctoolchain_usr_bin_clang-TestLogging.LogTestCase.test_with_dsym.log # # Possible results are ExpectedFailure, Failure, SkippedTest, UnexpectedSuccess, and Error. return f return f
def getLLDBScriptCommandsFactory( downloadBinary=True, buildAndroid=False, runTest=True, scriptExt='.sh', extra_cmake_args=None, depends_on_projects=None, ): if scriptExt is '.bat': pathSep = '.\\' else: pathSep = './' if extra_cmake_args is None: extra_cmake_args = [] if depends_on_projects is None: f = buildbot.process.factory.BuildFactory() else: f = LLVMBuildFactory(depends_on_projects=depends_on_projects) # Update scripts getShellCommandStep(f, name='update scripts', command=['updateScripts' + scriptExt]) # Acquire lock if downloadBinary: getShellCommandStep( f, name='acquire lock', command=[pathSep + 'acquireLock' + scriptExt, 'totBuild'], description='get') # Checkout source code getShellCommandStep(f, name='checkout source code', command=[ pathSep + 'checkoutSource' + scriptExt, WithProperties('%(revision)s') ]) # Set source revision f.addStep( SetProperty(name="set revision", command=[pathSep + 'getRevision' + scriptExt], property="got_revision", workdir="scripts")) # Configure getShellCommandStep(f, name='cmake local', command=[pathSep + 'cmake' + scriptExt] + extra_cmake_args) # Build getShellCommandStep(f, name='ninja build local', command=[pathSep + 'buildLocal' + scriptExt]) if buildAndroid: getShellCommandStep(f, name='build android', command=[pathSep + 'buildAndroid' + scriptExt]) # Get lldb-server binaries if downloadBinary: getShellCommandStep(f, name='get lldb-server binaries', command=[ pathSep + 'downloadBinaries' + scriptExt, WithProperties('%(got_revision)s') ]) # Test if runTest: f.addStep( LitTestCommand(name="run unit tests", command=[pathSep + 'testUnit' + scriptExt], description=["testing"], descriptionDone=["unit test"], workdir='scripts')) getTestSteps(f, scriptExt, pathSep) # upload test traces getShellCommandStep(f, name='upload test traces', command=[ pathSep + 'uploadTestTrace' + scriptExt, WithProperties('%(buildnumber)s'), WithProperties('%(buildername)s') ], flunkOnFailure=False) # Upload lldb-server binaries and trigger android builders if buildAndroid: getShellCommandStep(f, name='upload lldb-server binaries', command=[pathSep + 'uploadBinaries' + scriptExt]) f.addStep( trigger.Trigger(schedulerNames=['lldb_android_scheduler'], updateSourceStamp=False, waitForFinish=False)) # Release lock if downloadBinary: getShellCommandStep( f, name='release lock', command=[pathSep + 'releaseLock' + scriptExt, 'totBuild'], description='release', alwaysRun=True) return f
def _getClangCMakeBuildFactory( clean=True, test=True, cmake='cmake', jobs=None, # VS tools environment variable if using MSVC. For example, # %VS120COMNTOOLS% selects the 2013 toolchain. vs=None, vs_target_arch='x86', # Multi-stage compilation useTwoStage=False, testStage1=True, stage1_config='Release', stage2_config='Release', # Test-suite runTestSuite=False, nt_flags=[], submitURL=None, testerName=None, # Environmental variables for all steps. env={}, extra_cmake_args=[], # Extra repositories checkout_clang_tools_extra=True, checkout_compiler_rt=True, # Upload artifacts to Google Cloud Storage (for the llvmbisect tool) stage1_upload_directory=None, # Triggers trigger_after_stage1=[]): ############# PREPARING f = buildbot.process.factory.BuildFactory() addSVNUpdateSteps(f, checkout_clang_tools_extra=checkout_clang_tools_extra, checkout_compiler_rt=checkout_compiler_rt, checkout_test_suite=runTestSuite) # If jobs not defined, Ninja will choose a suitable value jobs_cmd = [] lit_args = "'-v" if jobs is not None: jobs_cmd = ["-j" + str(jobs)] lit_args += " -j" + str(jobs) + "'" else: lit_args += "'" ninja_cmd = ['ninja'] + jobs_cmd ninja_install_cmd = ['ninja', 'install'] + jobs_cmd ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd check_build_cmd = [ "sh", "-c", "test -e build.ninja && echo OK || echo Missing" ] if vs: check_build_cmd = [ "cmd", "/c", "if exist build.ninja (echo OK) " + " else (echo Missing & exit 1)" ] # Global configurations stage1_build = 'stage1' stage1_install = 'stage1.install' stage2_build = 'stage2' stage2_install = 'stage2.install' # Set up VS environment, if appropriate. if vs: f.addStep( SetProperty(command=builders_util.getVisualStudioEnvironment( vs, vs_target_arch), extract_fn=builders_util.extractSlaveEnvironment)) assert not env, "Can't have custom builder env vars with VS" env = Property('slave_env') ############# CLEANING if clean: f.addStep( ShellCommand(name='clean stage 1', command=['rm', '-rf', stage1_build], warnOnFailure=True, description='cleaning stage 1', descriptionDone='clean', workdir='.', env=env)) else: f.addStep( SetProperty(name="check ninja files 1", workdir=stage1_build, command=check_build_cmd, flunkOnFailure=False, property="exists_ninja_1")) ############# STAGE 1 f.addStep( ShellCommand( name='cmake stage 1', command=[ cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + stage1_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args, "-DCMAKE_INSTALL_PREFIX=../" + stage1_install ] + extra_cmake_args, haltOnFailure=True, description='cmake stage 1', workdir=stage1_build, doStepIf=lambda step: step.build.getProperty("exists_ninja_1" ) != "OK", env=env)) f.addStep( WarningCountingShellCommand(name='build stage 1', command=ninja_cmd, haltOnFailure=True, description='ninja all', workdir=stage1_build, env=env)) if test and testStage1: haltOnStage1Check = not useTwoStage and not runTestSuite f.addStep( lit_test_command.LitTestCommand( name='ninja check 1', command=ninja_check_cmd, haltOnFailure=haltOnStage1Check, description=["checking stage 1"], descriptionDone=["stage 1 checked"], workdir=stage1_build, env=env)) if useTwoStage or runTestSuite or stage1_upload_directory: f.addStep( ShellCommand(name='install stage 1', command=ninja_install_cmd, description='ninja install', workdir=stage1_build, env=env)) if stage1_upload_directory: addGCSUploadSteps(f, 'stage 1', stage1_install, stage1_upload_directory, env, gcs_url_property='stage1_package_gcs_url') # Compute the cmake define flag to set the C and C++ compiler to clang. Use # clang-cl if we used MSVC for stage1. if not vs: cc = 'clang' cxx = 'clang++' else: cc = 'clang-cl.exe' cxx = 'clang-cl.exe' ############# STAGE 2 if useTwoStage: # We always cleanly build the stage 2. If the compiler has been # changed on the stage 1, we cannot trust any of the intermediate file # from the old compiler. And if the stage 1 compiler is the same, we # should not build in the first place. f.addStep( ShellCommand(name='clean stage 2', command=['rm', '-rf', stage2_build], warnOnFailure=True, description='cleaning stage 2', descriptionDone='clean', workdir='.', env=env)) # Set the compiler using the CC and CXX environment variables to work around # backslash string escaping bugs somewhere between buildbot and cmake. The # env.exe helper is required to run the tests, so hopefully it's already on # PATH. cmake_cmd2 = [ 'env', WithProperties('CC=%(workdir)s/' + stage1_install + '/bin/' + cc), WithProperties('CXX=%(workdir)s/' + stage1_install + '/bin/' + cxx), cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + stage2_config, "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args, "-DCMAKE_INSTALL_PREFIX=../" + stage2_install ] + extra_cmake_args f.addStep( ShellCommand(name='cmake stage 2', command=cmake_cmd2, haltOnFailure=True, description='cmake stage 2', workdir=stage2_build, doStepIf=lambda step: step.build.getProperty( "exists_ninja_2") != "OK", env=env)) f.addStep( WarningCountingShellCommand(name='build stage 2', command=ninja_cmd, haltOnFailure=True, description='ninja all', workdir=stage2_build, env=env)) if test: f.addStep( lit_test_command.LitTestCommand( name='ninja check 2', command=ninja_check_cmd, haltOnFailure=not runTestSuite, description=["checking stage 2"], descriptionDone=["stage 2 checked"], workdir=stage2_build, env=env)) ############# TEST SUITE ## Test-Suite (stage 2 if built, stage 1 otherwise) if runTestSuite: compiler_path = stage1_install if useTwoStage: compiler_path = stage2_install f.addStep( ShellCommand(name='install stage 2', command=ninja_install_cmd, description='ninja install 2', workdir=stage2_build, env=env)) # Get generated python, lnt python = WithProperties('%(workdir)s/test/sandbox/bin/python') lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt') lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py') # Paths sandbox = WithProperties('%(workdir)s/test/sandbox') test_suite_dir = WithProperties('%(workdir)s/test/test-suite') # Get latest built Clang (stage1 or stage2) cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc) cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx) # LNT Command line (don't pass -jN. Users need to pass both --threads # and --build-threads in nt_flags to get the same effect) test_suite_cmd = [ python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx ] # Append any option provided by the user test_suite_cmd.extend(nt_flags) # Only submit if a URL has been specified if submitURL is not None: if not isinstance(submitURL, list): submitURL = [submitURL] for url in submitURL: test_suite_cmd.extend(['--submit', url]) if testerName: test_suite_cmd.extend(['--no-machdep-info', testerName]) # CC and CXX are needed as env for build-tools test_suite_env = copy.deepcopy(env) test_suite_env['CC'] = cc test_suite_env['CXX'] = cxx # Steps to prepare, build and run LNT f.addStep( ShellCommand(name='clean sandbox', command=['rm', '-rf', 'sandbox'], haltOnFailure=True, description='removing sandbox directory', workdir='test', env=env)) f.addStep( ShellCommand(name='recreate sandbox', command=['virtualenv', 'sandbox'], haltOnFailure=True, description='recreating sandbox', workdir='test', env=env)) f.addStep( ShellCommand(name='setup lit', command=[python, lnt_setup, 'develop'], haltOnFailure=True, description='setting up LNT in sandbox', workdir='test/sandbox', env=env)) f.addStep( commands.LitTestCommand.LitTestCommand( name='test-suite', command=test_suite_cmd, haltOnFailure=True, description=['running the test suite'], workdir='test/sandbox', logfiles={ 'configure.log': 'build/configure.log', 'build-tools.log': 'build/build-tools.log', 'test.log': 'build/test.log', 'report.json': 'build/report.json' }, env=test_suite_env)) return f