def get_configuration_build_steps(link, type, options=[]): from buildbot.steps.slave import RemoveDirectory from buildbot.steps.slave import MakeDirectory from buildbot.process.properties import Interpolate return [ MakeDirectory(name='create build directory', description=['preparing build directory'], descriptionDone=['create build directory'], dir=Interpolate('%(prop:workdir)s/build/build'), doStepIf=lambda step: (bool(options) and ('osx' in step.build.getProperty('buildername'))), hideStepIf=skipped_or_success), get_cmake_step(link, type, options), get_build_step(link, type, options), RemoveDirectory(name='remove build directory', description=['removing build directory'], descriptionDone=['remove build directory'], dir=Interpolate('%(prop:workdir)s/build/build'), doStepIf=lambda step: (bool(options) and ('osx' in step.build.getProperty('buildername'))), hideStepIf=skipped_or_success) ]
def getLLVMBuildFactoryAndPrepareForSourcecodeSteps( depends_on_projects = None, llvm_srcdir = None, obj_dir = None, install_dir = None, cleanBuildRequested = None, env = None, **kwargs): def cleanBuildRequestedByProperty(step): return step.build.getProperty("clean") if cleanBuildRequested is None: # We want a clean checkout only if requested by the property. cleanBuildRequested = cleanBuildRequestedByProperty f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, cleanBuildRequested=cleanBuildRequested, **kwargs) # Pass through all the extra arguments. # Remove the source code for a clean checkout if requested by property. # TODO: Some Windows slaves do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep(RemoveDirectory(name='clean-src-dir', dir=f.monorepo_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequestedByProperty, )) return f
def ros_debbuild(c, job_name, packages, url, distro, arch, rosdistro, version, machines, othermirror, keys, trigger_pkgs = None): gbp_args = ['-uc', '-us', '--git-ignore-branch', '--git-ignore-new', '--git-verbose', '--git-dist='+distro, '--git-arch='+arch] f = BuildFactory() # Remove the build directory. f.addStep( RemoveDirectory( name = job_name+'-clean', dir = Interpolate('%(prop:workdir)s'), hideStepIf = success, ) ) # Check out the repository master branch, since releases are tagged and not branched f.addStep( Git( repourl = url, branch = 'master', alwaysUseLatest = True, # this avoids broken builds when schedulers send wrong tag/rev mode = 'full' # clean out old versions ) ) # Update the cowbuilder f.addStep( ShellCommand( command = ['cowbuilder-update.py', distro, arch] + keys, hideStepIf = success ) ) # Need to build each package in order for package in packages: debian_pkg = 'ros-'+rosdistro+'-'+package.replace('_','-') # debian package name (ros-groovy-foo) branch_name = 'debian/'+debian_pkg+'_%(prop:release_version)s_'+distro # release branch from bloom deb_name = debian_pkg+'_%(prop:release_version)s'+distro final_name = debian_pkg+'_%(prop:release_version)s-%(prop:datestamp)s'+distro+'_'+arch+'.deb' # Check out the proper tag. Use --force to delete changes from previous deb stamping f.addStep( ShellCommand( haltOnFailure = True, name = package+'-checkout', command = ['git', 'checkout', Interpolate(branch_name), '--force'], hideStepIf = success ) ) # Download script for building the source deb f.addStep( FileDownload( name = job_name+'-grab-build-source-deb-script', mastersrc = 'scripts/build_source_deb.py', slavedest = Interpolate('%(prop:workdir)s/build_source_deb.py'), mode = 0755, hideStepIf = success ) )
def get_clean_step(): from buildbot.steps.slave import RemoveDirectory from buildbot.process.properties import Interpolate return [ RemoveDirectory(name='clean slave', description=['cleaning slave'], descriptionDone=['clean slave'], dir=Interpolate('%(prop:workdir)s'), alwaysRun=True, hideStepIf=skipped_or_success) ]
def add_artifact_pre_build_steps(job): artifacts_dir = Interpolate(join('%(prop:builddir)s', 'artifacts')) job.add_step( SetProperty(property="artifactsdir", value=artifacts_dir, hideStepIf=True)) job.add_step( RemoveDirectory(dir=Interpolate('%(prop:artifactsdir)s'), hideStepIf=True)) job.add_step( MakeDirectory(dir=Interpolate('%(prop:artifactsdir)s'), hideStepIf=True))
def __init__(self, source, uncleanWarnings, python="python", reactor="epoll", easy_install="easy_install", dependencies=[]): TwistedBaseFactory.__init__(self, python, source, uncleanWarnings) self.addStep(RemoveDirectory(self.virtualenv)) setupCommands = [{ 'name': 'create_virtualenv', 'description': ['creating', 'virtualenv'], 'descriptionDone': ['created', 'virtualenv'], 'command': ["virtualenv", "--never-download", self.virtualenv], }] for dependency in dependencies: setupCommands.append({ 'name': 'install_%s' % dependency, 'description': ['installing', dependency], 'descriptionDone': ['installed', dependency], 'command': [easy_install, dependency], }) setupCommands.append({ 'name': 'install_twisted', 'description': ['installing', 'twisted'], 'descriptionDone': ['installed', 'twisted'], 'command': [easy_install, "."] }) for command in setupCommands: self.addStep( shell.ShellCommand(env={"PATH": self.virtualenvPath}, haltOnFailure=True, **command)) self.addTrialStep(name=reactor, reactor=reactor, flunkOnFailure=True, warnOnFailure=False, workdir=self.virtualenv, env={"PATH": self.virtualenvPath})
def getLLVMBuildFactoryAndSVNSteps(depends_on_projects=None, llvm_srcdir=None, obj_dir=None, install_dir=None, cleanBuildRequested=None, env=None, **kwargs): def cleanBuildRequestedByProperty(step): return step.build.getProperty("clean") # Set defaults if not depends_on_projects: depends_on_projects = ['llvm', 'clang'] if cleanBuildRequested is None: # We want a clean checkout only if requested by the property. cleanBuildRequested = cleanBuildRequestedByProperty f = LLVMBuildFactory(depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir or "llvm", obj_dir=obj_dir or "build", install_dir=install_dir, cleanBuildRequested=cleanBuildRequested, **kwargs) # Pass through all the extra arguments. # Do a clean checkout if requested by a build property. # TODO: Some Windows slaves do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep( RemoveDirectory( name='clean-src-dir', dir=f.llvm_srcdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequestedByProperty, )) # Get the source code. f.addSVNSteps() return f
def rqg_win_factory(mtr_build_thread="130", config="Debug"): if config == 'Debug': do_debug_steps = True do_release_steps = False else: do_debug_steps = False do_release_steps = True f = factory.BuildFactory() # We shouldn't need it anymore since we are setting appverif in runall.pl now, but let it be here just in case f.addStep( ShellCommand(name="disable_app_verifier", command=["dojob", "appverif", "/n", "mysqld.exe"], doStepIf=do_release_steps)) # that's where pre-cloned trees (mariadb-server, rqg, mariadb-toolbox etc.) and local scripts are f.addStep( SetPropertyFromCommand(name="set_shared_dir", property="sharedir", command=["dojob", "echo E:\\buildbot"])) # that's where we will build servers f.addStep( SetPropertyFromCommand( name="set_bb_workdir", property="bb_workdir", command=["dojob", WithProperties("echo D:\\%(buildername)s")])) # that's where the main server (server under test) will be built f.addStep( SetPropertyFromCommand( name="set_builddir", property="builddir", command=[ "dojob", WithProperties("echo D:\\%(buildername)s\\build") ])) # logdir is where vardirs are written f.addStep( SetPropertyFromCommand( name="set_logdir", property="logdir", command=[ "dojob", WithProperties( "echo %(sharedir)s\\vardirs\\%(buildername)s\\%(branch)s-%(buildnumber)s" ) ])) f.addStep( ShellCommand( name="close_open_handles", command=[ "dojob", WithProperties( "cd /d %(bb_workdir)s && %(sharedir)s\\mariadb-tools\\buildbot\\unlock_handles.bat" ) ], alwaysRun=True)) # f.addStep(ShellCommand( # name= "kill_stale_mysqld", # command=["dojob", WithProperties("taskkill /IM mysqld.exe /F || tasklist")], # locks=[kill_mysqld_lock.access('exclusive')] # )); f.addStep( ShellCommand( name="kill_stale_mysqld", command=[ "dojob", WithProperties( "PowerShell -Command \"Get-WmiObject -class Win32_Process | Where-Object { $_.Name -eq 'mysqld.exe' -and $_.Path -like '*\%(buildername)s\*'} | Select-Object\" && PowerShell -Command \"Get-WmiObject -class Win32_Process | Where-Object { $_.Name -eq 'mysqld.exe' -and $_.Path -like '*\%(buildername)s\*'} | Remove-WmiObject\" && PowerShell -Command \"Get-WmiObject -class Win32_Process | Where-Object { $_.Name -eq 'mysqld.exe' -and $_.Path -like '*\%(buildername)s\*'} | Select-Object\"" ) ])) f.addStep( RemoveDirectory(name="remove_old_builds", dir=WithProperties("%(bb_workdir)s"))) f.addStep( RemoveDirectory(name="remove_old_logs", dir=WithProperties("%(logdir)s"))) f.addStep( ShellCommand( name="create_dirs", command=[ "dojob", WithProperties( "mkdir %(bb_workdir)s && mkdir %(bb_workdir)s\\build-last-release" ) ], timeout=3600)) f.addStep( ShellCommand( name="pull_server", command=[ "dojob", WithProperties( "git clone %(sharedir)s\\mariadb-server %(builddir)s && cd /d %(builddir)s && git remote set-url origin %(repository)s && git pull && git reset --hard %(revision)s" ) ], timeout=3600)) f.addStep( ShellCommand( name="pull_rqg_and_tools", command=[ "dojob", WithProperties( "cd /d %(sharedir)s\\rqg && git pull && cd /d %(sharedir)s\\mariadb-toolbox && git pull" ) ], locks=[git_rqg_lock.access('exclusive')], timeout=3600)) f.addStep( SetPropertyFromCommand( name="get_generator", property="vs_generator", command=[ "dojob", WithProperties("cat %(sharedir)s\\vs_generator.txt") ])) f.addStep( ShellCommand( name="version_info", command=[ "dojob", WithProperties( "cd /d %(builddir)s && git log -1 && cd /d %(sharedir)s\\rqg && git log -1 && cd /d %(sharedir)s\\mariadb-toolbox && git log -1" ) ], timeout=3600)) f.addStep( Compile( name="build", command=[ "dojob", WithProperties( "cd /d %(builddir)s && cmake . -G %(vs_generator)s && cmake --build . --config " + config) ], warningPattern=vsWarningPattern, warningExtractor=Compile.warnExtractFromRegexpGroups)) # We shouldn't need it anymore since we are setting appverif in runall.pl now # f.addStep(Test( # name = "enable_app_verifier", # doStepIf=do_release_steps, # command=["dojob", "appverif", "/verify", "mysqld.exe"] # )); # storage tests are currently broken on 10.2 (MDEV-9705) f.addStep( getMTR( doStepIf=do_release_steps, name="storage_engine", test_type="storage_engine", test_info="Storage engine test suites", timeout=3600, command=[ "dojob", WithProperties( "cd /d %(builddir)s\\mysql-test && perl mysql-test-run.pl --verbose-restart --force --suite=storage_engine-,storage_engine/*- --max-test-fail=0 --parallel=4" ) ])) f.addStep( Test( doStepIf=do_release_steps, name="transform", timeout=3600, env={"MTR_BUILD_THREAD": mtr_build_thread}, command=[ "dojob", WithProperties( "cd /d %(sharedir)s\\rqg && perl combinations.pl --config=%(sharedir)s\\mariadb-toolbox\\configs\\buildbot-transform.cc --run-all-combinations-once --force --basedir=%(builddir)s --workdir=%(logdir)s\\optim-transform || perl %(sharedir)s\\mariadb-toolbox\\scripts\\result_summary.pl %(logdir)s\\optim-transform\\trial*" ) ])) # We shouldn't need it anymore since we are setting appverif in runall.pl now # f.addStep(ShellCommand( # doStepIf=do_release_steps, # name= "disable_app_verifier_again", # command=["dojob", "appverif", "/n", "mysqld.exe"] # alwaysRun=True # )); f.addStep( Test( doStepIf=do_debug_steps, name="crash_tests", timeout=3600, env={"MTR_BUILD_THREAD": mtr_build_thread}, command=[ "dojob", WithProperties( "cd /d %(sharedir)s\\rqg && perl combinations.pl --config=%(sharedir)s\\mariadb-toolbox\\configs\\buildbot-no-comparison.cc --run-all-combinations-once --force --basedir=%(builddir)s --workdir=%(logdir)s\\optim-crash-tests || perl %(sharedir)s\\mariadb-toolbox\\scripts\\result_summary.pl %(logdir)s\\optim-crash-tests\\trial*" ) ])) f.addStep( ShellCommand( name="get_previous_release", command=[ "dojob", WithProperties( "perl %(sharedir)s\\mariadb-toolbox\\scripts\\last_release_tag.pl --source-tree=%(builddir)s --dest-tree=%(bb_workdir)s/build-last-release" ) ], timeout=3600, doStepIf=do_release_steps)) f.addStep( Compile( doStepIf=do_release_steps, name="build_previous_release", command=[ "dojob", WithProperties( "cd /d %(bb_workdir)s\\build-last-release && cmake . -G %(vs_generator)s && cmake --build . --config RelWithDebInfo" ) ], timeout=3600, warningPattern=vsWarningPattern, warningExtractor=Compile.warnExtractFromRegexpGroups)) f.addStep( Test( doStepIf=do_release_steps, name="comparison", timeout=3600, env={"MTR_BUILD_THREAD": mtr_build_thread}, command=[ "dojob", WithProperties( "cd /d %(sharedir)s\\rqg && perl combinations.pl --config=%(sharedir)s\\mariadb-toolbox\\configs\\buildbot-comparison.cc --run-all-combinations-once --force --basedir1=%(builddir)s --basedir2=%(bb_workdir)s\\build-last-release --workdir=%(logdir)s\\optim-comparison || perl %(sharedir)s\\mariadb-toolbox\\scripts\\result_summary.pl %(logdir)s\\optim-comparison\\trial*" ) ])) # f.addStep(Test( # name = "app_verifier", # doStepIf=False, # command=["dojob", WithProperties("dir %(logdir)s && appverif -export log -for mysqld.exe -with to=%(logdir)s\\appverif.xml && cat %(logdir)s\\appverif.xml")] # )); return f
def _addSteps4SystemCompiler( f, stage_idx = 0, clean = True, jobs = None, extra_configure_args = None, env = None): # Index is zero-based, so we want to use a human friendly number instead. stage_num = stage_idx + 1 # Directories to use on this stage. obj_dir = f.stage_objdirs[stage_idx] src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir) # This stage could use incremental build. # Clean stage1, only if requested. f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=clean )) f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx], dir=f.stage_installdirs[stage_idx], haltOnFailure=False, flunkOnFailure=False, doStepIf=clean )) # Reconcile the cmake options for this stage. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), # Do not expect warning free build by the system toolchain. ('-DLLVM_ENABLE_WERROR=', 'OFF'), ('-DCMAKE_COLOR_MAKEFILE=', 'OFF'), ]) # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Unix Makefiles'), ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) # Note: On this stage we do not care of warnings, as we build with # a system toolchain and cannot control the environment. # Warnings are likely, and we ignore them. # Create configuration files with cmake f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num, description=["stage%s cmake configure" % stage_num], haltOnFailure=True, flunkOnWarnings=False, options=cmake_args, path=src_dir, env=env, workdir=obj_dir, doStepIf=FileDoesNotExist("CMakeCache.txt"))) # Build clang by the system compiler f.addStep(ShellCommand(name="build-stage%s-compiler"% stage_num, command=['make', WithProperties("-j%s" % jobs), "-k"], haltOnFailure=True, flunkOnWarnings=False, description=["build stage%s compiler" % stage_num], env=env, workdir=obj_dir)) # Test stage1 compiler f.addStep(ShellCommand(name="test-stage%s-compiler"% stage_num, command=["make", WithProperties("-j%s" % jobs), "check-all"], # or "check-llvm", "check-clang" haltOnFailure=True, flunkOnWarnings=False, description=["test stage%s compiler" % stage_num], env=env, workdir=obj_dir)) # Install stage1 compiler f.addStep(ShellCommand(name="install-stage%s-compiler"% stage_num, command=["make", WithProperties("-j%s" % jobs), "install"], haltOnFailure=True, flunkOnWarnings=False, description=["install stage%s compiler" % stage_num], env=env, workdir=obj_dir))
def getClangWithLTOBuildFactory( depends_on_projects = None, clean = False, jobs = None, extra_configure_args = None, compare_last_2_stages = True, lto = None, # The string gets passed to -flto flag as is. Like -flto=thin. env = None): # Set defaults if depends_on_projects: depends_on_projects = list(depends_on_projects) else: # By default we link with LLD. depends_on_projects = ['llvm', 'clang', 'lld'] if lto is None: lto = 'ON' if jobs is None: jobs = "%(jobs)s" if extra_configure_args is None: extra_configure_args = [] else: extra_configure_args = list(extra_configure_args) # Make sure CMAKE_INSTALL_PREFIX and -G are not specified # in the extra_configure_args. We set them internally as needed. # TODO: assert extra_configure_args. install_prefix_specified = any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) assert True, "Please do not explicitly specify the install prefix for multi-stage build." # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir="llvm.src", stage_objdirs=[ "build/stage1", "build/stage2", "build/stage3", "build/stage4", ], stage_installdirs=[ "install/stage1", "install/stage2", "install/stage3", "install/stage4", ], staged_compiler_idx = 1) cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean # Do a clean checkout if requested. f.addStep(RemoveDirectory(name='clean-src-dir', dir=f.llvm_srcdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Get the source code. f.addSVNSteps() # Build with the system compiler first _addSteps4SystemCompiler(f, stage_idx=0, clean=cleanBuildRequested, jobs=jobs, extra_configure_args=extra_configure_args, env=merged_env) # Then build the compiler we would use for the bootstrap. _addSteps4StagedCompiler(f, stage_idx=1, withLTOSupport=True, jobs=jobs, extra_configure_args=extra_configure_args, env=merged_env) # Build all the remaining stages with exactly the same configuration. CmakeCommand.applyRequiredOptions(extra_configure_args, [ ('-DLLVM_ENABLE_LTO=', lto), ]) # If we build LLD, we would link with LLD. # Otherwise we link with the system linker. if 'lld' in depends_on_projects: CmakeCommand.applyRequiredOptions(extra_configure_args, [ ('-DLLVM_ENABLE_LLD=', 'ON'), ]) # The rest are test stages, which depend on the staged compiler we are ultimately after. s = f.staged_compiler_idx + 1 staged_install = f.stage_installdirs[f.staged_compiler_idx] for i in range(s, len(f.stage_objdirs[s:]) + s): configure_args = extra_configure_args[:] configure_args.append( WithProperties( "-DCMAKE_AR=%(workdir)s/" + staged_install + "/bin/llvm-ar" )) configure_args.append( WithProperties( "-DCMAKE_RANLIB=%(workdir)s/" + staged_install + "/bin/llvm-ranlib" )) _addSteps4StagedCompiler(f, stage_idx=i, use_stage_idx=f.staged_compiler_idx, jobs=jobs, extra_configure_args=configure_args, env=merged_env) if compare_last_2_stages: # Compare the compilers built on the last 2 stages if requested. diff_command = [ "diff", "-q", f.stage_installdirs[-2] + "/bin/clang", f.stage_installdirs[-1] + "/bin/clang", ] f.addStep( ShellCommand( name="compare-compilers", description=[ "compare", "stage%d" % (len(f.stage_installdirs)-1), "and", "stage%d" % len(f.stage_installdirs), "compilers", ], haltOnFailure=True, command=WithProperties(" ".join(diff_command)), workdir=".", env=merged_env ) ) return f
def _addSteps4StagedCompiler( f, stage_idx = 1, use_stage_idx = -1, withLTOSupport = False, jobs = None, extra_configure_args = None, env = None): if use_stage_idx < 0: use_stage_idx = stage_idx - 1 # Index is zero-based, so we want to use a human friendly number instead. stage_num = stage_idx + 1 # Directories to use on this stage. obj_dir = f.stage_objdirs[stage_idx] src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir) staged_install = f.stage_installdirs[use_stage_idx] # Always do a clean build for the staged compiler. f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, )) f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx], dir=f.stage_installdirs[stage_idx], haltOnFailure=False, flunkOnFailure=False, )) # Reconcile the cmake options for this stage. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ]) if withLTOSupport: CmakeCommand.applyDefaultOptions(cmake_args, [ # LTO Plugin dependency: ('-DLLVM_BINUTILS_INCDIR=', '/opt/binutils/include'), ]) # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) cmake_args.append( WithProperties( "-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++" )) cmake_args.append( WithProperties( "-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang" )) # Create configuration files with cmake f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num, description=["stage%s cmake configure" % stage_num], haltOnFailure=True, options=cmake_args, path=src_dir, env=env, workdir=obj_dir, doStepIf=FileDoesNotExist("CMakeCache.txt") )) if withLTOSupport: # Build LTO plugin if requested. f.addStep(NinjaCommand(name="build-stage%s-LLVMgold.so" % stage_num, targets=['lib/LLVMgold.so'], haltOnFailure=True, description=["stage%s build LLVMgold.so" % stage_num], env=env, workdir=obj_dir, )) # Build clang by the staged compiler f.addStep(NinjaCommand(name="build-stage%s-compiler" % stage_num, haltOnFailure=True, description=["build stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, )) # Test just built compiler f.addStep(NinjaCommand(name="test-stage%s-compiler"% stage_num, targets=["check-all"], haltOnFailure=True, description=["test stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, )) # Install just built compiler f.addStep(NinjaCommand(name="install-stage%s-compiler"% stage_num, targets=["install"], haltOnFailure=True, description=["install stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, ))
def getLLDWinBuildFactory( clean = True, # Default values for VS devenv and build configuration vs = None, # What to run to configure Visual Studio utils. target_arch = None, # Native. extra_configure_args = None, env = None): # Set defaults if vs is None: vs = r"""%VS140COMNTOOLS%""" # Visual Studio 2015. if extra_configure_args is None: extra_configure_args = [] if env is None: env = {} f = LLVMBuildFactory( depends_on_projects=['llvm', 'lld'], llvm_srcdir="llvm.src", llvm_objdir="llvm.obj") # Get LLVM and Lld f.addSVNSteps() # Clean directory, if requested. cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir, dir=f.llvm_objdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested )) # If set up environment step is requested, do this now. if vs: f.addStep(SetProperty( command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) assert not env, "Can't have custom builder env vars with VS" env = Property('slave_env') # Always build with ninja. cmake_options = ["-G", "Ninja"] # Reconsile configure args with the defaults we want. if not any(a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args): cmake_options.append('-DCMAKE_BUILD_TYPE=Release') if not any(a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args): cmake_options.append('-DLLVM_ENABLE_WERROR=ON') if not any(a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args): cmake_options.append('-DLLVM_ENABLE_ASSERTIONS=ON') if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args): cmake_options.append('-DLLVM_LIT_ARGS=\"-v\"') cmake_options += extra_configure_args # Note: ShellCommand does not pass the params with special symbols right. # The " ".join is a workaround for this bug. f.addStep(CmakeCommand(name="cmake-configure", description=["cmake configure"], haltOnFailure=True, warnOnWarnings=True, options=cmake_options, path="../%s" % f.llvm_srcdir, env=env, workdir=f.llvm_objdir, doStepIf=FileDoesNotExist( "./%s/CMakeCache.txt" % f.llvm_objdir))) # Build Lld. f.addStep(NinjaCommand(name='build lld', haltOnFailure=True, warnOnWarnings=True, description='build lld', workdir=f.llvm_objdir, env=env)) # Test Lld f.addStep(NinjaCommand(name='test lld', targets=['lld-test'], haltOnFailure=True, warnOnWarnings=True, description='test lld', workdir=f.llvm_objdir, env=env)) return f
def ros_debbuild(c, job_name, packages, url, distro, arch, rosdistro, version, machines, othermirror, keys, trigger_pkgs=None): gbp_args = [ '-uc', '-us', '--git-ignore-branch', '--git-ignore-new', '--git-verbose', '--git-dist=' + distro, '--git-arch=' + arch ] f = BuildFactory() # Remove the build directory. f.addStep( RemoveDirectory( name=job_name + '-clean', dir=Interpolate('%(prop:workdir)s'), hideStepIf=success, )) # Check out the repository master branch, since releases are tagged and not branched f.addStep( Git( repourl=url, branch='master', alwaysUseLatest= True, # this avoids broken builds when schedulers send wrong tag/rev mode='full' # clean out old versions )) # Update the cowbuilder f.addStep( ShellCommand(command=['cowbuilder-update.py', distro, arch] + keys, hideStepIf=success)) # Need to build each package in order for package in packages: debian_pkg = 'ros-' + rosdistro + '-' + package.replace( '_', '-') # debian package name (ros-groovy-foo) branch_name = 'debian/' + debian_pkg + '_%(prop:release_version)s_' + distro # release branch from bloom deb_name = debian_pkg + '_%(prop:release_version)s' + distro final_name = debian_pkg + '_%(prop:release_version)s-%(prop:datestamp)s' + distro + '_' + arch + '.deb' # Check out the proper tag. Use --force to delete changes from previous deb stamping f.addStep( ShellCommand(haltOnFailure=True, name=package + '-checkout', command=[ 'git', 'checkout', Interpolate(branch_name), '--force' ], hideStepIf=success)) # Build the source deb f.addStep( ShellCommand(haltOnFailure=True, name=package + '-buildsource', command=['git-buildpackage', '-S'] + gbp_args, descriptionDone=['sourcedeb', package])) # Upload sourcedeb to master (currently we are not actually syncing these with a public repo) f.addStep( FileUpload( name=package + '-uploadsource', slavesrc=Interpolate('%(prop:workdir)s/' + deb_name + '.dsc'), masterdest=Interpolate('sourcedebs/' + deb_name + '.dsc'), hideStepIf=success)) # Stamp the changelog, in a similar fashion to the ROS buildfarm f.addStep( SetPropertyFromCommand(command="date +%Y%m%d-%H%M-%z", property="datestamp", name=package + '-getstamp', hideStepIf=success)) f.addStep( ShellCommand( haltOnFailure=True, name=package + '-stampdeb', command=[ 'git-dch', '-a', '--ignore-branch', '--verbose', '-N', Interpolate('%(prop:release_version)s-%(prop:datestamp)s' + distro) ], descriptionDone=[ 'stamped changelog', Interpolate('%(prop:release_version)s'), Interpolate('%(prop:datestamp)s') ])) # download hooks f.addStep( FileDownload( name=package + '-grab-hooks', mastersrc='hooks/D05deps', slavedest=Interpolate('%(prop:workdir)s/hooks/D05deps'), hideStepIf=success, mode=0777 # make this executable for the cowbuilder )) # build the binary from the git working copy f.addStep( ShellCommand( haltOnFailure=True, name=package + '-buildbinary', command=[ 'git-buildpackage', '--git-pbuilder', '--git-export=WC', Interpolate('--git-export-dir=%(prop:workdir)s') ] + gbp_args, env={ 'DIST': distro, 'GIT_PBUILDER_OPTIONS': Interpolate( '--hookdir %(prop:workdir)s/hooks --override-config'), 'OTHERMIRROR': othermirror }, descriptionDone=['binarydeb', package])) # Upload binarydeb to master f.addStep( FileUpload(name=package + '-uploadbinary', slavesrc=Interpolate('%(prop:workdir)s/' + final_name), masterdest=Interpolate('binarydebs/' + final_name), hideStepIf=success)) # Add the binarydeb using reprepro updater script on master f.addStep( MasterShellCommand(name=package + 'includedeb', command=[ 'reprepro-include.bash', debian_pkg, Interpolate(final_name), distro, arch ], descriptionDone=['updated in apt', package])) # Trigger if needed if trigger_pkgs != None: f.addStep( Trigger(schedulerNames=[ t.replace('_', '-') + '-' + rosdistro + '-' + distro + '-' + arch + '-debtrigger' for t in trigger_pkgs ], waitForFinish=False, alwaysRun=True)) # Create trigger c['schedulers'].append( triggerable.Triggerable( name=job_name.replace('_', '-') + '-' + rosdistro + '-' + distro + '-' + arch + '-debtrigger', builderNames=[ job_name + '_' + rosdistro + '_' + distro + '_' + arch + '_debbuild', ])) # Add to builders c['builders'].append( BuilderConfig(name=job_name + '_' + rosdistro + '_' + distro + '_' + arch + '_debbuild', properties={'release_version': version}, slavenames=machines, factory=f)) # return name of builder created return job_name + '_' + rosdistro + '_' + distro + '_' + arch + '_debbuild'
def getToolchainBuildFactory( clean=False, test=True, env=None, # Environmental variables for all steps. extra_configure_args=None): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { "TERM" : "dumb", # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) src_dir = "llvm.src" obj_dir = "llvm.obj" install_dir = "llvm.install" f = LLVMBuildFactory( depends_on_projects=[ "llvm", "clang", "clang-tools-extra", "compiler-rt", "libcxx", "libcxxabi", "libunwind", "lld" ]) # Get Fuchsia SDK. sdk_dir = "fuchsia.sdk" sdk_platform = { "Linux": "linux-amd64", "Darwin": "mac-amd64", }[platform.system()] sdk_version = "latest" sdk_url = WithProperties( "https://chrome-infra-packages.appspot.com/dl/" "fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s", sdk_platform=lambda _: sdk_platform, sdk_version=lambda _: sdk_version) f.addStep(RemoveDirectory(name="clean-sdk", dir=sdk_dir, haltOnFailure=True)) f.addStep(ShellCommand(name="fetch-sdk", command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url], description=["download", "fuchsia sdk"], workdir=sdk_dir)) f.addStep(ShellCommand(name="extract-sdk", command=["unzip", "-fo", "sdk.cipd"], description=["extract", "fuchsia sdk"], workdir=sdk_dir)) cleanCheckoutRequested = lambda step: step.build.getProperty("clean", default=False) or clean # Clean up llvm sources. f.addStep(RemoveDirectory(name="clean-llvm.src", dir=src_dir, haltOnFailure=True, doStepIf=cleanCheckoutRequested)) # Get sources. for project in f.depends_on_projects: _, baseURL = svn_repos[project] f.addStep(SVN(name="svn-%s" % project, workdir=src_dir + "/" + project, baseURL=baseURL)) cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean # Clean up llvm build. f.addStep(RemoveDirectory(name="clean-llvm.obj", dir=obj_dir, haltOnFailure=True, doStepIf=cleanBuildRequested)) # Configure. if extra_configure_args is None: cmake_options = [] else: cmake_options = extra_configure_args[:] # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_options, [ ("-G", "Ninja"), ("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"), ("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"), ]) # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_options, [ ("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"), ("-DLLVM_ENABLE_LTO=", "OFF"), ]) cmake_options.append( WithProperties( "-DCMAKE_INSTALL_PREFIX=%(workdir)s/" + install_dir )) cmake_options.append( WithProperties( "-DFUCHSIA_SDK=%(workdir)s/" + sdk_dir )) CmakeCommand.applyRequiredOptions(cmake_options, [ ("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"), ]) f.addStep(CmakeCommand(name="cmake-configure", options=cmake_options, path='../' + src_dir + '/llvm', haltOnFailure=True, description=["configure"], workdir=obj_dir, env=merged_env, doStepIf=FileDoesNotExist("CMakeCache.txt"))) # Build distribution. f.addStep(NinjaCommand(name="ninja-build", targets=["stage2-distribution"], haltOnFailure=True, description=["build"], workdir=obj_dir, env=merged_env)) # Test llvm, clang and lld. f.addStep(NinjaCommand(name="check", targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")], haltOnFailure=True, description=["check"], workdir=obj_dir, env=merged_env, doStepIf=test)) # Install distribution. f.addStep(NinjaCommand(name="install", targets=["stage2-install-distribution"], haltOnFailure=True, description=["install"], workdir=obj_dir, env=merged_env)) return f
def getCUDATestsuiteBuildFactory( externals, # Directory with CUDA, thrust and gcc versions for testing. always_clean=True, test=False, useTwoStage=False, cmake='cmake', extra_cmake_args=None, # Extra CMake args for all stages. extra_ts_cmake_args=None, # extra cmake args for testsuite. jobs=None, cuda_jobs=1, # number of simultaneous CUDA apps to run env=None, # Environmental variables for all steps. enable_thrust_tests=False, split_thrust_tests=False, # Each thrust test is a separate executable. run_thrust_tests=False, enable_libcxx=True, # checkout/build libcxx to test with. gpu_arch_list=None, gpu_devices=None, # List of devices to make visible to CUDA stage1_config='Release', stage2_config='Release'): if extra_cmake_args is None: extra_cmake_args = [] if extra_ts_cmake_args is None: extra_ts_cmake_args = [] # Prepare environmental variables. Set here all env we want for all steps. merged_env = { 'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set # anything. merged_env.update(env) source_dir = 'llvm' # Should match the one used in getClangCMakeBuildFactory. stage1_build_dir = 'stage1' # Should match the one defined in getClangCMakeBuildFactory. stage2_build_dir = 'stage2' # Should match the one defined in getClangCMakeBuildFactory. install_dir = 'clang.install' if useTwoStage: clang_build_dir = stage2_build_dir else: clang_build_dir = stage1_build_dir # Build clang. f = ClangBuilder.getClangCMakeBuildFactory( clean=always_clean, test=test, cmake=cmake, extra_cmake_args=extra_cmake_args, jobs=jobs, env=merged_env, useTwoStage=useTwoStage, stage1_config=stage1_config, stage2_config=stage2_config, checkout_clang_tools_extra=False, checkout_compiler_rt=False, checkout_libcxx=enable_libcxx, checkout_lld=False, checkout_test_suite=True) cuda_test_env = { 'PYTHONPATH': WithProperties("%(workdir)s/" + source_dir + "/utils/lit:${PYTHONPATH}"), 'DESTDIR': WithProperties("%(workdir)s/" + install_dir), 'PATH': WithProperties("%(workdir)s/" + install_dir + "/usr/local/bin:${PATH}"), } merged_env.update(cuda_test_env) ts_build_dir = 'test-suite-build' f.addStep( RemoveDirectory(name="Remove old clang install directory", dir=install_dir)) # Install clang into directory pointed by $DESTDIR f.addStep( NinjaCommand(name='ninja install clang', targets=["install"], jobs=jobs, haltOnFailure=True, description=split("installing clang"), descriptionDone=split("Clang installation is done."), workdir=clang_build_dir, env=merged_env)) # Completely remove test suite build dir. f.addStep( RemoveDirectory(name="Remove old test-suite build directory", dir=ts_build_dir)) if extra_ts_cmake_args: cmake_args = extra_ts_cmake_args[:] else: cmake_args = [] # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ]) # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ]) cmake_args.append( WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" + clang_build_dir + "/bin/clang++")) cmake_args.append( WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + clang_build_dir + "/bin/clang")) cmake_args.append('-DTEST_SUITE_SUBDIRS=External'), # Limit to External tests only. if externals: cmake_args.append('-DTEST_SUITE_EXTERNALS_DIR=' + externals) if split_thrust_tests: cmake_args.append('-DTHRUST_SPLIT_TESTS=1') if gpu_arch_list: cmake_args.append('-DCUDA_GPU_ARCH=' + ';'.join(gpu_arch_list)) if cuda_jobs: cmake_args.append('-DCUDA_JOBS=%s' % cuda_jobs) # Then do fresh cmake configuration. f.addStep( CmakeCommand(name='cmake test-suite', description='cmake test-suite', haltOnFailure=True, options=cmake_args, path="../test/test-suite", workdir=ts_build_dir, env=merged_env)) # Always build simple CUDA tests. They serve as compilation # smoketests and will fail quickly if compiler has obvious issues # compiling CUDA files. f.addStep( NinjaCommand(name='ninja build simple CUDA tests', targets=["cuda-tests-simple"], jobs=jobs, haltOnFailure=True, description=split("building simple CUDA tests"), descriptionDone=split("simple CUDA tests built."), workdir=ts_build_dir, env=merged_env)) # Limit GPUs visible to CUDA. if gpu_devices: for gpu_id in gpu_devices: # make ID a string as it may be either an integer or a UUID string. gpu_id = str(gpu_id) gpu_env = dict(merged_env) gpu_env["CUDA_VISIBLE_DEVICES"] = gpu_id f.addStep( NinjaCommand( name='run simple CUDA tests on gpu %s' % gpu_id, targets=["check-cuda-simple"], jobs=1, # lit will parallelize the jobs haltOnFailure=True, description=split("Running simple CUDA tests on GPU %s" % gpu_id), descriptionDone=split("simple CUDA tests on GPU %s done." % gpu_id), workdir=ts_build_dir, env=gpu_env)) else: f.addStep( NinjaCommand( name='run simple CUDA tests', targets=["check-cuda-simple"], jobs=1, # lit will parallelize the jobs haltOnFailure=True, description=split("Running simple CUDA tests"), descriptionDone=split("simple CUDA tests done."), workdir=ts_build_dir, env=merged_env)) # If we've enabled thrust tests, build them now. # WARNING: This takes a lot of time to build. if (enable_thrust_tests): f.addStep( NinjaCommand(name='ninja build thrust', targets=["cuda-tests-thrust"], jobs=jobs, haltOnFailure=True, description=split("building thrust tests"), descriptionDone=split("thrust tests built."), workdir=ts_build_dir, env=merged_env)) # Run them. That also takes a while. # TODO(tra) we may want to run more than one instance so one # can be compiling tests while another is running them on GPU. if run_thrust_tests: f.addStep( NinjaCommand( name='run all CUDA tests', targets=["check"], jobs=1, # lit will parallelize the jobs. haltOnFailure=True, description=split("running all CUDA tests."), descriptionDone=split("all cuda tests done."), workdir=ts_build_dir, env=merged_env)) return f
def ros_branch_build(c, job_name, packages, url, branch, distro, arch, rosdistro, machines, othermirror, keys): gbp_args = ['-uc', '-us', '--git-ignore-branch', '--git-ignore-new', '--git-verbose', '--git-dist='+distro, '--git-arch='+arch] f = BuildFactory() # Remove the build directory. f.addStep( RemoveDirectory( name = job_name+'-clean', dir = Interpolate('%(prop:workdir)s'), hideStepIf = success, ) ) # Pulling the repo f.addStep( Git( repourl = url, branch = 'HEAD', alwaysUseLatest = True, # this avoids broken builds when schedulers send wrong tag/rev mode = 'full', # clean out old versions getDescription={'tags': True} ) ) # Check out the repository branch/commit/tag f.addStep( ShellCommand( haltOnFailure = True, name = 'checkout: ' + branch, command = ['git', 'checkout', branch], ) ) # get the short commit hash f.addStep( SetPropertyFromCommand( command="git rev-parse --short HEAD", property="commit_hash", name = 'commit-short-hash', hideStepIf = success ) ) # get the time stamp f.addStep( SetPropertyFromCommand( command='date +%Y%M%d%H', property='date_stamp', name = 'date-stamp', hideStepIf = success ) ) # Update the cowbuilder f.addStep( ShellCommand( command = ['cowbuilder-update.py', distro, arch] + keys, hideStepIf = success ) ) # Generate the changelog for the package f.addStep( ShellCommand( haltOnFailure = True, name = 'catkin_generate_changelog', command= ['catkin_generate_changelog', '-y'], descriptionDone = ['catkin_generate_changelog'] ) ) # Add all files including untracked ones f.addStep( ShellCommand( haltOnFailure = True, name = 'add_changelogs', command= ['git', 'add', '.'], descriptionDone = ['add_changelogs'] ) ) # Commit the changelog after updating it f.addStep( ShellCommand( haltOnFailure = True, name = 'update_changelogs', command= ['git', 'commit', '-m', '\"Updated changelogs\"'], descriptionDone = ['update_changelogs'] ) ) # Prepare the release without pushing it # Set very big number to avoid conflicts with available tags f.addStep( ShellCommand( haltOnFailure = True, name = 'catkin_prepare_release', command= ['catkin_prepare_release', '--version', '100.0.0', '--no-push', '-y'], descriptionDone = ['catkin_prepare_release'] ) ) # f.addStep( ShellCommand( haltOnFailure = True, name = 'git_bloom_generate_release', command = ['git-bloom-generate', '-y', 'rosrelease', rosdistro], ) ) f.addStep( ShellCommand( haltOnFailure = True, name = 'git_bloom_generate_debian', command = ['git-bloom-generate', '-y', 'rosdebian', '-a', '-p', 'release', rosdistro], ) ) # Get the tag number for the lastest commit f.addStep( SetPropertyFromCommand( command="git describe --tags", property="release_version", name = 'latest_tag', ) ) # Need to build each package in order for package in packages: debian_pkg = 'ros-'+rosdistro+'-'+package.replace('_','-') # debian package name (ros-groovy-foo) branch_name = 'debian/'+debian_pkg+'_%(prop:release_version)s-0_'+distro deb_name = debian_pkg+'_%(prop:release_version)s-0'+distro final_name = debian_pkg+'_%(prop:release_version)s-%(prop:date_stamp)s-%(prop:commit_hash)s-'+distro+'_'+arch+'.deb' # Check out the proper tag. Use --force to delete changes from previous deb stamping f.addStep( ShellCommand( haltOnFailure = True, name = package+'-checkout', command = ['git', 'checkout', Interpolate(branch_name), '--force'], hideStepIf = success ) ) # Stamp the changelog f.addStep( ShellCommand( haltOnFailure = True, name = package+'-stampdeb', command = ['gbp', 'dch', '-a', '--ignore-branch', '--verbose', '-N', Interpolate('%(prop:release_version)s-%(prop:date_stamp)s-%(prop:commit_hash)s-'+distro)], descriptionDone = ['stamped changelog', Interpolate('%(prop:release_version)s'), Interpolate('%(prop:date_stamp)s'), Interpolate('%(prop:commit_hash)s')] ) ) # download hooks f.addStep( FileDownload( name = package+'-grab-hooks', mastersrc = 'hooks/D05deps', slavedest = Interpolate('%(prop:workdir)s/hooks/D05deps'), hideStepIf = success, mode = 0777 # make this executable for the cowbuilder ) ) # Download script for building the binary deb f.addStep( FileDownload( name = job_name+'-grab-build-binary-deb-script', mastersrc = 'scripts/build_binary_deb.py', slavedest = Interpolate('%(prop:workdir)s/build_binary_deb.py'), mode = 0755, hideStepIf = success ) )
def getLLDBWindowsCMakeBuildFactory( clean=False, cmake='cmake', jobs="%(jobs)s", # Source directory containing a built python python_source_dir=r'C:/src/python', # Default values for VS devenv and build configuration vs=r"""%VS120COMNTOOLS%""", config='Release', target_arch='x86', extra_cmake_args=[], test=False, install=False): ############# PREPARING f = buildbot.process.factory.BuildFactory() # Determine Slave Environment and Set MSVC environment. f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f = getLLDBSource(f, 'llvm') build_cmd = ['ninja'] install_cmd = ['ninja', 'install'] test_cmd = ['ninja', 'check-lldb'] if jobs: build_cmd.append(WithProperties("-j%s" % jobs)) install_cmd.append(WithProperties("-j%s" % jobs)) test_cmd.append(WithProperties("-j%s" % jobs)) # Global configurations build_dir = 'build' ############# CLEANING cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) if config.lower() == 'release': python_lib = 'python27.lib' python_exe = 'python.exe' elif config.lower() == 'debug': python_lib = 'python27_d.lib' python_exe = 'python_d.exe' python_lib = os.path.join(python_source_dir, 'PCbuild', python_lib) python_exe = os.path.join(python_source_dir, 'PCbuild', python_exe) python_include = os.path.join(python_source_dir, 'Include') # Use batch files instead of ShellCommand directly, Windows quoting is # borked. FIXME: See buildbot ticket #595 and buildbot ticket #377. f.addStep( batch_file_download.BatchFileDownload( name='cmakegen', command=[ cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config, # Need to use our custom built version of python '-DPYTHON_LIBRARY=' + python_lib, '-DPYTHON_INCLUDE_DIR=' + python_include, '-DPYTHON_EXECUTABLE=' + python_exe, "-DCMAKE_INSTALL_PREFIX=../install" ] + extra_cmake_args, workdir=build_dir)) f.addStep( ShellCommand(name='cmake', command=['cmakegen.bat'], haltOnFailure=True, description='cmake gen', workdir=build_dir, env=Property('slave_env'))) f.addStep( WarningCountingShellCommand(name='build', command=build_cmd, haltOnFailure=True, description='ninja build', workdir=build_dir, env=Property('slave_env'))) ignoreInstallFail = bool(install != 'ignoreFail') f.addStep( ShellCommand(name='install', command=install_cmd, flunkOnFailure=ignoreInstallFail, description='ninja install', workdir=build_dir, doStepIf=bool(install), env=Property('slave_env'))) ignoreTestFail = bool(test != 'ignoreFail') f.addStep( ShellCommand(name='test', command=test_cmd, flunkOnFailure=ignoreTestFail, description='ninja test', workdir=build_dir, doStepIf=bool(test), env=Property('slave_env'))) return f
# Upload the pdb zip file. FileUpload(slavesrc=pdb_filename, masterdest=pdb_upload_filename, mode=0o664, haltOnFailure=False), ] + publish_exe_steps # Now make the factories. sdk_factory = BuildFactory() for step in checkout_steps + whl_steps + build_steps + publish_sdk_steps: sdk_factory.addStep(step) runtime_factory = BuildFactory() for step in checkout_steps + build_steps + publish_exe_steps: runtime_factory.addStep(step) rtdist_factory = BuildFactory() rtdist_factory.addStep(RemoveDirectory(dir="built/slave")) for step in checkout_steps + build_steps + publish_rtdist_steps: rtdist_factory.addStep(step) def windows_builder(buildtype, arch): if buildtype == "rtdist": factory = rtdist_factory elif buildtype == "runtime": factory = runtime_factory else: factory = sdk_factory if arch == "amd64": platform = "win_amd64" else:
def mkbuildsfactory(): f = factory.BuildFactory() f.addSteps([ gitStep, VirtualenvSetup(name='virtualenv setup', no_site_packages=True, virtualenv_packages=[ # required to build www packages (#2877) '--editable=master', '--editable=pkg', 'mock', # required to make wheels 'wheel', # ..and this is required to actually build them correctly (#2883) 'setuptools' ], virtualenv_dir='../sandbox', haltOnFailure=True), RemoveDirectory(dir='build/uploads'), MakeDirectory(dir='build/uploads'), ]) for name, workdir, command in [ ('buildbot', 'build/master', 'sdist'), ('buildbot-slave', 'build/slave', 'sdist'), ('buildbot-pkg', 'build/pkg', 'sdist'), ('buildbot-www', 'build/www/base', 'bdist_wheel'), ('buildbot-console-view', 'build/www/console_view', 'bdist_wheel'), ('buildbot-waterfall-view', 'build/www/waterfall_view', 'bdist_wheel'), ('buildbot-codeparameter', 'build/www/codeparameter', 'bdist_wheel'), ]: levels = workdir.count('/') + 1 sandbox = "../" * levels + "sandbox" extension = {'sdist': '.tar.gz', 'bdist_wheel': '-py2-none-any.whl'}[command] pkgname = name if command == 'bdist_wheel': pkgname = pkgname.replace('-', '_') # wheels always use _ f.addSteps([ # the 'buildbot-www' package requires that the sandbox be *active* so that it # can find 'buildbot' ShellCommand(command="rm -rf dist/*; . %s/bin/activate; python setup.py %s" % (sandbox, command), workdir=workdir, name=name, flunkOnFailure=True, haltOnFailure=False, env={'BUILDBOT_VERSION': '1latest'}), # wheels require a digit ShellCommand(command=""" mv %(workdir)s/dist/%(pkgname)s-1latest%(extension)s build/uploads """ % dict(pkgname=pkgname, extension=extension, workdir=workdir), name=name + " mv", flunkOnFailure=True, haltOnFailure=False, workdir='.'), ]) # now upload all of those. SFTP is annoying to script. script = textwrap.dedent("""\ # get the version from git, since 'buildbot.version' only has the x.y.z prefix VERSION=$(git describe --tags --always) version=$(mktemp) echo $VERSION >${version} readme=$(mktemp) cat <<EOF >${readme} This directory contains the latest versions of Buildbot, packaged as they would be for a release. These are most useful for developers wishing to use a working web UI without installing Node: pip intall http:s//ftp.buildbot.net/latest/buildbot-www-1latest-py2-none-any.whl The packages here are for ${VERSION}. EOF batchfile=$(mktemp) cat <<EOF >${batchfile} cd pub/latest put uploads/* put ${readme} README.txt chmod 644 README.txt put ${version} VERSION.txt chmod 644 VERSION.txt EOF sftp \ -b ${batchfile} \ -oPort=2200 \ -oIdentityFile=~/.ssh/ftp.key \ [email protected] rv=$? rm ${batchfile} ${readme} ${version} exit $rv""") f.addStep(ShellCommand(command=script, flunkOnFailure=True)) return f
def addCmakeSteps(f, cleanBuildRequested, obj_dir, install_dir=None, extra_configure_args=None, env=None, stage_name=None, **kwargs): # Consume is_legacy_mode if given. # TODO: Remove this once legacy mode gets dropped. kwargs.pop('is_legacy_mode', None) # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() if obj_dir is None: obj_dir = f.obj_dir # This is an incremental build, unless otherwise has been requested. # Remove obj and install dirs for a clean build. # TODO: Some Windows slaves do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep( RemoveDirectory( name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) if not f.is_legacy_mode: CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeToBuild( install_dir, obj_dir) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel), ]) f.addStep( RemoveDirectory( name='clean-%s-dir' % install_dir, dir=install_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Reconcile the cmake options for this build. # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_LIT_ARGS=', '-v -vv'), ]) # Create configuration files with cmake, unless this has been already done # for an incremental build. if stage_name: step_name = "cmake-configure-%s" % stage_name else: stage_name = "" step_name = "cmake-configure" src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) f.addStep( CmakeCommand( name=step_name, haltOnFailure=True, description=["Cmake", "configure", stage_name], options=cmake_args, path=src_dir, env=env, workdir=obj_dir, **kwargs # Pass through all the extra arguments. ))
def getLLDBuildFactory( clean = True, jobs = None, extra_configure_args = None, env = None): # Set defaults if jobs is None: jobs = "%(jobs)s" if extra_configure_args is None: extra_configure_args = [] # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'CC' : "clang", 'CXX' : "clang++", 'TERM' : 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) f = LLVMBuildFactory( depends_on_projects=['llvm', 'lld'], llvm_srcdir="llvm.src", llvm_objdir="llvm.obj") # Get LLVM and Lld f.addSVNSteps() # Clean directory, if requested. cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir, dir=f.llvm_objdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested )) # Create configuration files with cmake f.addStep(CmakeCommand(name="cmake-configure", description=["cmake configure"], haltOnFailure=True, options=extra_configure_args, path="../%s" % f.llvm_srcdir, env=merged_env, workdir=f.llvm_objdir, doStepIf=FileDoesNotExist( "./%s/CMakeCache.txt" % f.llvm_objdir))) # Build Lld f.addStep(ShellCommand(name="build_Lld", command=['nice', '-n', '10', 'make', WithProperties("-j%s" % jobs)], haltOnFailure=True, description=["build lld"], env=merged_env, workdir=f.llvm_objdir)) # Test Lld f.addStep(ShellCommand(name="test_lld", command=["make", "lld-test"], haltOnFailure=True, description=["test lld"], env=merged_env, workdir=f.llvm_objdir)) return f
def getPollyBuildFactory( clean=False, install=False, make='make', jobs=None, checkAll=False, env=None, extraCmakeArgs=None, testsuite=False,extraTestsuiteCmakeArgs=None, **kwargs): if extraCmakeArgs is None: extraCmakeArgs=[] if extraTestsuiteCmakeArgs is None: extraTestsuiteCmakeArgs = [] llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" llvm_instdir = "llvm.inst" testsuite_srcdir = "test-suite.src" testsuite_builddir = "test-suite.build" jobs_cmd = [] if jobs is not None: jobs_cmd = ["-j"+str(jobs)] build_cmd = [make] + jobs_cmd install_cmd = [make, 'install'] + jobs_cmd check_all_cmd = [make, 'check-all'] + jobs_cmd check_polly_cmd = [make, 'check-polly'] + jobs_cmd cmake_install = [] if install: cmake_install = ["-DCMAKE_INSTALL_PREFIX=../%s" % llvm_instdir] # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env: merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything. depends_on_projects = ['llvm','clang','polly'] if testsuite: # XRay tests in test-suite require compiler-rt depends_on_projects += ['compiler-rt'] cleanBuildRequestedByProperty = lambda step: step.build.getProperty("clean", False) cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=llvm_objdir, install_dir=llvm_instdir, cleanBuildRequested=cleanBuildRequested, **kwargs) # Pass through all the extra arguments. f.addStep(RemoveDirectory(name='clean-src-dir', dir=f.monorepo_dir, warnOnFailure=True, doStepIf=cleanBuildRequestedByProperty)) # Get the source code. f.addGetSourcecodeSteps(**kwargs) # Clean build dir f.addStep(RemoveDirectory(name='clean-build-dir', dir=llvm_objdir, warnOnFailure=True, doStepIf=cleanBuildRequested)) # Create configuration files with cmake cmakeCommand = ["cmake", "../%s/llvm" % llvm_srcdir, "-DCMAKE_COLOR_MAKEFILE=OFF", "-DPOLLY_TEST_DISABLE_BAR=ON", "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_POLLY_LINK_INTO_TOOLS=ON", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), ] + cmake_install + extraCmakeArgs f.addStep(ShellCommand(name="cmake-configure", command=cmakeCommand, haltOnFailure=False, description=["cmake configure"], workdir=llvm_objdir, env=merged_env)) # Build f.addStep(WarningCountingShellCommand(name="build", command=build_cmd, haltOnFailure=True, description=["build"], workdir=llvm_objdir, env=merged_env)) clangexe = "%(workdir)s/" + llvm_objdir + "/bin/clang" clangxxexe = "%(workdir)s/" + llvm_objdir + "/bin/clang++" litexe = "%(workdir)s/" + llvm_objdir + "/bin/llvm-lit" sizeexe = "%(workdir)s/" + llvm_objdir + "/bin/llvm-size" # Clean install dir if install: f.addStep(RemoveDirectory(name='clean-install-dir', dir=llvm_instdir, haltOnFailure=False, doStepIf=cleanBuildRequested)) f.addStep(ShellCommand(name="install", command=install_cmd, haltOnFailure=True, description=["install"], workdir=llvm_objdir, env=merged_env)) # If installing, use the installed version of clang. clangexe = "%(workdir)s/" + llvm_instdir + "/bin/clang" clangxxexe = "%(workdir)s/" + llvm_instdir + "/bin/clang++" sizeexe = "%(workdir)s/" + llvm_instdir + "/bin/llvm-size" # Test if checkAll: f.addStep(LitTestCommand(name="check_all", command=check_all_cmd, haltOnFailure=False, description=["check all"], workdir=llvm_objdir, env=merged_env)) else: f.addStep(LitTestCommand(name="check_polly", command=check_polly_cmd, haltOnFailure=False, description=["check polly"], workdir=llvm_objdir, env=merged_env)) if testsuite: f.addStep(RemoveDirectory(name='test-suite_clean-src-dir', dir=testsuite_srcdir, haltOnFailure=False, warnOnFailure=True, doStepIf=cleanBuildRequestedByProperty)) f.addGetSourcecodeForProject( project='test-suite', src_dir=testsuite_srcdir, alwaysUseLatest=True) f.addStep(RemoveDirectory(name='test-suite_clean-build-dir', dir=testsuite_builddir, haltOnFailure=False, warnOnFailure=True)) # -Wno-unused-command-line-argument is needed because linking will not uses the "-mllvm -polly" argument. f.addStep(ShellCommand(name='test-suite_cmake-configure', description=["Test-Suite: cmake"], command=["cmake", '-B', testsuite_builddir, '-S', testsuite_srcdir, "-DCMAKE_BUILD_TYPE=Release", "-DTEST_SUITE_COLLECT_STATS=ON", "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", "-DTEST_SUITE_EXTRA_CXX_FLAGS=-mllvm -polly", "-DTEST_SUITE_LIT_FLAGS=-vv;-o;report.json", WithProperties("-DCMAKE_C_COMPILER=" + clangexe), WithProperties("-DCMAKE_CXX_COMPILER=" + clangxxexe), WithProperties("-DTEST_SUITE_LLVM_SIZE=" + sizeexe), WithProperties("-DTEST_SUITE_LIT=" + litexe), ] + extraTestsuiteCmakeArgs, haltOnFailure=True, workdir='.', env=merged_env)) f.addStep(WarningCountingShellCommand(name='test-suite_build', description=["Test-Suite: build"], # Continue building; programs that don't compile will fail with NOEXE. command=[make, 'all', '-k0'] + jobs_cmd, haltOnFailure=False, flunkOnFailure=True, workdir=testsuite_builddir, env=merged_env)) f.addStep(LitTestCommand(name='test-suite_run', description=['Test-Suite: run'], command=[WithProperties(litexe), '-vv', '-o', 'report.json', '.'], haltOnFailure=True, workdir=testsuite_builddir, logfiles={ 'test.log' : 'test.log', 'report.json': 'report.json' }, env=merged_env)) return f
def getLLDBCMakeBuildFactory( clean=False, cmake='cmake', jobs="%(jobs)s", # Source directory containing a built python python_source_dir=None, # Default values for VS devenv and build configuration vs=None, config='Release', target_arch='x86', extra_cmake_args=None, test=False, install=False): ############# PREPARING f = buildbot.process.factory.BuildFactory() # Determine Slave Environment and Set MSVC environment. if vs: f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f = getLLDBSource(f, 'llvm') build_cmd = ['ninja'] install_cmd = ['ninja', 'install'] test_cmd = ['ninja', 'check-lldb'] if jobs: build_cmd.append(WithProperties("-j%s" % jobs)) install_cmd.append(WithProperties("-j%s" % jobs)) test_cmd.append(WithProperties("-j%s" % jobs)) # Global configurations build_dir = 'build' # get full path to build directory f.addStep( SetProperty(name="get_builddir", command=["pwd"], property="builddir", description="set build dir", workdir=build_dir)) ############# CLEANING cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) cmake_cmd = [ "cmake", "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config, "-DCMAKE_INSTALL_PREFIX=../install" ] if python_source_dir: cmake_cmd.append("-DPYTHON_HOME=" + python_source_dir) if extra_cmake_args: cmake_cmd += extra_cmake_args # Note: ShellCommand does not pass the params with special symbols right. # The " ".join is a workaround for this bug. f.addStep( ShellCommand(name="cmake-configure", description=["cmake configure"], command=WithProperties(" ".join(cmake_cmd)), haltOnFailure=True, warnOnWarnings=True, workdir=build_dir, env=Property('slave_env'))) f.addStep( WarningCountingShellCommand(name='build', command=build_cmd, haltOnFailure=True, description='ninja build', workdir=build_dir, env=Property('slave_env'))) ignoreInstallFail = bool(install != 'ignoreFail') f.addStep( ShellCommand(name='install', command=install_cmd, flunkOnFailure=ignoreInstallFail, description='ninja install', workdir=build_dir, doStepIf=bool(install), env=Property('slave_env'))) ignoreTestFail = bool(test != 'ignoreFail') f.addStep( ShellCommand(name='test', command=test_cmd, flunkOnFailure=ignoreTestFail, timeout=2400, description='ninja test', workdir=build_dir, doStepIf=bool(test), env=Property('slave_env'))) return f
def getCmakeBuildFactory(depends_on_projects=None, llvm_srcdir=None, obj_dir=None, install_dir=None, clean=False, extra_configure_args=None, env=None, **kwargs): # Set defaults if not depends_on_projects: depends_on_projects = ['llvm', 'clang'] if extra_configure_args is None: extra_configure_args = [] # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb' # Be cautious and disable color output from all tools. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) if not obj_dir: obj_dir = "build" if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeToBuild( install_dir, obj_dir) CmakeCommand.applyRequiredOptions(extra_configure_args, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel), ]) cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir or "llvm.src", obj_dir=obj_dir, install_dir=install_dir, cleanBuildRequested=cleanBuildRequested, **kwargs # Pass through all the extra arguments. ) # Directories to use on this stage. src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) # Do a clean checkout if requested. f.addStep( RemoveDirectory( name='clean-src-dir', dir=f.llvm_srcdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Get the source code. f.addSVNSteps() # This is an incremental build, unless otherwise has been requested. # Remove obj and install dirs for a clean build. f.addStep( RemoveDirectory( name='clean-%s-dir' % f.obj_dir, dir=f.obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) if f.install_dir: f.addStep( RemoveDirectory( name='clean-%s-dir' % f.install_dir, dir=f.install_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Reconcile the cmake options for this build. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ]) # Create configuration files with cmake, unless this has been already done # for an incremental build. f.addStep( CmakeCommand( name="cmake-configure", description=["cmake configure"], haltOnFailure=True, options=cmake_args, path=src_dir, env=env, workdir=obj_dir, doStepIf=FileDoesNotExist("CMakeCache.txt"), **kwargs # Pass through all the extra arguments. )) return f
def addCmakeSteps(f, cleanBuildRequested, obj_dir, install_dir=None, extra_configure_args=None, env=None, stage_name=None, **kwargs): # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # This is an incremental build, unless otherwise has been requested. # Remove obj and install dirs for a clean build. # TODO: Some Windows slaves do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep( RemoveDirectory( name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeToBuild( install_dir, obj_dir) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel), ]) f.addStep( RemoveDirectory( name='clean-%s-dir' % install_dir, dir=install_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Reconcile the cmake options for this build. # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ('-DLLVM_LIT_ARGS=', '"-v"'), ]) # Create configuration files with cmake, unless this has been already done # for an incremental build. if stage_name: step_name = "cmake-configure-%s" % stage_name else: stage_name = "" step_name = "cmake-configure" src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir) f.addStep( CmakeCommand( name=step_name, description=["Cmake", "configure", stage_name], options=cmake_args, path=src_dir, haltOnFailure=kwargs.get('haltOnFailure', True), env=env, workdir=obj_dir, **kwargs # Pass through all the extra arguments. ))
def getSanitizerBuildFactoryII( clean=False, sanity_check=True, sanitizers=[ 'sanitizer', 'asan', 'lsan', 'msan', 'tsan', 'ubsan', 'dfsan' ], common_cmake_options=None, # FIXME: For backward compatibility. Will be removed. extra_configure_args=[], prefixCommand=["nice", "-n", "10"], # For backward compatibility. env=None, jobs="%(jobs)s", timeout=1200): llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM': 'dumb', # Make sure Clang doesn't use color escape sequences. } if env: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) f = BuildFactory() # Clean directory, if requested. cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( RemoveDirectory(name='clean ' + llvm_objdir, dir=llvm_objdir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) # Get llvm, clang, ompiler-rt, libcxx, libcxxabi, libunwind f.addStep( SVN(name='svn-llvm', mode='update', description='svn-llvm', descriptionDone='svn-llvm', baseURL='http://llvm.org/svn/llvm-project/llvm/', defaultBranch='trunk', workdir=llvm_srcdir)) f.addStep( SVN(name='svn-clang', mode='update', description='svn-clang', descriptionDone='svn-clang', baseURL='http://llvm.org/svn/llvm-project/cfe/', defaultBranch='trunk', workdir='%s/tools/clang' % llvm_srcdir)) f.addStep( SVN(name='svn-compiler-rt', mode='update', description='svn-compiler-rt', descriptionDone='svn--compiler-rt', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', defaultBranch='trunk', workdir='%s/projects/compiler-rt' % llvm_srcdir)) f.addStep( SVN(name='svn-libcxx', mode='update', description='svn-libcxx', descriptionDone='svn-libcxx', baseURL='http://llvm.org/svn/llvm-project/libcxx/', defaultBranch='trunk', workdir='%s/projects/libcxx' % llvm_srcdir)) f.addStep( SVN(name='svn-libcxxabi', mode='update', description='svn-libcxxabi', descriptionDone='svn-libcxxabi', baseURL='http://llvm.org/svn/llvm-project/libcxxabi/', defaultBranch='trunk', workdir='%s/projects/libcxxabi' % llvm_srcdir)) f.addStep( SVN(name='svn-libunwind', mode='update', description='svn-libunwind', descriptionDone='svn-libunwind', baseURL='http://llvm.org/svn/llvm-project/libunwind/', defaultBranch='trunk', workdir='%s/projects/libunwind' % llvm_srcdir)) # Run annotated command for sanitizer. if sanity_check: f.addStep( AnnotatedCommand( name="lint check", description="lint check", timeout=timeout, haltOnFailure=False, #True, warnOnWarnings=True, command=["./check_lint.sh"], workdir="%s/projects/compiler-rt/lib/sanitizer_common/scripts" % llvm_srcdir, env=merged_env)) # Always build with ninja. cmakeCommand = ["cmake", "-G", "Ninja"] # Reconsile configure args with the defaults we want. if not any( a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args): cmakeCommand.append('-DCMAKE_BUILD_TYPE=Release') if not any( a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args): cmakeCommand.append('-DLLVM_ENABLE_WERROR=OFF') if not any( a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args): cmakeCommand.append('-DLLVM_ENABLE_ASSERTIONS=ON') if not any( a.startswith('-DCMAKE_C_COMPILER') for a in extra_configure_args): cmakeCommand.append('-DCMAKE_C_COMPILER=clang') if not any( a.startswith('-DCMAKE_CXX_COMPILER') for a in extra_configure_args): cmakeCommand.append('-DCMAKE_CXX_COMPILER=clang++') if not any( a.startswith('-DCMAKE_CXX_FLAGS') for a in extra_configure_args): cmakeCommand.append('-DCMAKE_CXX_FLAGS=\"-std=c++11 -stdlib=libc++\"') if not any( a.startswith('-DCMAKE_EXE_LINKER_FLAGS') for a in extra_configure_args): cmakeCommand.append('-DCMAKE_EXE_LINKER_FLAGS=-lcxxrt') if not any( a.startswith('-DLIBCXXABI_USE_LLVM_UNWINDER=') for a in extra_configure_args): cmakeCommand.append('-DLIBCXXABI_USE_LLVM_UNWINDER=ON') if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args): cmakeCommand.append('-DLLVM_LIT_ARGS=\"-v\"') cmakeCommand += extra_configure_args + ["../%s" % llvm_srcdir] # Note: ShellCommand does not pass the params with special symbols right. # The " ".join is a workaround for this bug. f.addStep( ShellCommand( name="cmake-configure", description=["cmake configure"], haltOnFailure=False, #True, warnOnWarnings=True, command=WithProperties(" ".join(cmakeCommand)), env=merged_env, workdir=llvm_objdir, doStepIf=FileDoesNotExist("./%s/CMakeCache.txt" % llvm_objdir))) # Build everything. f.addStep( NinjaCommand( name='build', haltOnFailure=False, #True, warnOnWarnings=True, description=['building', 'with', 'ninja'], descriptionDone=['built', 'with', 'ninja'], workdir=llvm_objdir, env=merged_env)) # Run tests for each of the requested sanitizers. if sanitizers: for s in sanitizers: f.addStep( NinjaCommand( name='test %s' % s, targets=['check-%s' % s], haltOnFailure=False, #True, description=['testing', '%s' % s], descriptionDone=['test', '%s' % s], workdir=llvm_objdir, env=merged_env)) return f
def getLLDBCMakeBuildFactory( clean=False, jobs="%(jobs)s", # Source directory containing a built python python_source_dir=None, # Default values for VS devenv and build configuration vs=None, config='Release', target_arch='x86', extra_cmake_args=None, test=False, testTimeout=2400, install=False): ############# PREPARING # Global configurations build_dir = 'build' f = LLVMBuildFactory(is_legacy_mode=False, depends_on_projects=["llvm", "clang", "lldb", "lld"], obj_dir=build_dir) # Determine Slave Environment and Set MSVC environment. if vs: f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f.addGetSourcecodeSteps() build_cmd = ['ninja'] install_cmd = ['ninja', 'install'] test_cmd = ['ninja', 'check-lldb'] if jobs: build_cmd.append(WithProperties("-j%s" % jobs)) install_cmd.append(WithProperties("-j%s" % jobs)) test_cmd.append(WithProperties("-j%s" % jobs)) ############# CLEANING cleanBuildRequested = lambda step: clean or step.build.getProperty( "clean", default=step.build.getProperty("clean_obj")) f.addStep( RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, f.obj_dir) cmake_options = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=" + config, "-DCMAKE_INSTALL_PREFIX=../install", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), ] if python_source_dir: cmake_options.append("-DPYTHON_HOME=" + python_source_dir) if extra_cmake_args: cmake_options += extra_cmake_args f.addStep( CmakeCommand(name="cmake-configure", description=["cmake configure"], haltOnFailure=True, options=cmake_options, path=rel_src_dir, env=Property('slave_env'), workdir=build_dir)) f.addStep( WarningCountingShellCommand(name='build', command=build_cmd, haltOnFailure=True, description='ninja build', workdir=build_dir, env=Property('slave_env'))) ignoreInstallFail = bool(install != 'ignoreFail') f.addStep( ShellCommand(name='install', command=install_cmd, flunkOnFailure=ignoreInstallFail, description='ninja install', workdir=build_dir, doStepIf=bool(install), env=Property('slave_env'))) ignoreTestFail = bool(test != 'ignoreFail') f.addStep( ShellCommand(name='test', command=test_cmd, flunkOnFailure=ignoreTestFail, timeout=testTimeout, description='ninja test', workdir=build_dir, doStepIf=bool(test), env=Property('slave_env'))) return f
def getSanitizerWindowsBuildFactory( clean=False, cmake='cmake', # Default values for VS devenv and build configuration vs=r"""%VS120COMNTOOLS%""", config='Release', target_arch='x86', extra_cmake_args=[]): ############# PREPARING f = buildbot.process.factory.BuildFactory() # Kill any stale symbolizer processes for the last run. If there are any # stale processes, the build will fail during linking. This can happen to # any process, but it is most likely to happen to llvm-symbolizer if its # pipe isn't closed. taskkill_cmd = 'taskkill /f /im llvm-symbolizer.exe || exit /b 0' f.addStep( ShellCommand(name='taskkill', description='kill stale processes', command=['cmd', '/c', taskkill_cmd], haltOnFailure=False)) # Determine Slave Environment and Set MSVC environment. f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f = getSource(f, 'llvm') # Global configurations build_dir = 'build' ############# CLEANING cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep( RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) f.addStep( ShellCommand( name='cmake', command=[ cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config, "-DLLVM_ENABLE_ASSERTIONS=ON" ] + extra_cmake_args, haltOnFailure=True, workdir=build_dir, env=Property('slave_env'))) # Build compiler-rt first to speed up detection of Windows-specific # compiler-time errors in the sanitizer runtimes. f.addStep( NinjaCommand(name='build compiler-rt', targets=['compiler-rt'], haltOnFailure=True, description='ninja compiler-rt', workdir=build_dir, env=Property('slave_env'))) # Build Clang and LLD next so that most compilation errors occur in a build # step. f.addStep( NinjaCommand(name='build clang lld', targets=['clang', 'lld'], haltOnFailure=True, description='ninja clang lld', workdir=build_dir, env=Property('slave_env'))) # Only run sanitizer tests. # Don't build targets that are not required in order to speed up the cycle. for target in [ 'check-asan', 'check-asan-dynamic', 'check-sanitizer', 'check-cfi' ]: f.addStep( NinjaCommand(name='run %s' % target, targets=[target], haltOnFailure=False, description='ninja %s' % target, workdir=build_dir, env=Property('slave_env'))) return f
def ros_branch_build(c, job_name, packages, url, branch, distro, arch, rosdistro, machines, othermirror, keys, trigger_pkgs=None): gbp_args = [ '-uc', '-us', '--git-ignore-branch', '--git-ignore-new', '--git-verbose', '--git-dist=' + distro, '--git-arch=' + arch ] with open(os.path.dirname(os.path.realpath(__file__)) + "/spec.yaml") as file: spec_list = yaml.full_load(file) f = BuildFactory() # Remove the build directory. f.addStep( RemoveDirectory( name=job_name + '-clean', dir=Interpolate('%(prop:workdir)s'), hideStepIf=success, )) # Check out the repository master branch, since releases are tagged and not branched f.addStep( Git( repourl=url, branch=branch, alwaysUseLatest= True, # this avoids broken builds when schedulers send wrong tag/rev mode='full', # clean out old versions getDescription={'tags': True})) # Update the cowbuilder f.addStep( ShellCommand(command=['cowbuilder-update.py', distro, arch] + keys, hideStepIf=success)) # Generate the changelog for the package f.addStep( ShellCommand(haltOnFailure=True, name='catkin_generate_changelog', command=['catkin_generate_changelog', '-y'], descriptionDone=['catkin_generate_changelog'])) # Add all files including untracked ones f.addStep( ShellCommand(haltOnFailure=True, name='add_changelogs', command=['git', 'add', '.'], descriptionDone=['add_changelogs'])) # Commit the changelog after updating it f.addStep( ShellCommand(haltOnFailure=True, name='update_changelogs', command=['git', 'commit', '-m', '\"Updated changelogs\"'], descriptionDone=['update_changelogs'])) # Prepare the release without pushing it f.addStep( ShellCommand(haltOnFailure=True, name='catkin_prepare_release', command=[ 'catkin_prepare_release', '--bump', 'minor', '--no-push', '-y' ], descriptionDone=['catkin_prepare_release'])) # f.addStep( ShellCommand( haltOnFailure=True, name='git_bloom_generate_release', command=['git-bloom-generate', '-y', 'rosrelease', rosdistro], )) f.addStep( ShellCommand( haltOnFailure=True, name='git_bloom_generate_debian', command=[ 'git-bloom-generate', '-y', 'rosdebian', '-a', '-p', 'release', rosdistro ], )) # Get the tag number for the lastest commit f.addStep( SetPropertyFromCommand( command="git describe --tags", property="release_version", name='latest_tag', )) # Need to build each package in order for package in packages: debian_pkg = 'ros-' + rosdistro + '-' + package.replace( '_', '-') # debian package name (ros-groovy-foo) branch_name = 'debian/' + debian_pkg + '_%(prop:release_version)s-0_' + distro deb_name = debian_pkg + '_%(prop:release_version)s-0' + distro final_name = debian_pkg + '_%(prop:release_version)s-%(prop:datestamp)s' + distro + '_' + arch + '.deb' # Check out the proper tag. Use --force to delete changes from previous deb stamping f.addStep( ShellCommand(haltOnFailure=True, name=package + '-checkout', command=[ 'git', 'checkout', Interpolate(branch_name), '--force' ], hideStepIf=success)) # A hack for generating the debian folder so we could build the lastest commit of the specified branch # f.addStep( # ShellCommand( # haltOnFailure = True, # name = package+'-bloom_generate', # command= ['bloom-generate', 'rosdebian'], # descriptionDone = ['bloom_generate', package] # ) # ) # Download script for building the source deb f.addStep( FileDownload( name=job_name + '-grab-build-source-deb-script', mastersrc='scripts/build_source_deb.py', slavedest=Interpolate('%(prop:workdir)s/build_source_deb.py'), mode=0755, hideStepIf=success))
def getSanitizerWindowsBuildFactory( clean=False, cmake='cmake', # Default values for VS devenv and build configuration vs=r"""%VS120COMNTOOLS%""", config='Release', target_arch='x86', extra_cmake_args=[]): ############# PREPARING f = buildbot.process.factory.BuildFactory() # Determine Slave Environment and Set MSVC environment. f.addStep(SetProperty( command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractSlaveEnvironment)) f = getSource(f,'llvm') # Global configurations build_dir='build' ############# CLEANING cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean f.addStep(RemoveDirectory(name='clean '+build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested )) f.addStep(ShellCommand(name='cmake', command=[cmake, "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE="+config, "-DLLVM_ENABLE_ASSERTIONS=ON"] + extra_cmake_args, haltOnFailure=True, workdir=build_dir, env=Property('slave_env'))) # Build compiler-rt first to speed up detection of Windows-specific # compiler-time errors in the sanitizers runtime. f.addStep(NinjaCommand(name='build compiler-rt', targets=['compiler-rt'], haltOnFailure=True, description='ninja compiler-rt', workdir=build_dir, env=Property('slave_env'))) # Only run sanitizer tests. # Don't build targets that are not required in order to speed up the cycle. test_targets = ['check-asan','check-asan-dynamic','check-sanitizer', 'check-cfi'] f.addStep(NinjaCommand(name='run tests', targets=test_targets, haltOnFailure=True, description='ninja test', workdir=build_dir, env=Property('slave_env'))) return f