def build_coverity(): remove_build = steps.RemoveDirectory("build") remove_src = steps.RemoveDirectory("src") create_build = steps.MakeDirectory("build") download_src_archive = steps.FileDownload( mastersrc=util.Property("src_archive"), workerdest="src.tar.xz", workdir="src") extract_src_archive = steps.ShellCommand( name="Extract source archive", command=["tar", "xJf", "src.tar.xz"], workdir="src") cmake_step = steps.CMake(path="../src/", definitions=util.Property("cmake_defs", {}), options=util.Property("cmake_opts", []), workdir="build", env=env) make_step = steps.Compile(command=[ "cov-build", "--dir", "cov-int", "make", "-j", "16", "-l", "32" ], workdir="build", env=env) compress = steps.ShellCommand( command=["tar", "czvf", "gnuradio.tgz", "cov-int"], workdir="build") upload = steps.ShellCommand(command=[ "curl", "--form", "token=" + tokens.coverityToken, "--form", "[email protected]", "--form", "[email protected]", "--form", util.Interpolate("version=%(prop:revision)s"), "--form", util.Interpolate( "description=\"Weekly Buildbot submission for %(prop:branch)s branch \"" ), "https://scan.coverity.com/builds?project=GNURadio" ], workdir="build") factory = util.BuildFactory() factory.addStep(remove_build) factory.addStep(remove_src) factory.addStep(create_build) factory.addStep(download_src_archive) factory.addStep(extract_src_archive) factory.addStep(cmake_step) factory.addStep(make_step) factory.addStep(compress) factory.addStep(upload) return factory
def getLLVMBuildFactoryAndPrepareForSourcecodeSteps(depends_on_projects=None, llvm_srcdir=None, obj_dir=None, install_dir=None, cleanBuildRequested=None, env=None, **kwargs): def cleanBuildRequestedByProperty(step): return step.build.getProperty("clean") if cleanBuildRequested is None: # We want a clean checkout only if requested by the property. cleanBuildRequested = cleanBuildRequestedByProperty f = LLVMBuildFactory(depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=obj_dir, install_dir=install_dir, cleanBuildRequested=cleanBuildRequested, **kwargs) # Pass through all the extra arguments. # Remove the source code for a clean checkout if requested by property. # TODO: Some Windows workers do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep( steps.RemoveDirectory( name='clean-src-dir', dir=f.monorepo_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequestedByProperty, )) return f
def build_coverage(): remove_build = steps.RemoveDirectory("build") create_build = steps.MakeDirectory("build") cmake_step = steps.CMake(path=util.Property("src_dir"), definitions=util.Property("cmake_defs", {}), options=util.Property("cmake_opts", []), workdir="build", env=env) @util.renderer def join_make_opts(props): make_opts = props.getProperty("make_opts", []) return ["make"] + make_opts make_step = steps.Compile(command=join_make_opts, workdir="build", env=env) test_coverage = steps.ShellCommand(command=["make", "coverage"], workdir="build") upload_coverage_data = steps.ShellCommand(command=[ "bash", "-c", util.Interpolate("bash <(curl -s https://codecov.io/bash) -t " + tokens.codecovToken + " -C %(prop:revision)s -f coverage.info.cleaned") ], workdir="build") factory = util.BuildFactory() factory.addStep(remove_build) factory.addStep(create_build) factory.addStep(cmake_step) factory.addStep(make_step) factory.addStep(test_coverage) factory.addStep(upload_coverage_data) return factory
def __init__(self): ZcashBaseFactory.__init__(self) self.addSteps([ steps.MakeDirectory( dir='build/ibd-datadir', name='Create datadir', ), sh('touch', 'ibd-datadir/zcash.conf', name='Create zcash.conf'), InitialBlockDownloadTimeRunner( 'ibd-datadir', name='time-InitialBlockDownload', ), steps.RemoveDirectory( dir='build/ibd-datadir', name='Remove datadir', alwaysRun=True, ), steps.POST(urllib.parse.urljoin(CODESPEED_URL, '/result/add/json/'), data={'json': getPerfJson}, auth=('buildbot', CODESPEED_PASS), verify=CODESPEED_CERT, doStepIf=lambda s: s.getProperty('publish', False), hideStepIf=lambda results, s: results==util.SKIPPED, ), ])
def build_and_test(): remove_build = steps.RemoveDirectory("build") remove_src = steps.RemoveDirectory("src") create_build = steps.MakeDirectory("build") download_src_archive = steps.FileDownload( mastersrc=util.Property("src_archive"), workerdest="src.tar.xz", workdir="src") extract_src_archive = steps.ShellCommand( name="Extract source archive", command=["tar", "xJf", "src.tar.xz"], workdir="src") cmake_step = steps.CMake(path="../src/", definitions=util.Property("cmake_defs", {}), options=util.Property("cmake_opts", []), workdir="build", env=env) @util.renderer def join_make_opts(props): make_opts = props.getProperty("make_opts", []) return ["make"] + make_opts make_step = steps.Compile(command=join_make_opts, workdir="build", env=env) @util.renderer def parse_test_excludes(props): command = ["ctest", "--output-on-failure", "--timeout", "120"] excludes = props.getProperty('test_excludes', []) excludes.append("qtgui") if excludes is not None: command += ["-E", "|".join(excludes)] return command test_step = steps.Test(command=parse_test_excludes, workdir="build") factory = util.BuildFactory() factory.addStep(remove_build) factory.addStep(remove_src) factory.addStep(create_build) factory.addStep(download_src_archive) factory.addStep(extract_src_archive) factory.addStep(cmake_step) factory.addStep(make_step) factory.addStep(test_step) return factory
def build_volk_PR(): create_src = steps.MakeDirectory(name="create src directory", dir="volk") clone_step = steps.GitHub(name="fetch PR source", repourl=util.Property("repository"), mode="full", method="fresh", submodules=True, retryFetch=True, clobberOnFailure=True, workdir="volk") rm_src_dir = steps.RemoveDirectory( dir=util.Interpolate( os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s", "%(prop:github.base.ref)s")), hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS, ) copy_src = steps.CopyDirectory( name="copy src to srcdir", src="volk", dest=util.Interpolate( os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s", "%(prop:github.base.ref)s"), ), hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS, ) # load builders.json with definitions on how to build things parent_path = os.path.dirname(__file__) with open(os.path.join(parent_path, "volk_builders.json"), "r") as builders_file: build_config = json.loads(builders_file.read()) trigger_builds = custom_steps.BuildTrigger( name="trigger the right builders", build_config=build_config, schedulerNames=["trigger"], runner="pull", set_properties={ "pr_base": util.Property("github.base.ref"), "src_dir": util.Interpolate( os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s")) }, test_merge=False, updateSourceStamp=False, waitForFinish=True) factory = util.BuildFactory() factory.addStep(create_src) factory.addStep(clone_step) factory.addStep(rm_src_dir) factory.addStep(copy_src) factory.addStep(trigger_builds) return factory
def build_and_test(): remove_build = steps.RemoveDirectory("build") create_build = steps.MakeDirectory("build") cmake_step = steps.CMake(path=util.Property("src_dir"), definitions=util.Property("cmake_defs", {}), options=util.Property("cmake_opts", []), workdir="build", env=env) @util.renderer def join_make_opts(props): make_opts = props.getProperty("make_opts", []) return ["make"] + make_opts make_step = steps.Compile(command=join_make_opts, workdir="build", env=env) def parse_exclude_file(rc, stdout, stderr): exclude_tests = json.loads(stdout) return {"test_excludes": exclude_tests} load_exclude_file = steps.SetPropertyFromCommand( command=["cat", os.path.join("/config", "test_excludes.json")], extract_fn=parse_exclude_file, doStepIf=lambda steps: steps.getProperty("exclude_file", False)) @util.renderer def parse_test_excludes(props): command = ["ctest", "--output-on-failure", "--timeout", "10"] excludes = props.getProperty("test_excludes", None) if excludes is not None: command += ["-E", "|".join(excludes)] return command test_step = steps.Test(command=parse_test_excludes, workdir="build") factory = util.BuildFactory() factory.addStep(remove_build) factory.addStep(create_build) factory.addStep(load_exclude_file) factory.addStep(cmake_step) factory.addStep(make_step) factory.addStep(test_step) return factory
def _addSteps4StagedCompiler(f, stage_idx=1, use_stage_idx=-1, jobs=None, extra_configure_args=None, env=None): if use_stage_idx < 0: use_stage_idx = stage_idx - 1 # Index is zero-based, so we want to use a human friendly number instead. stage_num = stage_idx + 1 # Directories to use on this stage. obj_dir = f.stage_objdirs[stage_idx] src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir) install_dir = LLVMBuildFactory.pathRelativeTo( f.stage_installdirs[stage_idx], obj_dir) staged_install = f.stage_installdirs[use_stage_idx] # Always do a clean build for the staged compiler. f.addStep( steps.RemoveDirectory( name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, )) f.addStep( steps.RemoveDirectory( name='clean-%s-dir' % f.stage_installdirs[stage_idx], dir=f.stage_installdirs[stage_idx], haltOnFailure=False, flunkOnFailure=False, )) # Reconcile the cmake options for this stage. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), ]) # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) cmake_args.append( WithProperties("-DCMAKE_CXX_COMPILER=%(builddir)s/" + staged_install + "/bin/clang++")) cmake_args.append( WithProperties("-DCMAKE_C_COMPILER=%(builddir)s/" + staged_install + "/bin/clang")) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) # Create configuration files with cmake f.addStep( CmakeCommand( name="cmake-configure-stage%s" % stage_num, description=["stage%s cmake configure" % stage_num], haltOnFailure=True, options=cmake_args, path=src_dir, env=env, workdir=obj_dir, )) # Build clang by the staged compiler f.addStep( NinjaCommand( name="build-stage%s-compiler" % stage_num, jobs=jobs, haltOnFailure=True, description=["build stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, )) # Test just built compiler f.addStep( NinjaCommand( name="test-stage%s-compiler" % stage_num, targets=["check-all"], jobs=jobs, haltOnFailure=True, description=["test stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, )) # Install just built compiler f.addStep( NinjaCommand( name="install-stage%s-compiler" % stage_num, targets=["install"], jobs=jobs, haltOnFailure=True, description=["install stage%s compiler" % stage_num], timeout=10800, # LTO could take time. env=env, workdir=obj_dir, ))
def _addSteps4SystemCompiler(f, stage_idx=0, clean=True, jobs=None, extra_configure_args=None, env=None): # Index is zero-based, so we want to use a human friendly number instead. stage_num = stage_idx + 1 # Directories to use on this stage. obj_dir = f.stage_objdirs[stage_idx] src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir) install_dir = LLVMBuildFactory.pathRelativeTo( f.stage_installdirs[stage_idx], obj_dir) # This stage could use incremental build. # Clean stage1, only if requested. f.addStep( steps.RemoveDirectory(name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=clean)) f.addStep( steps.RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx], dir=f.stage_installdirs[stage_idx], haltOnFailure=False, flunkOnFailure=False, doStepIf=clean)) # Reconcile the cmake options for this stage. # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() # Set proper defaults. CmakeCommand.applyDefaultOptions( cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), ('-DLLVM_BUILD_TESTS=', 'ON'), ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), # Do not expect warning free build by the system toolchain. ('-DLLVM_ENABLE_WERROR=', 'OFF'), ]) # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_args, [ ('-G', 'Ninja'), ('-DCMAKE_INSTALL_PREFIX=', install_dir), ]) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) # Note: On this stage we do not care of warnings, as we build with # a system toolchain and cannot control the environment. # Warnings are likely, and we ignore them. # Create configuration files with cmake f.addStep( CmakeCommand( name="cmake-configure-stage%s" % stage_num, description=["stage%s cmake configure" % stage_num], haltOnFailure=True, flunkOnWarnings=False, options=cmake_args, path=src_dir, env=env, workdir=obj_dir, )) # Build clang by the system compiler f.addStep( NinjaCommand( name="build-stage%s-compiler" % stage_num, jobs=jobs, haltOnFailure=True, flunkOnWarnings=False, description=["build stage%s compiler" % stage_num], env=env, workdir=obj_dir, )) # Test stage1 compiler f.addStep( NinjaCommand( name="test-stage%s-compiler" % stage_num, targets=["check-all"], # or "check-llvm", "check-clang" jobs=jobs, haltOnFailure=True, flunkOnWarnings=False, description=["test stage%s compiler" % stage_num], env=env, workdir=obj_dir, )) # Install stage1 compiler f.addStep( NinjaCommand( name="install-stage%s-compiler" % stage_num, targets=["install"], jobs=jobs, haltOnFailure=True, description=["install stage%s compiler" % stage_num], env=env, workdir=obj_dir, ))
def getFuchsiaToolchainBuildFactory( test=True, env=None, # Environmental variables for all steps. extra_configure_args=None, **kwargs): # Prepare environmental variables. Set here all env we want everywhere. merged_env = { "TERM": "dumb", # Make sure Clang doesn't use color escape sequences. } if env is not None: # Overwrite pre-set items with the given ones, so user can set anything. merged_env.update(env) src_dir = "llvm.src" obj_dir = "llvm.obj" install_dir = "llvm.install" f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( depends_on_projects=[ "llvm", "clang", "clang-tools-extra", "compiler-rt", "libcxx", "libcxxabi", "libunwind", "lld" ], llvm_srcdir=src_dir, obj_dir=obj_dir, install_dir=install_dir, **kwargs) # Pass through all the extra arguments. # Get Fuchsia SDK. sdk_dir = "fuchsia.sdk" sdk_platform = { "Linux": "linux-amd64", "Darwin": "mac-amd64", }[platform.system()] sdk_version = "latest" sdk_url = WithProperties( "https://chrome-infra-packages.appspot.com/dl/" "fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s", sdk_platform=lambda _: sdk_platform, sdk_version=lambda _: sdk_version) f.addStep( steps.RemoveDirectory(name="clean-sdk", dir=sdk_dir, haltOnFailure=True)) f.addStep( ShellCommand(name="fetch-sdk", command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url], description=["download", "fuchsia sdk"], workdir=sdk_dir)) f.addStep( ShellCommand(name="extract-sdk", command=["unzip", "-fo", "sdk.cipd"], description=["extract", "fuchsia sdk"], workdir=sdk_dir)) cleanBuildRequested = lambda step: step.build.getProperty("clean", default=True) # Clean up llvm build. f.addStep( steps.RemoveDirectory(name="clean-llvm.obj", dir=obj_dir, haltOnFailure=True, doStepIf=cleanBuildRequested)) f.addStep( steps.RemoveDirectory(name="clean-llvm.install", dir=install_dir, haltOnFailure=True, doStepIf=cleanBuildRequested)) # Configure. if extra_configure_args is None: cmake_options = [] else: cmake_options = extra_configure_args[:] # Some options are required for this stage no matter what. CmakeCommand.applyRequiredOptions(cmake_options, [ ("-G", "Ninja"), ("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"), ("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"), ]) # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_options, [ ("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"), ("-DLLVM_ENABLE_LTO=", "OFF"), ]) cmake_options.append( WithProperties("-DCMAKE_INSTALL_PREFIX=%(builddir)s/" + install_dir)) cmake_options.append( WithProperties("-DFUCHSIA_SDK=%(builddir)s/" + sdk_dir)) CmakeCommand.applyRequiredOptions(cmake_options, [ ("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"), ]) f.addStep( CmakeCommand(name="cmake-configure", options=cmake_options, path='../' + src_dir + '/llvm', haltOnFailure=True, description=["configure"], workdir=obj_dir, env=merged_env, doStepIf=FileDoesNotExist("CMakeCache.txt"))) # Build distribution. f.addStep( NinjaCommand(name="ninja-build", targets=["stage2-distribution"], haltOnFailure=True, description=["build"], workdir=obj_dir, env=merged_env)) # Test llvm, clang and lld. f.addStep( NinjaCommand( name="check", targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")], haltOnFailure=True, description=["check"], workdir=obj_dir, env=merged_env, doStepIf=test)) # Install distribution. f.addStep( NinjaCommand(name="install", targets=["stage2-install-distribution"], haltOnFailure=True, description=["install"], workdir=obj_dir, env=merged_env)) return f
def Clean(**kwargs): return steps.RemoveDirectory( name = "clean", description = "cleaning", descriptionDone = "clean", **kwargs)
def getLLDBCMakeBuildFactory( clean=False, jobs="%(jobs)s", # Source directory containing a built python python_source_dir=None, # Default values for VS devenv and build configuration vs=None, config='Release', target_arch='x86', extra_cmake_args=None, test=False, testTimeout=2400, install=False): ############# PREPARING # Global configurations build_dir = 'build' f = LLVMBuildFactory(depends_on_projects=["llvm", "clang", "lldb", "lld"], obj_dir=build_dir) env = {} # Determine Slave Environment and Set MSVC environment. if vs and vs != 'manual': f.addStep( SetProperty(command=getVisualStudioEnvironment(vs, target_arch), extract_fn=extractVSEnvironment)) env = Property('vs_env') f.addGetSourcecodeSteps() build_cmd = ['ninja'] install_cmd = ['ninja', 'install'] test_cmd = ['ninja', 'check-lldb'] if jobs: build_cmd.append(WithProperties("-j%s" % jobs)) install_cmd.append(WithProperties("-j%s" % jobs)) test_cmd.append(WithProperties("-j%s" % jobs)) ############# CLEANING cleanBuildRequested = lambda step: clean or step.build.getProperty( "clean", default=step.build.getProperty("clean_obj")) f.addStep( steps.RemoveDirectory(name='clean ' + build_dir, dir=build_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested)) rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, f.obj_dir) cmake_options = [ "-G", "Ninja", "-DCMAKE_BUILD_TYPE=" + config, "-DCMAKE_INSTALL_PREFIX=../install", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), ] if python_source_dir: cmake_options.append("-DPYTHON_HOME=" + python_source_dir) if extra_cmake_args: cmake_options += extra_cmake_args f.addStep( CmakeCommand(name="cmake-configure", description=["cmake configure"], haltOnFailure=True, options=cmake_options, path=rel_src_dir, env=env, workdir=build_dir)) f.addStep( WarningCountingShellCommand(name='build', command=build_cmd, haltOnFailure=True, description='ninja build', workdir=build_dir, env=env)) ignoreInstallFail = bool(install != 'ignoreFail') f.addStep( ShellCommand(name='install', command=install_cmd, flunkOnFailure=ignoreInstallFail, description='ninja install', workdir=build_dir, doStepIf=bool(install), env=env)) ignoreTestFail = bool(test != 'ignoreFail') f.addStep( ShellCommand(name='test', command=test_cmd, flunkOnFailure=ignoreTestFail, timeout=testTimeout, description='ninja test', workdir=build_dir, doStepIf=bool(test), env=env)) return f
def getPollyBuildFactory( clean=False, install=False, make='make', jobs=None, checkAll=False, env=None, extraCmakeArgs=None, testsuite=False,extraTestsuiteCmakeArgs=None, **kwargs): if extraCmakeArgs is None: extraCmakeArgs=[] if extraTestsuiteCmakeArgs is None: extraTestsuiteCmakeArgs = [] llvm_srcdir = "llvm.src" llvm_objdir = "llvm.obj" llvm_instdir = "llvm.inst" testsuite_srcdir = "test-suite.src" testsuite_builddir = "test-suite.build" jobs_cmd = [] if jobs is not None: jobs_cmd = ["-j{}".format(jobs)] build_cmd = [make] + jobs_cmd install_cmd = [make, 'install'] + jobs_cmd check_all_cmd = [make, 'check-all'] + jobs_cmd check_polly_cmd = [make, 'check-polly'] + jobs_cmd cmake_install = [] if install: cmake_install = ["-DCMAKE_INSTALL_PREFIX=../%s" % llvm_instdir] # Prepare environmental variables. Set here all env we want everywhere. merged_env = { 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. } if env: merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything. depends_on_projects = ['llvm','clang','polly'] if testsuite: # XRay tests in test-suite require compiler-rt depends_on_projects += ['compiler-rt'] cleanBuildRequestedByProperty = lambda step: step.build.getProperty("clean", False) cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir=llvm_srcdir, obj_dir=llvm_objdir, install_dir=llvm_instdir, cleanBuildRequested=cleanBuildRequested, **kwargs) # Pass through all the extra arguments. f.addStep(steps.RemoveDirectory(name='clean-src-dir', dir=f.monorepo_dir, warnOnFailure=True, doStepIf=cleanBuildRequestedByProperty)) # Get the source code. f.addGetSourcecodeSteps(**kwargs) # Clean build dir f.addStep(steps.RemoveDirectory(name='clean-build-dir', dir=llvm_objdir, warnOnFailure=True, doStepIf=cleanBuildRequested)) # Create configuration files with cmake cmakeCommand = ["cmake", "../%s/llvm" % llvm_srcdir, "-DCMAKE_COLOR_MAKEFILE=OFF", "-DPOLLY_TEST_DISABLE_BAR=ON", "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_POLLY_LINK_INTO_TOOLS=ON", "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects), ] + cmake_install + extraCmakeArgs f.addStep(ShellCommand(name="cmake-configure", command=cmakeCommand, haltOnFailure=False, description=["cmake configure"], workdir=llvm_objdir, env=merged_env)) # Build f.addStep(WarningCountingShellCommand(name="build", command=build_cmd, haltOnFailure=True, description=["build"], workdir=llvm_objdir, env=merged_env)) clangexe = "%(builddir)s/" + llvm_objdir + "/bin/clang" clangxxexe = "%(builddir)s/" + llvm_objdir + "/bin/clang++" litexe = "%(builddir)s/" + llvm_objdir + "/bin/llvm-lit" sizeexe = "%(builddir)s/" + llvm_objdir + "/bin/llvm-size" # Clean install dir if install: f.addStep(steps.RemoveDirectory(name='clean-install-dir', dir=llvm_instdir, haltOnFailure=False, doStepIf=cleanBuildRequested)) f.addStep(ShellCommand(name="install", command=install_cmd, haltOnFailure=True, description=["install"], workdir=llvm_objdir, env=merged_env)) # If installing, use the installed version of clang. clangexe = "%(builddir)s/" + llvm_instdir + "/bin/clang" clangxxexe = "%(builddir)s/" + llvm_instdir + "/bin/clang++" sizeexe = "%(builddir)s/" + llvm_instdir + "/bin/llvm-size" # Test if checkAll: f.addStep(LitTestCommand(name="check_all", command=check_all_cmd, haltOnFailure=False, description=["check all"], workdir=llvm_objdir, env=merged_env)) else: f.addStep(LitTestCommand(name="check_polly", command=check_polly_cmd, haltOnFailure=False, description=["check polly"], workdir=llvm_objdir, env=merged_env)) if testsuite: f.addStep(steps.RemoveDirectory(name='test-suite_clean-src-dir', dir=testsuite_srcdir, haltOnFailure=False, warnOnFailure=True, doStepIf=cleanBuildRequestedByProperty)) f.addGetSourcecodeForProject( project='test-suite', src_dir=testsuite_srcdir, alwaysUseLatest=True) f.addStep(steps.RemoveDirectory(name='test-suite_clean-build-dir', dir=testsuite_builddir, haltOnFailure=False, warnOnFailure=True)) # -Wno-unused-command-line-argument is needed because linking will not uses the "-mllvm -polly" argument. f.addStep(ShellCommand(name='test-suite_cmake-configure', description=["Test-Suite: cmake"], command=["cmake", '-B', testsuite_builddir, '-S', testsuite_srcdir, "-DCMAKE_BUILD_TYPE=Release", "-DTEST_SUITE_COLLECT_STATS=ON", "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", "-DTEST_SUITE_EXTRA_CXX_FLAGS=-mllvm -polly", "-DTEST_SUITE_LIT_FLAGS=-vv;-o;report.json", WithProperties("-DCMAKE_C_COMPILER=" + clangexe), WithProperties("-DCMAKE_CXX_COMPILER=" + clangxxexe), WithProperties("-DTEST_SUITE_LLVM_SIZE=" + sizeexe), WithProperties("-DTEST_SUITE_LIT=" + litexe), ] + extraTestsuiteCmakeArgs, haltOnFailure=True, workdir='.', env=merged_env)) f.addStep(WarningCountingShellCommand(name='test-suite_build', description=["Test-Suite: build"], # Continue building; programs that don't compile will fail with NOEXE. command=[make, 'all', '-k0'] + jobs_cmd, haltOnFailure=False, flunkOnFailure=True, workdir=testsuite_builddir, env=merged_env)) f.addStep(LitTestCommand(name='test-suite_run', description=['Test-Suite: run'], command=[WithProperties(litexe), '-vv', '-o', 'report.json', '.'], haltOnFailure=True, workdir=testsuite_builddir, logfiles={ 'test.log' : 'test.log', 'report.json': 'report.json' }, env=merged_env)) return f
def addCmakeSteps(f, cleanBuildRequested, obj_dir, generator=None, install_dir=None, extra_configure_args=None, env=None, stage_name=None, **kwargs): # Make a local copy of the configure args, as we are going to modify that. if extra_configure_args: cmake_args = extra_configure_args[:] else: cmake_args = list() if obj_dir is None: obj_dir = f.obj_dir # This is an incremental build, unless otherwise has been requested. # Remove obj and install dirs for a clean build. # TODO: Some Windows workers do not handle RemoveDirectory command well. # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows. f.addStep( steps.RemoveDirectory( name='clean-%s-dir' % obj_dir, dir=obj_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), ]) if install_dir: install_dir_rel = LLVMBuildFactory.pathRelativeTo(install_dir, obj_dir) CmakeCommand.applyRequiredOptions(cmake_args, [ ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel), ]) f.addStep( steps.RemoveDirectory( name='clean-%s-dir' % install_dir, dir=install_dir, haltOnFailure=False, flunkOnFailure=False, doStepIf=cleanBuildRequested, )) # Reconcile the cmake options for this build. # Set proper defaults. CmakeCommand.applyDefaultOptions(cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'Release'), ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), ('-DLLVM_LIT_ARGS=', '-v -vv'), ]) # Create configuration files with cmake, unless this has been already done # for an incremental build. if stage_name: step_name = "cmake-configure-%s" % stage_name else: stage_name = "" step_name = "cmake-configure" src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir) # Make a local copy of the configure args, as we are going to modify that. definitions = dict() options = list() for d in cmake_args: if isinstance(d, str) and d.startswith("-D"): k, v = d[2:].split('=', 1) definitions[k] = v else: options.append(d) f.addStep( CmakeCommand( name=step_name, haltOnFailure=True, description=["Cmake", "configure", stage_name], generator=generator, definitions=definitions, options=options, path=src_dir, env=env or {}, workdir=obj_dir, **kwargs # Pass through all the extra arguments. ))
def build_weekly(): create_src = steps.MakeDirectory(name="create src directory", dir="src") clone_step = steps.GitHub(name="fetch PR source", repourl=util.Property("repository"), mode="full", method="fresh", submodules=True, clobberOnFailure=True, getDescription=True, workdir="src") rm_src_dir = steps.RemoveDirectory(dir=util.Interpolate( os.path.join(_WEEKLY_SRC_BASE, "%(prop:branch)s", "%(prop:commit-description)s"))) copy_src = steps.CopyDirectory(name="copy src to srcdir", src="src", dest=util.Interpolate( os.path.join( _WEEKLY_SRC_BASE, "%(prop:branch)s", "%(prop:commit-description)s"), )) set_merge_property = steps.SetProperty( property=util.Interpolate("merge_%(prop:branch)s"), value=True, hideStepIf=True, ) # load builders.json with definitions on how to build things parent_path = os.path.dirname(__file__) with open(os.path.join(parent_path, "builders.json"), "r") as builders_file: build_config = json.loads(builders_file.read()) # now we have all necessary merge properties together, # we can actually kickoff builds for them trigger_builds = custom_steps.BuildTrigger( name="trigger all builders", build_config=build_config, schedulerNames=["trigger"], runner="time", set_properties={ "src_dir": util.Interpolate( os.path.join(_WEEKLY_SRC_BASE, "%(prop:branch)s", "%(prop:commit-description)s")), "got_revision": util.Property("got_revision"), }, updateSourceStamp=True, waitForFinish=True) factory = util.BuildFactory() factory.addStep(create_src) factory.addStep(clone_step) factory.addStep(rm_src_dir) factory.addStep(copy_src) factory.addStep(set_merge_property) factory.addStep(trigger_builds) return factory
def __init__(self, *args, **kwargs): util.BuildFactory.__init__(self, *args, **kwargs) title = 'Liri OS' releasever = '30' self.addSteps([ ImagePropertiesStep(name='set properties'), steps.ShellCommand( name='update container', haltOnFailure=True, command=['dnf', 'update', '-y'], ), steps.ShellCommand( name='install tools', haltOnFailure=True, command=[ 'dnf', 'install', '-y', 'git', 'spin-kickstarts', 'pykickstart', 'livecd-tools' ], ), steps.Git( name='checkout sources', codebase=util.Property('codebase'), repourl=util.Property('repository'), branch=util.Property('branch'), mode='incremental', submodules=True, shallow=True, ), steps.ShellCommand( name='ksflatten', haltOnFailure=True, command=[ 'ksflatten', Interpolate('--config=%(prop:product)s-livecd.ks'), '-o', 'livecd.ks' ], ), steps.RemoveDirectory( name='clean cache', dir='/build/cache', doStepIf=IsCacheDisabled, ), steps.ShellCommand( name='build image', haltOnFailure=True, timeout=60 * 60, command=[ 'livecd-creator', '--releasever=' + releasever, '--config=livecd.ks', Interpolate('--fslabel=%(prop:imgname)s'), '--title', title, Interpolate('--product=%(prop:product)s'), '--cache=/build/cache' ], ), steps.ShellCommand( name='checksum', haltOnFailure=True, command=[ 'bash', '-c', Interpolate( 'sha256sum -b --tag %(prop:isofilename)s > /repo/images/nightly/%(prop:checksumfilename)s' ) ], ), steps.ShellCommand( name='move file', command=[ 'mv', Interpolate('%(prop:isofilename)s'), '/repo/images/nightly/' ], ), steps.ShellCommand( name='remove old images', command=[ 'bash', '-c', 'find /repo/images/nightly -type f -mtime +7 -exec rm {} \;' ], ) ])
def build_PR(): # @util.renderer # def check_mergeable(props): # mergeable = props.getProperty("github.mergeable", False) # return mergeable # check_mergeables = steps.Assert( # check_mergeable, # name="check if PR was mergeable", # haltOnFailure=True # ) create_src = steps.MakeDirectory(name="create src directory", dir="src") clone_step = steps.GitHub(name="fetch PR source", repourl=util.Property("repository"), mode="full", method="fresh", submodules=True, retryFetch=True, clobberOnFailure=True, workdir="src") set_merge_property = steps.SetProperty( name="set merge property", property=util.Interpolate("merge_%(prop:github.base.ref)s"), value=True, hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS, ) create_merge_branch = steps.ShellCommand( name="create test_merge branch for further steps", command=[ "git", "branch", "-f", util.Interpolate("test_merge_%(prop:github.base.ref)s") ], workdir="src") rm_src_dir = steps.RemoveDirectory( dir=util.Interpolate( os.path.join(_BASEPATH, "pull", "%(prop:github.number)s", "%(prop:github.base.ref)s")), hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS, ) copy_src = steps.CopyDirectory( name="copy src to srcdir", src="src", dest=util.Interpolate( os.path.join(_BASEPATH, "pull", "%(prop:github.number)s", "%(prop:github.base.ref)s"), ), hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS, ) master_steps = sequences.mergeability_sequence("master", "maint", _PULL_SRC_BASE) next_steps = sequences.mergeability_sequence("next", "master", _PULL_SRC_BASE) python3_steps = sequences.mergeability_sequence("python3", "next", _PULL_SRC_BASE) # load builders.json with definitions on how to build things parent_path = os.path.dirname(__file__) with open(os.path.join(parent_path, "builders.json"), "r") as builders_file: build_config = json.loads(builders_file.read()) trigger_builds = custom_steps.BuildTrigger( name="trigger the right builders", build_config=build_config, schedulerNames=["trigger"], runner="pull", set_properties={ "pr_base": util.Property("github.base.ref"), "src_dir": util.Interpolate( os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s")) }, updateSourceStamp=False, waitForFinish=True) factory = util.BuildFactory() factory.addStep(create_src) factory.addStep(clone_step) factory.addStep(set_merge_property) factory.addStep(create_merge_branch) factory.addStep(rm_src_dir) factory.addStep(copy_src) factory.addSteps(master_steps) factory.addSteps(next_steps) factory.addSteps(python3_steps) factory.addStep(trigger_builds) return factory