def masterConfig(): c = {} from buildbot.plugins import schedulers, steps, util class SleepAndInterrupt(steps.ShellSequence): @defer.inlineCallbacks def run(self): if self.worker.worker_system == "nt": sleep = "waitfor SomethingThatIsNeverHappening /t 100 >nul 2>&1" else: sleep = ["sleep", "100"] d = self.runShellSequence([util.ShellArg(sleep)]) yield asyncSleep(1) self.interrupt("just testing") res = yield d return res c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] f = util.BuildFactory() f.addStep(SleepAndInterrupt()) c['builders'] = [ util.BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def addBuilder(self, worker, factory): module = self.importWithReload(factory) self.masterConf['builders'].append( util.BuilderConfig(name=worker, workernames=[worker], factory=module.getFactory())) self.shedulers.addScheduler(module.getPropertyes(), worker) self.builders.append(worker)
def add_builders(b): for branch in config['branch_to_namespace_mapping']: realm = config['branch_to_namespace_mapping'][branch] buildername = "wwdicebot-" + realm b['builders'].append( util.BuilderConfig(name=buildername, workernames=["klaital-worker"], factory=_make_factory(), tags=['wwdicebot', 'home', 'discord', 'bots']))
def getPullRequestBuilder(props, pretty_branch_name): builders = [] for jdk in common.getJDKBuilds(props): jdk_props = dict(props) jdk_props['jdk'] = str(jdk) builders.append( util.BuilderConfig(name=pretty_branch_name + " Pull Request Build JDK " + str(jdk), workernames=workers, factory=build.getPullRequestPipeline(), collapseRequests=True, properties=jdk_props, locks=[mvn_lock.access('exclusive')])) builders.append( util.BuilderConfig(name=pretty_branch_name + " Pull Request Reports JDK " + str(jdk), workernames=workers, factory=reports.getPullRequestPipeline(), collapseRequests=True, properties=jdk_props, locks=[mvn_lock.access('exclusive')])) builders.append( util.BuilderConfig(name=pretty_branch_name + " Pull Request Markdown", workernames=workers, factory=markdown.getPullRequestPipeline(), properties=props, collapseRequests=True)) # builders.append(util.BuilderConfig( # name=pretty_branch_name + " Pull Request Database Tests", # workernames=workers, # factory=database.getPullRequestPipeline(), # properties=props, # collapseRequests=True, # locks=[db_lock.access('exclusive')])) return builders
def add_all_builders(b): for s_name in SERVICES: factory = _make_factory(s_name, SERVICES[s_name]) for branch in get_all_possible_branch_names(): realm = branch[len('deploy-'):] b.append( util.BuilderConfig( name=f"{s_name}_{realm}", workernames=WORKERNAMES, factory=factory, locks=[helpers.WORKER_LOCK.access('exclusive')], tags=[s_name, realm]))
def create_builders(repos_and_packages): ret = [] for repo in repos_and_packages: for artifact in repo['artifacts']: print 'building builder for artifact', artifact['artifact'] build_factory = util.BuildFactory() build_factory.addStep( MonorepoGitStep(artifact['path'], repourl=repo['repo'], mode='full', method='copy')) has_assets = artifact.get('assets', None) != None if artifact['type'] == 'npm': add_npm_build_steps(build_factory, has_assets) ret.append( util.BuilderConfig( name=artifact['artifact'], workernames=["build-npm"], factory=build_factory, properties={'owners': artifact.get('owners', [])})) elif artifact['type'] == 'docker-npm': add_docker_npm_build_steps(build_factory, artifact['artifact'], has_assets) ret.append( util.BuilderConfig( name=artifact['artifact'], workernames=["build-docker-npm"], factory=build_factory, properties={'owners': artifact.get('owners', [])})) artifactsrc_yml_build_factory = util.BuildFactory() add_artifactsrc_yml_build_steps(artifactsrc_yml_build_factory, repo['repo']) ret.append( util.BuilderConfig(name='build-artifactsrc-yml', workernames=["build-artifactsrc-yml"], factory=artifactsrc_yml_build_factory)) return ret
def add_builders(b): for branch in config['branch_deployment_configs']: realm = config['branch_deployment_configs'][branch]['namespace'] buildername = "vitasa-" + realm tags = ['vita', config['branch_deployment_configs'][branch]['namespace']] if config['branch_deployment_configs'][branch]['kubeconfig'] == config['home_kubeconfig']: tags.append('home') b['builders'].append(util.BuilderConfig(name=buildername, workernames=["vitasa-worker"], factory=_make_factory(config['branch_deployment_configs'][branch]), tags=tags ))
def get_builders(): factory = util.BuildFactory() # check out the source factory.addStep( steps.Git(repourl='git://github.com/buildbot/hello-world.git', mode='incremental')) # run the tests (note that this will require that 'trial' is installed) factory.addStep( steps.ShellCommand(command=["trial", "hello"], env={"PYTHONPATH": "."})) return [ util.BuilderConfig(name="runtests", workernames=["default"], factory=factory) ]
def masterConfig(): global num_reconfig num_reconfig += 1 c = {} from buildbot.plugins import schedulers, steps, util c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] f = util.BuildFactory() f.addStep( steps.SetPropertyFromCommand(property="test", command=["echo", "foo"])) c['builders'] = [ util.BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def nix_build_config( worker_names: list[str], enable_cachix: bool ) -> util.BuilderConfig: """ Builds one nix flake attribute. """ factory = util.BuildFactory() factory.addStep( NixBuildCommand( env={}, name="Build flake attr", command=[ "nix", "build", "-L", "--out-link", util.Interpolate("result-%(prop:attr)s"), util.Property("drv_path"), ], haltOnFailure=True, ) ) if enable_cachix: factory.addStep( steps.ShellCommand( name="Upload cachix", env=dict(CACHIX_AUTH_TOKEN=util.Secret("cachix-token")), command=[ "cachix", "push", util.Secret("cachix-name"), util.Interpolate("result-%(prop:attr)s"), ], ) ) factory.addStep(UpdateBuildOutput(name="Update build output", branches=["master"])) return util.BuilderConfig( name="nix-build", workernames=worker_names, properties=[], collapseRequests=False, env={}, factory=factory, )
def error(message): try: name = constructicon_name + '-' + builder_name except NameError: name = constructicon_name global errors name += '-uniquifier-{}'.format(errors) errors += 1 all_builders.append( util.BuilderConfig(name=name, description=git_state + ' error: ' + message, slavenames=['none'], factory=util.BuildFactory())) all_schedulers.append( ForceScheduler( name=name + '-force', builderNames=[name], )) log.msg('builder {}: {}'.format(name, message))
def nix_update_flake_config( worker_names: list[str], projectname: str, github_token_secret: str ) -> util.BuilderConfig: """ Updates the flake an opens a PR for it. """ factory = util.BuildFactory() url_with_secret = util.Interpolate( f"https://*****:*****@github.com/{projectname}" ) factory.addStep( steps.Git( repourl=url_with_secret, method="clean", submodules=True, haltOnFailure=True, ) ) factory.addStep( steps.ShellCommand( name="Update flakes", env=dict( GIT_AUTHOR_NAME="buildbot", GIT_AUTHOR_EMAIL="*****@*****.**", GIT_COMMITTER_NAME="buildbot", GIT_COMMITTER_EMAIL="*****@*****.**", ), command=[ "nix", "flake", "update", "--commit-lock-file", "--commit-lockfile-summary", "flake.lock: Update", ], haltOnFailure=True, ) ) factory.addStep( steps.ShellCommand( name="Force-Push to update_flake_lock branch", command=[ "git", "push", "--force", "origin", "HEAD:refs/heads/update_flake_lock", ], haltOnFailure=True, ) ) factory.addStep( CreatePr( name="Create pull-request", env=dict(GITHUB_TOKEN=util.Secret(github_token_secret)), command=[ "gh", "pr", "create", "--repo", projectname, "--title", "flake.lock: Update", "--body", "Automatic buildbot update", "--head", "refs/heads/update_flake_lock", "--base", "master", ], ) ) return util.BuilderConfig( name="nix-update-flake", workernames=worker_names, factory=factory, properties=dict(virtual_builder_name="nix-update-flake"), )
def getPerPlatformBuilders(self, platform): if not platform.canBuild(self): return # Don't use os.path.join as builder is a linux image src_path = "{0}/src".format("/data") build_path = "{0}/build".format("/data") # daily_builds_path is used in Package step on master side daily_builds_path = os.path.join(config.daily_builds_dir, self.name) # Ensure last path component doesn't get removed here and in packaging step daily_builds_url = urlp.urljoin(config.daily_builds_url + '/', self.name + '/') configure_path = src_path + "/configure" env = platform.getEnv(self) # Setup ccache as the compiler, use already set CXX as real compiler or environement CXX from docker image env['CXX'] = 'ccache {0}'.format(env.get('CXX', '${CXX}')) f = util.BuildFactory() f.workdir = "" f.useProgress = False self.addCleanSteps(f, platform, env=env) self.addConfigureSteps(f, platform, configure_path=configure_path, env=env) self.addBuildSteps(f, platform, env=env) self.addTestsSteps(f, platform, env=env) self.addPackagingSteps(f, platform, env=env, src_path=src_path, daily_builds_path=daily_builds_path, daily_builds_url=daily_builds_url) locks = [ lock_build.access('counting'), self.lock_src.access("counting") ] if platform.lock_access: locks.append(platform.lock_access(self)) yield util.BuilderConfig( name=self.names['bld-platform'](platform), workernames=workers.workers_by_type['builder'], workerbuilddir=build_path, factory=f, locks=locks, tags=["build", self.name, platform.name], properties={ "buildname": self.name, "platformname": platform.name, "workerimage": platform.getWorkerImage(self), }, )
warnOnFailure=False, flunkOnFailure=False)) # factory.addStep(steps.ShellCommand( # name="remove dangling docker images", # logEnviron=False, # command=["/bin/sh", "-c", "docker images --filter dangling=true --quiet | xargs docker rmi"], # haltOnFailure=False, # warnOnFailure=False, # flunkOnFailure=False) # ) c['builders'] = [] c['builders'].append( util.BuilderConfig(name="cite-build", workernames=workernames, factory=factory)) ####### REPORTERS c['services'] = [ CiteGitHubStatusPush(baseURL=Config.get("github", "api_url"), token=Config.get("github", "api_token"), verbose=True) ] # minimalistic config to activate new web UI c['www'] = dict(port=8010, plugins=dict(waterfall_view={}, console_view={}), change_hook_dialects={'github': {}}) ####### DB URL
from buildbot.plugins import util, steps site_vitrine_builder = util.BuilderConfig( name='SiteVitrine', workername='site_vitrine', factory=util.BuildFactory([ steps.Git(repourl='[email protected]:CentraleFitness/site_vitrine.git', mode='incremental'), steps.ShellCommand(command=["yarn"]), steps.ShellCommand(command=["yarn", "build"]), steps.ShellCommand(command=["forever", "stop", "bin/www"]), steps.ShellCommand(command=["forever", "start", "bin/www"]), ])) site_vitrine_builder_alt = util.BuilderConfig( name='SiteVitrine_npm', workername='site_vitrine', factory=util.BuildFactory([ steps.Git(repourl='[email protected]:CentraleFitness/site_vitrine.git', mode='incremental'), steps.ShellCommand(command=["npm", "install"]), steps.ShellCommand(command=["npm", "run", "build"]), steps.ShellCommand(command=["forever", "stop", "bin/www"]), steps.ShellCommand(command=["forever", "start", "bin/www"]), ]))
from buildbot.plugins import util, steps from .helpers.steps import venv_step, service_step PYTHON_EX = "/var/buildbot/workers/backoffice_api/venv/bin/python3" backoffice_api_builder = util.BuilderConfig( name='BackofficeApi', workername='backoffice_api', factory=util.BuildFactory([ steps.Git( repourl='[email protected]:CentraleFitness/backoffice-server.git', mode='incremental'), steps.ShellCommand( command=["mv", "config/config_prod.py", "config/config.py"]), venv_step('backoffice_api', 'BackofficeApi'), steps.ShellCommand( command=[PYTHON_EX, "manage.py", "migrate"]), service_step('backoffice_api', pidfile="/var/run/backoffice_api.pid") ]) )
builders = [] architecture_testing_list = get_arches() for kernel_branch in branches_list: for arch in architecture_testing_list: for toolchain in arch["toolchain"]: tags = [] tags.append(toolchain["name"]) tags.append(arch["name"]) tags.append(kernel_branch) builders.append( util.BuilderConfig(name=kernel_branch + ':' + arch["name"] + ':' + toolchain["name"], tags=tags, workernames=get_workers_for( arch["name"], toolchain["name"]), factory=download_new_patch_and_build_kernel( kernel_branch, arch["name"]))) builders.append( util.BuilderConfig(name='gentoo_sources', workernames=get_workers_for("gentoo_sources", None), factory=test_gentoo_sources())) builders.append( util.BuilderConfig(name='other_sources', workernames=get_workers_for("other_sources", None), factory=test_gentoo_sources())) builders.append(
# # Builders that build the CLIP OS Dockerized build environment images to be # then used by the clipos.workers.DockerLatentWorker: docker_buildenv_image_builders = [] for flavor, props in clipos.workers.DockerLatentWorker.FLAVORS.items(): docker_buildenv_image_builders.append( util.BuilderConfig( name='docker-worker-image env:{}'.format( flavor), # keep this short description=line( """Build the Docker image to use as a Buildbot Docker latent worker based upon a {} userland.""").format( props['description']), tags=['docker-worker-image', "docker-env:{}".format(flavor)], # Temporary: Always build on the worker that have access to the # Docker socket: workernames=[ docker_operations_worker.name, ], factory=clipos.build_factories.BuildDockerImage( flavor=flavor, buildmaster_setup=setup), )) # The builder that generates the repo dir and git lfs archive artifacts from # scratch: repo_sync_builder = util.BuilderConfig( name='repo-sync', description=line( """Synchronize the CLIP OS source tree and produce a tarball from the
def getGlobalBuilders(self, platforms): ret = list() f = util.BuildFactory() f.workdir = "" f.useProgress = False f.addStep( steps.Git( mode="incremental", repourl=self.giturl, branch=self.branch, locks=[self.lock_src.access("exclusive")], )) if len(self.PATCHES): f.addStep( scummsteps.Patch( base_dir=config.configuration_dir, patches=self.PATCHES, locks=[self.lock_src.access("exclusive")], )) if self.nightly is not None: # Trigger nightly scheduler to let it know the source stamp f.addStep( steps.Trigger( name="Updating source stamp", schedulerNames=["nightly-scheduler-{0}".format(self.name)], set_properties={ 'got_revision': util.Property('got_revision', defaultWhenFalse=False), }, updateSourceStamp=True, hideStepIf=(lambda r, s: r == util.SUCCESS), )) f.addStep( steps.Trigger(name="Building all platforms", schedulerNames=[self.names['sch-build']], set_properties={ 'got_revision': util.Property('got_revision', defaultWhenFalse=False), 'clean': util.Property('clean', defaultWhenFalse=False), 'package': util.Property('package', defaultWhenFalse=False) }, updateSourceStamp=True, waitForFinish=True)) ret.append( util.BuilderConfig( name=self.names['bld-fetch'], workernames=workers.workers_by_type['fetcher'], workerbuilddir="/data/src/{0}".format(self.name), factory=f, tags=["fetch", self.name], locks=[lock_build.access('counting')], )) if self.nightly is not None: f = util.BuildFactory() f.addStep( steps.Trigger( name="Building all platforms", schedulerNames=[self.names['sch-build']], updateSourceStamp=True, waitForFinish=True, set_properties={ 'got_revision': util.Property('got_revision', defaultWhenFalse=False), 'clean': True, 'package': True, # Ensure our tag is put first and is split from the others 'owner': ' Nightly build ', })) ret.append( util.BuilderConfig( name=self.names['bld-nightly'], # We use fetcher worker here as it will prevent building of other stuff like if a change had happened workernames=workers.workers_by_type['fetcher'], workerbuilddir="/data/triggers/nightly-{0}".format( self.name), factory=f, tags=["nightly", self.name], locks=[lock_build.access('counting')])) snapshots_path = os.path.join(config.snapshots_dir, self.name) # Builder to clean packages f = util.BuildFactory() f.addStep( scummsteps.CleanupSnapshots( dstpath=snapshots_path, buildname=self.name, platformnames=[ platform.name for platform in platforms if platform.canPackage(self) ], dry_run=util.Property("dry_run", False), keep_builds=getattr(config, 'snapshots_keep_builds', 14), obsolete=timedelta( days=getattr(config, 'snapshots_obsolete_days', 30)), cleanup_unknown=getattr(config, 'snapshots_clean_unknown', True), )) ret.append( util.BuilderConfig( name=self.names['bld-clean'], workernames=workers.workers_by_type['fetcher'], workerbuilddir="/data/triggers/cleanup-{0}".format(self.name), factory=f, tags=["cleanup", self.name], locks=[lock_build.access('counting')])) return ret
def make_builder_config(repo_url, name, worker_name, config, lock, snapshots_dir, snapshots_url, snapshots_default_max): builder = util.BuildFactory() builder.addStep( steps.SetProperties(name="Worker Config File", properties=config, hideStepIf=True)) builder.addStep( steps.SetPropertiesFromEnv( variables=["WORKER_HOST", "WORKER_REPO_DIR"], hideStepIf=True)) # TODO: use `reference` to a common volume instead? or make the builder # dependent on another fetch-only builder so only one builder tries to pull it? builder.addStep( steps.GitHub(repourl=repo_url, workdir=Property("WORKER_REPO_DIR", None), logEnviron=False, getDescription={ "always": True, "tags": True })) builder.addStep( FileExistsSetProperty(name="config.mk Existence Check", property="already_configured", file="%s/config.mk" % builder.workdir, hideStepIf=True)) compilation_environment = Property("env", {}) builder.addStep( steps.Configure(command=compute_configure, env=compilation_environment, doStepIf=ConfigChecker().needs_configuration)) builder.addStep( steps.SetPropertyFromCommand(name="Python (Worker)", property="cpu_count", command=["python", "-c", GET_CPU_COUNT], flunkOnFailure=False, warnOnFailure=True, hideStepIf=True, description="getting CPU count", descriptionDone="got CPU count")) # In at least Buildbot 0.9.12, warningPattern and suppressionList are not # renderable, so just get the properties from the config file immediately compiler_warning_pattern = config.get( "compiler_warning_pattern", r"^([^:]+):(\d+):(?:\d+:)? [Ww]arning: (.*)$") compiler_warning_extractor = steps.Compile.warnExtractFromRegexpGroups compiler_suppression_file = Property("compiler_suppression_file", None) compiler_suppression_list = config.get("compiler_suppression_list", None) builder.addStep( steps.Compile(command=["make", Interpolate("-j%(prop:cpu_count:~1)s")], env=compilation_environment, warningPattern=compiler_warning_pattern, warningExtractor=compiler_warning_extractor, suppressionFile=compiler_suppression_file, suppressionList=compiler_suppression_list)) builder.addStep( steps.Test(command=[ "make", Interpolate("%(prop:can_run_tests:" "#?|test|test/runner)s") ], env=compilation_environment, warningPattern=compiler_warning_pattern, warningExtractor=compiler_warning_extractor, suppressionFile=compiler_suppression_file, suppressionList=compiler_suppression_list, haltOnFailure=True, flunkOnFailure=True)) if snapshots_dir is not None and snapshots_url is not None: if snapshots_dir and snapshots_dir[-1] is not "/": snapshots_dir += "/" if snapshots_url and snapshots_url[-1] is not "/": snapshots_url += "/" snapshots_dir = "%s%%(prop:branch)s/" % snapshots_dir snapshots_url = "%s%%(prop:branch)s/" % snapshots_url builder.addStep( steps.SetProperty(name="Computed By %s" % path.basename(__file__), property="package_name", value=compute_package_name, hideStepIf=True, doStepIf=should_package)) builder.addStep( Package(package_name=Property("package_name"), package_files=Property("package_files", None), package_format=Property("package_archive_format"), make_target=Property("package_make_target"), split_debug_package=Property("split_debug_package", True), extra_files=Property("package_extra_files", None), package_script=Interpolate(config.get( "package_script", "")), env=compilation_environment, doStepIf=should_package)) latest_link = Interpolate("%s%%(prop:buildername)s-latest." "%%(prop:package_archive_format:-tar.xz)s" % snapshots_dir) make_uploader_steps(builder=builder, snapshots_dir=snapshots_dir, snapshots_url=snapshots_url, publish_name="archive", property_name="package_filename", latest_link=latest_link, do_step_if=should_package) latest_link = Interpolate("%s%%(prop:buildername)s" "-latest-debug-symbols.tar.xz" % snapshots_dir) make_uploader_steps(builder=builder, snapshots_dir=snapshots_dir, snapshots_url=snapshots_url, publish_name="debug archive", property_name="debug_package_filename", latest_link=latest_link, do_step_if=should_package_debug) builder.addStep( MasterCleanSnapshots( name="clean old snapshots", workdir=Interpolate(snapshots_dir), file_prefix=Interpolate("%(prop:buildername)s-"), num_to_keep=Property("num_snapshots_to_keep", snapshots_default_max), secondary_file_suffix="-debug-symbols", file_extensions=r"\.(?:tar(?:\.[xg]z)?|[a-z]{3,4})$", doStepIf=should_package, hideStepIf=True)) return util.BuilderConfig(name=name, workername=worker_name, collapseRequests=True, factory=builder, nextBuild=pick_next_build, locks=[lock.access("counting")])
####### BUILDERS # The 'builders' list defines the Builders, which tell Buildbot how to perform a build: # what steps, and which workers can execute them. Note that any particular build will # only take place on one worker. build_lock = util.MasterLock("Build") c["builders"] = [] c["builders"].extend(generate_builder(target) for target in test_targets) c["builders"].append( util.BuilderConfig( name="build", workernames=["director-worker"], factory=build_factory, locks=[build_lock.access("exclusive")], ) ) lingo_factory = util.BuildFactory() lingo_factory.addStep(checkout_step) lingo_factory.addStep(download_step) lingo_directory = "./engines/director/lingo/tests/" lingo_factory.addStep( GenerateStartMovieCommands( directory=lingo_directory, game_id="directortest", name="Generate lingo test commands", command=["find", lingo_directory, "-name", "*.lingo", "-printf", "%P\n"],
def getBuildersForBranch(props): pretty_branch_name = props['branch_pretty'] deb_props = dict(props) deb_props['image'] = random.choice({{docker_debian_worker_images}}) cent_props = dict(props) cent_props['image'] = random.choice({{docker_centos_worker_images}}) builders = getPullRequestBuilder(props, pretty_branch_name) #Only one maven build, per branch, at a time branch_mvn_lock = util.MasterLock(pretty_branch_name + "mvn_lock") for jdk in common.getJDKBuilds(props): jdk_props = dict(props) jdk_props['jdk'] = str(jdk) builders.append( util.BuilderConfig( name=pretty_branch_name + " Build JDK " + str(jdk), workernames=workers, factory=build.getBuildPipeline(), properties=jdk_props, collapseRequests=True, #A note on these locks: We want a single maven build per branch, # AND a single maven build per worker locks=[ mvn_lock.access('exclusive'), branch_mvn_lock.access('exclusive') ])) report_props = dict(jdk_props) report_props['cores'] = '1' builders.append( util.BuilderConfig( name=pretty_branch_name + " Reports JDK " + str(jdk), workernames=workers, factory=reports.getBuildPipeline(), properties=jdk_props, collapseRequests=True, #A note on these locks: We want a single maven build per branch, # AND a single maven build per worker locks=[ mvn_lock.access('exclusive'), branch_mvn_lock.access('exclusive') ])) release_props = dict(props) #We use the first listed JDK since that (should) be the lowest, most common version release_props['jdk'] = str(common.getJDKBuilds(props)[0]) builders.append( util.BuilderConfig( name=pretty_branch_name + " Release", workernames=workers, factory=release.getBuildPipeline(), properties=release_props, collapseRequests=True, #Note: We want a single maven build per worker, but since this is a release we don't # care if there are other maven builds running elsewhere locks=[mvn_lock.access('exclusive')])) builders.append( util.BuilderConfig(name=pretty_branch_name + " Markdown", workernames=workers, factory=markdown.getBuildPipeline(), properties=props, collapseRequests=True)) # builders.append(util.BuilderConfig( # name=pretty_branch_name + " Database Tests", # workernames=workers, # factory=database.getBuildPipeline(), # properties=props, # collapseRequests=True, # locks=[db_lock.access('exclusive')])) builders.append( util.BuilderConfig(name=pretty_branch_name + " Debian Packaging", workernames=workers, factory=debs.getBuildPipeline(), properties=deb_props, collapseRequests=True, locks=[deb_lock.access('exclusive')])) for distro in (7, 8): el_props = dict(props) el_props['el_version'] = distro if 7 == distro: el_props['image'] = f"cent{distro}" lock = el7_lock elif 8 == distro: el_props['image'] = f"rocky{distro}" lock = el8_lock if "Develop" == pretty_branch_name: #Set the RPM branch to master el_props['rpmspec_override'] = "master" #Override/set a bunch of the build props since the RPM's dont relaly have a develop... builders.append( util.BuilderConfig(name=pretty_branch_name + f" el{distro} RPM Packaging", workernames=workers, factory=rpms.getBuildPipeline(), properties=el_props, collapseRequests=True, locks=[lock.access('exclusive')])) if props['deploy_env']: deploy_props = dict(props) deploy_props['deploy_suite'] = '{{ repo_deploy_suite }}' deploy_props['package_repo_host'] = "{{ repo_host }}" deploy_props['key_url'] = "{{ key_url }}" deploy_props['key_id'] = "{{ key_id }}" builders.append( util.BuilderConfig( name=pretty_branch_name + " Ansible Deploy", workernames=workers, factory=ansible.getBuildPipeline(), properties=deploy_props, collapseRequests=True, #Ensure that no one is changing the package databases while we're deploying! locks=[ deb_lock.access('exclusive'), el7_lock.access('exclusive'), el8_lock.access('exclusive') ])) return builders
def make_builder_config(repo_url, name, worker_name, config, lock, snapshots_dir, snapshots_url, snapshots_default_max): if snapshots_dir and snapshots_dir[-1] is not "/": snapshots_dir += "/" if snapshots_url and snapshots_url[-1] is not "/": snapshots_url += "/" builder = util.BuildFactory() builder.addStep( steps.SetProperties(name="Worker Config File", properties=config, hideStepIf=True)) builder.addStep( steps.SetPropertiesFromEnv( variables=["WORKER_HOST", "WORKER_REPO_DIR"], hideStepIf=True)) # TODO: use `reference` to a common volume instead? or make the builder # dependent on another fetch-only builder so only one builder tries to pull it? builder.addStep( steps.GitHub(repourl=repo_url, workdir=Property("WORKER_REPO_DIR", None), logEnviron=False, getDescription={ "always": True, "tags": True })) builder.addStep( FileExistsSetProperty(name="config.mk Existence Check", property="already_configured", file="%s/config.mk" % builder.workdir, hideStepIf=True)) compilation_environment = Property("env", {}) builder.addStep( steps.Configure(command=compute_configure, env=compilation_environment, doStepIf=is_not_configured)) builder.addStep( steps.SetPropertyFromCommand(name="Python (Worker)", property="cpu_count", command=["python", "-c", GET_CPU_COUNT], flunkOnFailure=False, warnOnFailure=True, hideStepIf=True, description="getting CPU count", descriptionDone="got CPU count")) # In at least Buildbot 0.9.12, warningPattern and suppressionList are not # renderable, so just get the properties from the config file immediately compiler_warning_pattern = config.get( "compiler_warning_pattern", r"^([^:]+):(\d+):(?:\d+:)? [Ww]arning: (.*)$") compiler_warning_extractor = steps.Compile.warnExtractFromRegexpGroups compiler_suppression_file = Property("compiler_suppression_file", None) compiler_suppression_list = config.get("compiler_suppression_list", None) builder.addStep( steps.Compile(command=["make", Interpolate("-j%(prop:cpu_count:~1)s")], env=compilation_environment, warningPattern=compiler_warning_pattern, warningExtractor=compiler_warning_extractor, suppressionFile=compiler_suppression_file, suppressionList=compiler_suppression_list)) builder.addStep( steps.Test(command=[ "make", Interpolate("%(prop:can_run_tests:" "#?|test|test/runner)s") ], env=compilation_environment, warningPattern=compiler_warning_pattern, warningExtractor=compiler_warning_extractor, suppressionFile=compiler_suppression_file, suppressionList=compiler_suppression_list, haltOnFailure=True, flunkOnFailure=True)) if snapshots_dir is not None and snapshots_url is not None: builder.addStep( steps.SetProperty(name="Computed By %s" % path.basename(__file__), property="package_name", value=compute_package_name, hideStepIf=True, doStepIf=should_package)) builder.addStep( Package(package_name=Property("package_name"), package_format=Property("package_archive_format"), make_target=Property("package_make_target"), package_directory=Property("package_directory", None), strip_binaries=Property("package_strip_binaries", None), env=compilation_environment, doStepIf=should_package)) source_path = Property("package_filename") target_path = Interpolate("%s%%(prop:package_filename)s" % snapshots_dir) target_url = Interpolate("%s%%(prop:package_filename)s" % snapshots_url) # This is not an ideal target link calculation since the archive format # in package_filename might be fixed up by the Package step, but here # only None is converted into tar.xz, which is not exactly the same target_link = Interpolate("%s%%(prop:buildername)s-latest." "%%(prop:package_archive_format:-tar.xz)s" % snapshots_dir) builder.addStep( CleaningFileUpload(name="publish", workersrc=source_path, masterdest=target_path, url=target_url, clean=True, doStepIf=should_package)) builder.addStep( steps.MasterShellCommand( name="update latest archive", command=["ln", "-sf", target_path, target_link], logEnviron=False, doStepIf=should_package)) builder.addStep( MasterCleanSnapshots( name="clean old snapshots", workdir=snapshots_dir, file_prefix=Interpolate("%(prop:buildername)s-"), num_to_keep=Property("num_snapshots_to_keep", snapshots_default_max), doStepIf=should_package)) return util.BuilderConfig(name=name, workername=worker_name, collapseRequests=True, factory=builder, nextBuild=pick_next_build, locks=[lock.access("exclusive")])
resources = set(resources + get_builder_base_spec('resources')) #get - ignore builder_spec.remove('get') #append f = factory(constructicon_name, builder_name, deps, precommands + commands, upload, zip, unzip, url, resources) unused = builder_spec.unused() if unused: error('unused configuration keys\n' + pprint.pformat(unused)) continue all_builders.append( util.BuilderConfig( name=full_builder_name, description=global_repo_urls[constructicon_name] + ' ' + git_state + ' on cybertron ' + common.cybertron_git_state() + ' in devastator ' + common.git_state(), slavenames=slave_names, factory=f, locks=[slave_lock.access('exclusive')], )) #schedulers schedulers = get_constructicon_spec(constructicon_spec, 'schedulers') if not check(schedulers, 'schedulers', [ [lambda x: isinstance(x, Config), 'is not a dict'], [ lambda x: all([type(i) == str for i in x.keys()]), "has a key that isn't a str" ], [ lambda x: all([isinstance(j, Config) for i, j in x.items()]), "has a value that isn't a dict"
def make_config(worker_name, worker_password, worker_port, git_repo, branch, poll_interval, builder_name, project_name, project_url, buildbot_url, buildbot_web_port, buildbot_from_email): return { 'workers': [worker.Worker(worker_name, worker_password)], 'protocols': { 'pb': { 'port': worker_port } }, 'change_source': [ changes.GitPoller( git_repo, workdir='gitpoller-workdir', branch=branch, pollinterval=poll_interval, ), ], 'schedulers': [ schedulers.SingleBranchScheduler( name="all", change_filter=util.ChangeFilter(branch=branch), treeStableTimer=poll_interval, builderNames=[builder_name], ), schedulers.ForceScheduler( name="force", builderNames=[builder_name], ), ], 'builders': [ util.BuilderConfig( name=builder_name, workernames=[worker_name], factory=util.BuildFactory([ # check out the source steps.Git(repourl=git_repo, mode='incremental'), # run the tests steps.ShellCommand(command=[ "direnv", "allow", ".", ], ), steps.ShellCommand( command=[ "direnv", "exec", ".", "make", "check", ], env={ 'NIX_REMOTE': 'daemon', }, # If we have to rebuild our dependencies from scratch, # we can go a long time without receiving output from # the compiler. Default timeout is 20 mins, bump to # 1hr. timeout=60 * 60, ), ]), ), ], 'status': [], 'title': project_name, 'titleURL': project_url, 'buildbotURL': buildbot_url, 'www': { 'port': buildbot_web_port, 'plugins': { 'waterfall_view': {}, }, }, 'db': { 'db_url': "sqlite:///state.sqlite", }, 'services': [ reporters.MailNotifier( fromaddr=buildbot_from_email, # TODO(jml): Currently sending mail for all builds. We should # send mail under fewer circumstances once we have a better # idea about what we actually want. # # http://buildbot.readthedocs.io/en/latest/manual/cfg-reporters.html?highlight=github#mailnotifier-arguments mode='all', # XXX: Temporarily hard-code until we can figure out how to # get these automatically from commits. extraRecipients=[ "*****@*****.**", "*****@*****.**", ], ) ], }
def nix_eval_config( worker_names: list[str], github_token_secret: str ) -> util.BuilderConfig: """ Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel. For each evaluated attribute a new build pipeline is started. If all builds succeed and the build was for a PR opened by the flake update bot, this PR is merged. """ factory = util.BuildFactory() # check out the source factory.addStep( steps.GitHub( repourl=util.Property("repository"), method="clean", submodules=True ) ) factory.addStep( NixEvalCommand( env={}, name="Eval flake", command=[ "nix", "run", "github:nix-community/nix-eval-jobs", "--", "--workers", "8", "--gc-roots-dir", # FIXME: don't hardcode this "/var/lib/buildbot-worker/gcroot", "--flake", ".#hydraJobs", ], haltOnFailure=True, ) ) # Merge flake-update pull requests if CI succeeds factory.addStep( MergePr( name="Merge pull-request", env=dict(GITHUB_TOKEN=util.Secret(github_token_secret)), github_token_secret=util.Secret(github_token_secret), base_branches=["master"], owners=["doctor-cluster-bot"], command=[ "gh", "pr", "merge", "--repo", util.Property("project"), "--rebase", util.Property("pullrequesturl"), ], ) ) return util.BuilderConfig( name="nix-eval", workernames=worker_names, factory=factory, properties=dict(virtual_builder_name="nix-eval"), )
c["schedulers"] = [ schedulers.SingleBranchScheduler( name=config.TRIGGER, change_filter=util.ChangeFilter(category="mediasdk"), treeStableTimer=config.BUILDBOT_TREE_STABLE_TIMER, builderNames=[config.TRIGGER]) ] for builder_name, properties in config.FLOW.get_prepared_builders().items(): if properties.get('add_triggerable_sheduler', True): c["schedulers"].append( schedulers.Triggerable(name=builder_name, builderNames=[builder_name])) c["builders"].append( util.BuilderConfig(name=builder_name, workernames=get_workers(properties.get("worker")), factory=properties['factory'])) # Push status of build to the Github c["services"] = [ reporters.GitHubStatusPush( token=config.GITHUB_TOKEN, context=util.Interpolate("buildbot/%(prop:buildername)s"), startDescription="Started", endDescription="Done", verbose=True) ] # Will be useful for implementing build notifications in the future # reporters.GitHubCommentPush(token=config.GITHUB_TOKEN, # startDescription="Started (comment)", # endDescription="Done (comment)",
def getPerPlatformBuilders(self, platform): if not platform.canBuild(self): return [] # Don't use os.path.join as builder is a linux image src_path = "{0}/src/{1}".format("/data", self.name) configure_path = src_path + "/configure" build_path = "{0}/builds/{1}/{2}".format("/data", platform.name, self.name) # snapshots_path is used in Package step on master side snapshots_path = os.path.join(config.snapshots_dir, self.name) # Ensure last path component doesn't get removed here and in packaging step snapshots_url = urlp.urljoin(config.snapshots_url + '/', self.name + '/') env = platform.getEnv(self) f = util.BuildFactory() f.workdir = "" f.useProgress = False self.addCleanSteps(f, platform, env=env) self.addConfigureSteps(f, platform, configure_path=configure_path, env=env) self.addBuildSteps(f, platform, env=env) self.addTestsSteps(f, platform, env=env) self.addPackagingSteps(f, platform, env=env, src_path=src_path, snapshots_path=snapshots_path, snapshots_url=snapshots_url) locks = [ lock_build.access('counting'), self.lock_src.access("counting") ] if platform.lock_access: locks.append(platform.lock_access(self)) return [ util.BuilderConfig( name=self.names['bld-platform'](platform), workernames=workers.workers_by_type['builder'], workerbuilddir=build_path, factory=f, locks=locks, tags=["build", self.name, platform.name], properties={ "platformname": platform.name, "workerimage": platform.getWorkerImage(self), }, ) ]
def get(workers): """ construct a list of builders with the given list of workers """ builders = [] builders.append( util.BuilderConfig( name="pull_request_runner", tags=["control", "gnuradio", "pull"], workernames=[ w.name for w in filterWorkers(workers, "tasks", "control") ], factory=build_PR())) builders.append( util.BuilderConfig( name="volk_pull_request_runner", tags=["control", "volk", "pull"], workernames=[ w.name for w in filterWorkers(workers, "tasks", "control") ], factory=build_volk_PR())) builders.append( util.BuilderConfig( name="repo_push_runner", tags=["control", "push"], workernames=[ w.name for w in filterWorkers(workers, "tasks", "control") ], factory=build_push())) builders.append( util.BuilderConfig( name="weekly_runner", tags=["control", "weekly"], workernames=[ w.name for w in filterWorkers(workers, "tasks", "control") ], factory=build_weekly())) build_workers = filterWorkers(workers, "tasks", "build") distros = [ w.properties.getProperty("distro") for w in build_workers if w.properties.getProperty("distro", None) ] distros = list(set(distros)) for distro in distros: builders.append( util.BuilderConfig( name="build_" + distro, tags=["build"], workernames=[ w.name for w in filterWorkers(build_workers, "distro", distro) ], factory=build_and_test())) coverity_workers = filterWorkers(workers, "tasks", "coverity") builders.append( util.BuilderConfig(name="test_coverity", tags=["test", "coverity"], workernames=[w.name for w in coverity_workers], factory=build_coverity())) builders.append( util.BuilderConfig(name="test_coverage", tags=["test", "coverage"], workernames=[w.name for w in coverity_workers], factory=build_coverage())) return builders
from buildbot.plugins import util, steps from .helpers.steps import venv_step, service_step PYTHON_EX = "/var/buildbot/workers/email_api/venv/bin/python3" email_api_builder = util.BuilderConfig( name='EmailApi', workername='email_api', factory=util.BuildFactory([ steps.Git(repourl='[email protected]:CentraleFitness/email_api.git', mode='incremental'), venv_step('email_api', 'EmailApi', 'requirements.txt'), steps.ShellCommand( command=["mv", "config/buildbot.py", "config/config.py"]), steps.ShellCommand(command=[PYTHON_EX, "manage.py", "migrate"]), steps.ShellCommand(command=[PYTHON_EX, "manage.py", "collectstatic"]), service_step('email_api', pidfile="/var/run/email_api.pid") ]))