class CrossbowReport(CrossbowBuilder):
steps = Extend([
SetPropertyFromCommand(
'crossbow_job_id',
extract_fn=lambda stdout, stderr: stdout.strip(),
command=Crossbow(
args=[
'--github-token', util.Secret('ursabot/github_token'),
'latest-prefix', crossbow_prefix
]
),
workdir='arrow/dev/tasks'
),
Crossbow(
args=util.FlattenList([
'--github-token', util.Secret('ursabot/github_token'),
'report',
'--send',
'--poll',
'--poll-max-minutes', 120,
'--poll-interval-minutes', 15,
'--sender-name', 'Crossbow',
'--sender-email', '*****@*****.**',
'--recipient-email', '*****@*****.**',
'--smtp-user', util.Secret('crossbow/smtp_user'),
'--smtp-password', util.Secret('crossbow/smtp_password'),
util.Property('crossbow_job_id')
]),
workdir='arrow/dev/tasks'
)
])
def loadSigningKey():
pathFrom = "s3://{{ s3_private_bucket }}/{{ groups['master'][0] }}/key/signing.key"
pathTo = "-"
command = 'cp'
return shellCommand(
command=util.Interpolate("aws --endpoint-url {{ s3_host }} s3 " + command + " " + pathFrom + " " + pathTo + " | gpg --import"),
env={
"AWS_ACCESS_KEY_ID": util.Secret("s3.private_access_key"),
"AWS_SECRET_ACCESS_KEY": util.Secret("s3.private_secret_key")
},
name="Load signing key")
def getLatestBuildRevision():
pathFrom = "s3://{{ s3_public_bucket }}/builds/%(prop:branch_pretty)s/latest.txt"
pathTo = "-"
command = 'cp'
return steps.SetPropertyFromCommand(
command=['aws', '--endpoint-url', '{{ s3_host }}', 's3', command, util.Interpolate(pathFrom), util.Interpolate(pathTo)],
env={
"AWS_ACCESS_KEY_ID": util.Secret("s3.public_access_key"),
"AWS_SECRET_ACCESS_KEY": util.Secret("s3.public_secret_key")
},
# Note: We're overwriting this value to set it to the built revision rather than whatever it defaults to
property="got_revision",
flunkOnFailure=True,
haltOnFailure=True,
name="Get latest build version")
class CrossbowTrigger(DockerBuilder):
tags = ['crossbow']
steps = [
GitHub(name='Clone Arrow',
repourl=util.Property('repository'),
workdir='arrow',
mode='full'),
GitHub(
name='Clone Crossbow',
# TODO(kszucs): read it from the comment and set as a property
repourl='https://github.com/ursa-labs/crossbow',
workdir='crossbow',
branch='master',
mode='full',
# quite misleasing option, but it prevents checking out the branch
# set in the sourcestamp by the pull request, which refers to arrow
alwaysUseLatest=True),
Crossbow(args=util.FlattenList([
'--github-token',
util.Secret('ursabot/github_token'), 'submit', '--output',
'job.yml', '--job-prefix', 'ursabot', '--arrow-remote',
util.Property('repository'),
util.Property('crossbow_args', [])
]),
workdir='arrow/dev/tasks',
result_file='job.yml')
]
images = images.filter(name='crossbow', tag='worker')
def nix_build_config(
worker_names: list[str], enable_cachix: bool
) -> util.BuilderConfig:
"""
Builds one nix flake attribute.
"""
factory = util.BuildFactory()
factory.addStep(
NixBuildCommand(
env={},
name="Build flake attr",
command=[
"nix",
"build",
"-L",
"--out-link",
util.Interpolate("result-%(prop:attr)s"),
util.Property("drv_path"),
],
haltOnFailure=True,
)
)
if enable_cachix:
factory.addStep(
steps.ShellCommand(
name="Upload cachix",
env=dict(CACHIX_AUTH_TOKEN=util.Secret("cachix-token")),
command=[
"cachix",
"push",
util.Secret("cachix-name"),
util.Interpolate("result-%(prop:attr)s"),
],
)
)
factory.addStep(UpdateBuildOutput(name="Update build output", branches=["master"]))
return util.BuilderConfig(
name="nix-build",
workernames=worker_names,
properties=[],
collapseRequests=False,
env={},
factory=factory,
)
def test_constructor_secrets(self):
fake_storage_service = FakeSecretStorage()
secret_service = SecretManager()
secret_service.services = [fake_storage_service]
yield secret_service.setServiceParent(self.master)
fake_storage_service.reconfigService(secretdict={"passkey": "1234"})
bs = yield self.createWorker('bot', util.Secret('passkey'))
yield bs.startService()
self.assertEqual(bs.password, '1234')
def setUp(self):
self.setUpTestReactor()
fakeStorageService = FakeSecretStorage()
fakeStorageService.reconfigService(
secretdict={"secret_key": self._SECRET})
secretService = SecretManager()
secretService.services = [fakeStorageService]
self.changeHook = _prepare_github_change_hook(
self, strict=True, secret=util.Secret("secret_key"))
self.changeHook.master.addService(secretService)
def setUp(self):
self.setup_test_reactor()
self.master = fakeMasterForHooks(self)
fakeStorageService = FakeSecretStorage()
fakeStorageService.reconfigService(secretdict={"secret_key": self._SECRET})
self.secretService = SecretManager()
self.secretService.services = [fakeStorageService]
self.master.addService(self.secretService)
self.changeHook = change_hook.ChangeHookResource(
dialects={'gitlab': {'secret': util.Secret("secret_key")}},
master=self.master)
def AWSStep(command, name, doStepIf=True, hideStepIf=False, access=util.Secret("s3.public_access_key"), secret=util.Secret("s3.public_secret_key")):
commandAry = list()
commandAry.extend(['aws', '--endpoint-url', '{{ s3_host }}']),
if type(command) == list:
commandAry.extend(command)
else:
commandAry.append(command)
return shellCommand(
command=commandAry,
env={
"AWS_ACCESS_KEY_ID": access,
"AWS_SECRET_ACCESS_KEY": secret
},
name=name,
doStepIf=doStepIf,
hideStepIf=hideStepIf)
def writeBuildResultsToDatabase(**kwargs):
"""Call the script to save results to the database"""
return [
steps.SetPropertyFromCommand(
name="Save test results to the database",
command=[
util.Interpolate(
"%(prop:builddir)s/scripts/write_build_results.py"),
"--run-id",
util.Property("buildId"),
util.Property("jsonResultsFile"), "--database-info",
util.Secret("dataBaseInfo.json")
],
extract_fn=extractDatabaseBuildid,
**kwargs)
]
class CrossbowSubmit(CrossbowBuilder):
"""Submit crossbow jobs
This builder is driven via buildbot properties, the `crossbow_args`
property is either set by the github hook which parses the github comments
like `@ursabot package -g conda` (ror more see commands.py) or by
explicitly passing by NightlySchedulers.
"""
steps = Extend([
Crossbow(args=util.FlattenList([
'--output-file', 'result.yaml', '--github-token',
util.Secret('kszucs/github_status_token'), 'submit',
'--arrow-remote', arrow_repository, '--job-prefix',
crossbow_prefix,
util.Property('crossbow_args', [])
]),
workdir='arrow/dev/tasks',
result_file='result.yaml')
])
def masterConfig(use_interpolation):
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import schedulers, steps, util
c['services'] = [
FakeSecretReporter('http://example.com/hook',
auth=('user', Interpolate('%(secret:httppasswd)s')))
]
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
c['secretsProviders'] = [
FakeSecretStorage(
secretdict={
"foo": "secretvalue",
"something": "more",
'httppasswd': 'myhttppasswd'
})
]
f = BuildFactory()
if use_interpolation:
if os.name == "posix":
# on posix we can also check whether the password was passed to the command
command = Interpolate(
'echo %(secret:foo)s | ' +
'sed "s/secretvalue/The password was there/"')
else:
command = Interpolate('echo %(secret:foo)s')
else:
command = ['echo', util.Secret('foo')]
f.addStep(steps.ShellCommand(command=command))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
class CrossbowTrigger(DockerBuilder):
tags = ['crossbow']
env = {
'GIT_COMMITTER_NAME': 'ursabot',
'GIT_COMMITTER_EMAIL': '*****@*****.**'
}
steps = [
GitHub(
name='Clone Arrow',
repourl=util.Property('repository'),
workdir='arrow',
mode='full'
),
GitHub(
name='Clone Crossbow',
repourl=util.Property('crossbow_repo'),
workdir='crossbow',
branch='master',
mode='full',
# quite misleasing option, but it prevents checking out the branch
# set in the sourcestamp by the pull request, which refers to arrow
alwaysUseLatest=True
),
Crossbow(
args=util.FlattenList([
'--github-token', util.Secret('ursabot/github_token'),
'submit',
'--output', 'job.yml',
'--job-prefix', 'ursabot',
'--arrow-remote', util.Property('repository'),
util.Property('crossbow_args', [])
]),
workdir='arrow/dev/tasks',
result_file='job.yml'
)
]
images = images.filter(
name='crossbow',
tag='worker'
)
def createBuildfactory():
factory = util.BuildFactory()
factory.addSteps(common.cloneRepository())
factory.addStep(steps.ShellCommand(
name=util.Interpolate("Register in the Docker Registry %(prop:dockerRegistryUkrl)s"),
command=["docker", "login", util.Property("dockerRegistryUrl"),
"--username", constants.DOCKER_REGISTRY_USER_NAME,
"--password", util.Secret("dockerRegistryPassword")
],
haltOnFailure=True
))
factory.addSteps(common.downloadAndRunScript(
name=util.Interpolate("Build docker image for %(prop:target)s"),
scriptName="build_maxscale_docker_image.py",
args=[
"--product", util.Property("mdbciProductName"),
"--product-version", util.Property("target"),
"--name", util.Property("dockerProductName"),
"--tag", util.Property("target"),
"--registry", util.Property("dockerRegistryUrl")
],
workdir=util.Interpolate("%(prop:builddir)s/build/maxscale/"),
))
return factory
def getBuildPipeline():
debsClone = steps.Git(repourl="{{ source_deb_repo_url }}",
branch=util.Property('branch'),
alwaysUseLatest=True,
mode="full",
method="fresh",
flunkOnFailure=True,
haltOnFailure=True,
name="Cloning deb packaging configs")
debsVersion = steps.SetPropertyFromCommand(
command="git rev-parse HEAD",
property="deb_script_rev",
flunkOnFailure=True,
haltOnFailure=True,
workdir="build",
name="Get Debian script revision")
removeSymlinks = common.shellCommand(
command=['rm', '-rf', 'binaries', 'outputs'],
alwaysRun=True,
name="Prep cloned repo for CI use")
debsFetch = common.syncAWS(
pathFrom="s3://{{ s3_public_bucket }}/builds/{{ builds_fragment }}",
pathTo=
"binaries/%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/",
name="Fetch build from S3")
debsBuild = common.shellSequence(commands=[
common.shellArg(command=[
'dch', '--changelog', 'opencast/debian/changelog', '--newversion',
util.Interpolate(
'%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s-%(prop:buildnumber)s-%(prop:short_revision)s'
), '-b', '-D', 'unstable', '-u', 'low', '--empty',
util.Interpolate(
'Opencast revision %(prop:got_revision)s, packaged with Debian scripts version %(prop:deb_script_rev)s'
)
],
logname='dch'),
common.shellArg(command=[
'rm', '-f',
util.Interpolate(
"binaries/%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/revision.txt"
)
],
logname='cleanup'),
common.shellArg(command=util.Interpolate(
'echo "source library.sh\ndoOpencast %(prop:pkg_major_version)s.%(prop:pkg_minor_version)s %(prop:branch)s %(prop:got_revision)s" | tee build.sh'
),
logname='write'),
common.shellArg(command=util.Interpolate(
'ln -s opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s.orig.tar.xz opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s-%(prop:buildnumber)s.orig.tar.xz'
),
logname='link'),
common.shellArg(command=['bash', 'build.sh'], logname='build'),
common.shellArg(command=util.Interpolate(
'echo "Opencast version %(prop:got_revision)s packaged with version %(prop:deb_script_rev)s" | tee outputs/%(prop:oc_commit)s/revision.txt'
),
logname='revision')
],
env={
"NAME":
"Buildbot",
"EMAIL":
"buildbot@{{ groups['master'][0] }}",
"SIGNING_KEY":
util.Interpolate(
"%(prop:signing_key)s")
},
name="Build debs")
debRepoClone = steps.Git(repourl="{{ source_deb_packaging_repo_url }}",
branch="{{ deb_packaging_repo_branch }}",
alwaysUseLatest=True,
mode="full",
method="fresh",
flunkOnFailure=True,
haltOnFailure=True,
name="Cloning deb repo configs")
debRepoLoadKeys = common.shellCommand(command=['./build-keys'],
name="Loading signing keys")
debRepoCreate = common.shellCommand(
command=[
'./create-branch',
util.Interpolate("%(prop:pkg_major_version)s.x")
],
name=util.Interpolate(
"Ensuring %(prop:pkg_major_version)s.x repos exist"))
debRepoIngest = common.shellCommand(
command=[
'./include-binaries',
util.Interpolate("%(prop:pkg_major_version)s.x"),
util.Interpolate("%(prop:repo_component)s"),
util.Interpolate(
"outputs/%(prop:revision)s/opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.x-%(prop:buildnumber)s-%(prop:short_revision)s_amd64.changes"
)
],
name=util.Interpolate(
f"Adding build to %(prop:pkg_major_version)s.x-%(prop:repo_component)s"
))
debRepoPrune = common.shellCommand(
command=[
'./clean-unstable-repo',
util.Interpolate("%(prop:pkg_major_version)s.x")
],
name=util.Interpolate(
f"Pruning %(prop:pkg_major_version)s.x unstable repository"))
debRepoPublish = common.shellCommand(
command=[
"./publish-branch",
util.Interpolate("%(prop:pkg_major_version)s.x"),
util.Interpolate("%(prop:signing_key)s")
],
name=util.Interpolate("Publishing %(prop:pkg_major_version)s.x"),
env={
"AWS_ACCESS_KEY_ID": util.Secret("s3.public_access_key"),
"AWS_SECRET_ACCESS_KEY": util.Secret("s3.public_secret_key")
})
f_package_debs = util.BuildFactory()
f_package_debs.addStep(common.getPreflightChecks())
f_package_debs.addStep(debsClone)
f_package_debs.addStep(debsVersion)
f_package_debs.addStep(common.getLatestBuildRevision())
f_package_debs.addStep(common.getShortBuildRevision())
f_package_debs.addStep(removeSymlinks)
f_package_debs.addStep(debsFetch)
f_package_debs.addStep(common.loadSigningKey())
f_package_debs.addStep(debsBuild)
f_package_debs.addStep(debRepoClone)
f_package_debs.addStep(debRepoLoadKeys)
f_package_debs.addStep(common.deployS3fsSecrets())
f_package_debs.addStep(common.mountS3fs())
f_package_debs.addStep(debRepoCreate)
f_package_debs.addStep(debRepoIngest)
f_package_debs.addStep(debRepoPrune)
f_package_debs.addStep(debRepoPublish)
f_package_debs.addStep(common.unloadSigningKey())
f_package_debs.addStep(common.unmountS3fs())
f_package_debs.addStep(common.cleanupS3Secrets())
f_package_debs.addStep(common.getClean())
return f_package_debs
def nix_eval_config(
worker_names: list[str], github_token_secret: str
) -> util.BuilderConfig:
"""
Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel.
For each evaluated attribute a new build pipeline is started.
If all builds succeed and the build was for a PR opened by the flake update bot,
this PR is merged.
"""
factory = util.BuildFactory()
# check out the source
factory.addStep(
steps.GitHub(
repourl=util.Property("repository"), method="clean", submodules=True
)
)
factory.addStep(
NixEvalCommand(
env={},
name="Eval flake",
command=[
"nix",
"run",
"github:nix-community/nix-eval-jobs",
"--",
"--workers",
"8",
"--gc-roots-dir",
# FIXME: don't hardcode this
"/var/lib/buildbot-worker/gcroot",
"--flake",
".#hydraJobs",
],
haltOnFailure=True,
)
)
# Merge flake-update pull requests if CI succeeds
factory.addStep(
MergePr(
name="Merge pull-request",
env=dict(GITHUB_TOKEN=util.Secret(github_token_secret)),
github_token_secret=util.Secret(github_token_secret),
base_branches=["master"],
owners=["doctor-cluster-bot"],
command=[
"gh",
"pr",
"merge",
"--repo",
util.Property("project"),
"--rebase",
util.Property("pullrequesturl"),
],
)
)
return util.BuilderConfig(
name="nix-eval",
workernames=worker_names,
factory=factory,
properties=dict(virtual_builder_name="nix-eval"),
)
def nix_update_flake_config(
worker_names: list[str], projectname: str, github_token_secret: str
) -> util.BuilderConfig:
"""
Updates the flake an opens a PR for it.
"""
factory = util.BuildFactory()
url_with_secret = util.Interpolate(
f"https://*****:*****@github.com/{projectname}"
)
factory.addStep(
steps.Git(
repourl=url_with_secret,
method="clean",
submodules=True,
haltOnFailure=True,
)
)
factory.addStep(
steps.ShellCommand(
name="Update flakes",
env=dict(
GIT_AUTHOR_NAME="buildbot",
GIT_AUTHOR_EMAIL="*****@*****.**",
GIT_COMMITTER_NAME="buildbot",
GIT_COMMITTER_EMAIL="*****@*****.**",
),
command=[
"nix",
"flake",
"update",
"--commit-lock-file",
"--commit-lockfile-summary",
"flake.lock: Update",
],
haltOnFailure=True,
)
)
factory.addStep(
steps.ShellCommand(
name="Force-Push to update_flake_lock branch",
command=[
"git",
"push",
"--force",
"origin",
"HEAD:refs/heads/update_flake_lock",
],
haltOnFailure=True,
)
)
factory.addStep(
CreatePr(
name="Create pull-request",
env=dict(GITHUB_TOKEN=util.Secret(github_token_secret)),
command=[
"gh",
"pr",
"create",
"--repo",
projectname,
"--title",
"flake.lock: Update",
"--body",
"Automatic buildbot update",
"--head",
"refs/heads/update_flake_lock",
"--base",
"master",
],
)
)
return util.BuilderConfig(
name="nix-update-flake",
workernames=worker_names,
factory=factory,
properties=dict(virtual_builder_name="nix-update-flake"),
)
def ros_testbuild(c, job_name, url, branch, distro, arch, rosdistro, machines,
othermirror, keys, source=True, locks=[]):
# Create a Job for Source
if source:
project_name = '_'.join([job_name, rosdistro, 'source_build'])
c['change_source'].append(
GitPoller(
repourl=url,
name=url,
branch=branch,
category=project_name,
pollAtLaunch=True,
)
)
c['schedulers'].append(
schedulers.SingleBranchScheduler(
name=project_name,
builderNames=[project_name,],
change_filter=util.ChangeFilter(category=project_name)
)
)
c['schedulers'].append(
schedulers.Nightly(
name = project_name+'-nightly-master',
codebases = {url:{'repository':url,'branch':'master'}},
builderNames = [project_name,],
hour=3,
minute=0,
)
)
c['schedulers'].append(
schedulers.Nightly(
name = project_name+'-nightly-develop',
codebases = {url:{'repository':url,'branch':'develop'}},
builderNames = [project_name,],
hour=5,
minute=0,
)
)
c['schedulers'].append(
schedulers.ForceScheduler(
name=project_name+'-force',
codebases = [util.CodebaseParameter("",
branch=util.ChoiceStringParameter(
name="branch",
choices=["master", "devel"],
default="master"),
repository=util.FixedParameter(name="repository", default=url),
)],
builderNames=[project_name,],
)
)
else:
r_owner, r_name = (url.split(':')[1])[:-4].split('/')
project_name = '_'.join([job_name, rosdistro, 'pr_build'])
c['change_source'].append(
GitPRPoller(
owner=r_owner,
repo=r_name,
category=project_name,
branches=[branch],
pollInterval=10*60,
pollAtLaunch=True,
token=util.Secret("OathToken"),
repository_type='ssh'
)
)
c['schedulers'].append(
schedulers.SingleBranchScheduler(
name=project_name,
builderNames=[project_name,],
change_filter=util.ChangeFilter(category=project_name)
)
)
# Directory which will be bind-mounted
binddir = '/tmp/'+project_name
dockerworkdir = '/tmp/test/'
f = BuildFactory()
# Remove any old crud in build/src folder
f.addStep(
ShellCommand(
name='rm src',
command=['rm', '-rf', 'build/src'],
hideStepIf=success,
workdir=Interpolate('%(prop:builddir)s')
)
)
# Check out repository (to /build/src)
f.addStep(
Git(
repourl=util.Property('repository', default=url),
branch=util.Property('branch', default=branch),
alwaysUseLatest=True,
mode='full',
workdir=Interpolate('%(prop:builddir)s/build/src')
)
)
# Download testbuild_docker.py script from master
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='scripts/testbuild_docker.py',
workerdest=('testbuild_docker.py'),
hideStepIf=success
)
)
# Download Dockerfile_test script from master
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='docker_components/Dockerfile_test',
workerdest=('Dockerfile_test'),
hideStepIf=success
)
)
# Download docker-compose.py script from master
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='docker_components/docker-compose-test.yaml',
workerdest=('docker-compose-test.yaml'),
hideStepIf=success
)
)
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='docker_components/rosdep_private.yaml',
workerdest=('rosdep_private.yaml'),
hideStepIf=success
)
)
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='scripts/docker-container.py',
workerdest=('docker-container.py'),
hideStepIf=success
)
)
# create docker work environment
f.addStep(
ShellCommand(
command=['python','docker-container.py', job_name],
hideStepIf=success,
workdir=Interpolate('%(prop:builddir)s/build/')
)
)
# Make and run tests in a docker container
f.addStep(
ShellCommand(
name=job_name+'-build',
command=['docker', 'run',
'-v', 'ros-buildbot-docker_deb_repository:/home/package',
'--name='+project_name,
'scalable-env:'+job_name,
'python', '/tmp/build/testbuild_docker.py', binddir,
rosdistro],
descriptionDone=['make and test', job_name]
)
)
f.addStep(
ShellCommand(
name=job_name+'-copytestresults',
command=['docker', 'cp', project_name + ':' +binddir + '/testresults',
'testresults'],
logfiles={'tests': 'testresults'},
descriptionDone=['testresults', job_name]
)
)
f.addStep(
ShellCommand(
name=job_name+'-rm_container',
command=['docker', 'rm', project_name],
descriptionDone=['remove docker container', job_name]
)
)
f.addStep(
ShellCommand(
name=job_name+'-rm_image',
command=['docker', 'image', 'rm', 'scalable-env:'+job_name],
descriptionDone=['remove docker image', job_name]
)
)
c['builders'].append(
BuilderConfig(
name=project_name,
workernames=machines,
factory=f,
locks=locks
)
)
# return the name of the job created
return project_name
