def run(self):
# run the command to get the list of targets
cmd = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
# if the command passes extract the list of stages
result = cmd.results()
if result == util.SUCCESS:
# create a ShellCommand for each stage and add them to the build
self.build.addStepsAfterCurrentStep([
common.shellCommand(
command=['ansible',
'-e', 'ansible_user={{ buildbot_user }}',
util.Interpolate(
'--private-key=%(prop:builddir)s/%(prop:deploy_env)s'),
'-i', util.Interpolate(
"{{ buildbot_config }}/envs/" + target),
'admin_node',
'-m', 'copy',
'-a', util.Interpolate(
'src={{ buildbot_config }}/opencast-ingest.sh dest=opencast-ingest.sh')],
name="Copy ingest script to " + target + " env",
haltOnFailure=False,
flunkOnFailure=True)
for target in self.extract_targets(self.observer.getStdout())
])
return result
def run(self):
# run the command to get the list of targets
cmd = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
# if the command passes extract the list of stages
result = cmd.results()
if result == util.SUCCESS:
# create a ShellCommand for each stage and add them to the build
self.build.addStepsAfterCurrentStep([
common.shellCommand(
command=["ansible",
util.Interpolate(
'--private-key=%(prop:builddir)s/%(prop:deploy_env)s'),
"-i", util.Interpolate(
"{{ buildbot_config }}/envs/" + target),
"admin_node",
"-m", "shell", "-a", "bash opencast-ingest.sh",
"--extra-vars", util.Interpolate(" ".join(params))],
name="Ingest media to " + target + " env",
haltOnFailure=False,
flunkOnFailure=True)
for target in self.extract_targets(self.observer.getStdout())
])
return result
def run(self):
# run the command to get the list of targets
cmd = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
# if the command passes extract the list of stages
result = cmd.results()
if result == util.SUCCESS:
# create a ShellCommand for each stage and add them to the build
self.build.addStepsAfterCurrentStep([
common.shellCommand(
command=['ansible-playbook',
'-b',
util.Interpolate(
'--private-key=%(prop:builddir)s/%(prop:deploy_env)s'),
'-i', util.Interpolate(
"{{ buildbot_config }}/envs/" + target),
'uninstall.yml', 'opencast.yml', 'reset.yml',
'--extra-vars', util.Interpolate(" ".join(params))],
name="Deploy Opencast to " + target + " env",
haltOnFailure=False,
flunkOnFailure=True)
for target in self.extract_targets(self.observer.getStdout())
])
return result
def __getBasePipeline():
npm_install = common.shellSequence(commands=[
common.shellArg(command=['npm', 'install'], logname='npm_install'),
],
workdir="build/docs/guides",
name="Running npm install",
haltOnFailure=True)
npmCheck = common.shellSequence(
commands=[
common.shellArg(command=['npm', 'test'],
haltOnFailure=False,
logname='markdown-cli'),
],
workdir="build/docs/guides",
name="Check Markdown doc formatting with markdown-cli",
haltOnFailure=False)
pip_install = common.shellSequence(commands=[
common.shellArg(command=[
'python3', '-m', 'pip', 'install', '-r', 'requirements.txt'
],
haltOnFailure=False,
logname='markdown-cli'),
],
workdir="build/docs/guides",
name="Running pip install",
haltOnFailure=True)
build = common.shellCommand(command=['./.style-and-markdown-build.sh'],
name="Running tests and building docs",
env={
"LC_ALL": "en_US.UTF-8",
"LANG": "en_US.UTF-8",
"OC_CTYPE": "en_US.UTF-8",
"PATH": "/builder/.local/bin:${PATH}"
},
haltOnFailure=False,
flunkOnFailure=True)
markdown = GenerateMarkdownCommands(command='ls -d */',
name="Determining available docs",
workdir="build/docs/guides",
haltOnFailure=True,
flunkOnFailure=True)
f_build = util.BuildFactory()
f_build.addStep(common.getClone())
f_build.addStep(npm_install)
f_build.addStep(npmCheck)
f_build.addStep(pip_install)
f_build.addStep(build)
f_build.addStep(markdown)
return f_build
def getBuildPipeline():
stampVersion = common.shellCommand(
command=util.Interpolate("echo '%(prop:got_revision)s' | tee revision.txt"),
name="Stamping the build")
updateBuild = common.copyAWS(
pathFrom="revision.txt",
pathTo="s3://{{ s3_public_bucket }}/builds/%(prop:branch_pretty)s/latest.txt",
name="Update latest build marker in S3")
updateCrowdin = common.shellCommand(
command=util.Interpolate("echo api_key: '%(secret:crowdin.key)s' >> .crowdin.yaml; echo crowdin --config .crowdin.yaml upload sources -b %(prop:branch)s"),
doStepIf={{ push_crowdin }},
hideStepIf={{ not push_crowdin }},
name="Update Crowdin translation keys")
f_build = __getBasePipeline()
f_build.addStep(common.getWorkerPrep())
def run(self):
# run the command to get the list of targets
cmd = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
# if the command passes extract the list of stages
result = cmd.results()
if result == util.SUCCESS:
# create a ShellCommand for each stage and add them to the build
self.build.addStepsAfterCurrentStep([
common.shellCommand(command=['mkdocs', 'build'],
name="Build " + target[:-1] + " docs",
workdir="build/docs/guides/" + target,
env={
"LC_ALL": "en_US.utf8",
"LANG": "en_US.utf8",
"OC_CTYPE": "en_US.utf8",
"PATH": "/builder/.local/bin:${PATH}"
},
haltOnFailure=False,
flunkOnFailure=True)
for target in self.extract_targets(self.observer.getStdout())
])
return result
def getBuildPipeline():
clone = steps.Git(repourl="{{ ansible_scripts_url }}",
branch=util.Property('branch'),
alwaysUseLatest=True,
mode="full",
method="fresh")
version = steps.SetPropertyFromCommand(
command="git rev-parse HEAD",
property="ansible_script_rev",
flunkOnFailure=True,
warnOnFailure=True,
haltOnFailure=True,
workdir="build",
name="Get ansible script revision")
deps = common.shellCommand(
command=['ansible-galaxy', 'install', '-r', 'requirements.yml'],
name="Installing Ansible dependencies")
secrets = common.copyAWS(
pathFrom="s3://{{ s3_private_bucket }}/{{ groups['master'][0] }}/env/%(prop:deploy_env)s",
pathTo="%(prop:builddir)s/%(prop:deploy_env)s",
name="Fetching deploy key")
permissions = common.shellCommand(
command=['chmod', '600', util.Interpolate("%(prop:builddir)s/%(prop:deploy_env)s")],
name="Fixing deploy key permissions")
install = GenerateInstallCommands(
command=util.Interpolate("ls {{ buildbot_config }}/envs/ | grep %(prop:deploy_env)s"),
name="Determining install targets",
haltOnFailure=True,
flunkOnFailure=True)
deploy = GenerateDeployCommands(
command=util.Interpolate("ls {{ buildbot_config }}/envs/ | grep %(prop:deploy_env)s"),
name="Determining deploy targets",
haltOnFailure=True,
flunkOnFailure=True)
sleep = common.shellCommand(
command=["sleep", "300"],
name="Sleeping to let Opencast finish starting up")
# We aren't using -u here because this is executing in the same directory as the checked out ansible scripts, which
# contains a group_vars/all.yml files specifying ansible_user
ingest = GenerateIngestCommands(
command=util.Interpolate("ls {{ buildbot_config }}/envs/ | grep %(prop:deploy_env)s"),
name="Determining ingest targets",
haltOnFailure=True,
flunkOnFailure=True)
cleanup = common.shellCommand(
command=['rm', '-rf',
util.Interpolate("%(prop:builddir)s/%(prop:deploy_env)s")],
alwaysRun=True,
name="Cleanup")
f_ansible = util.BuildFactory()
f_ansible.addStep(clone)
f_ansible.addStep(version)
f_ansible.addStep(deps)
f_ansible.addStep(secrets)
f_ansible.addStep(permissions)
f_ansible.addStep(install)
f_ansible.addStep(deploy)
f_ansible.addStep(sleep)
f_ansible.addStep(ingest)
f_ansible.addStep(cleanup)
return f_ansible
# -*- python -*-
# ex: set filetype=python:
from buildbot.plugins import util
import common
getBuildSize = common.shellCommand(
command=['du', '-ch'],
name='Getting current build dir size')
uploadTarballs = common.syncAWS(
pathFrom="build",
pathTo="s3://{{ s3_public_bucket }}/builds/{{ builds_fragment }}",
name="Upload build to S3")
def __getBasePipeline():
f_build = util.BuildFactory()
f_build.addStep(common.getPreflightChecks())
f_build.addStep(common.getClone())
f_build.addStep(common.setLocale())
f_build.addStep(common.setTimezone())
return f_build
def getPullRequestPipeline():
f_build = __getBasePipeline()
f_build.addStep(common.getWorkerPrep())
def getBuildPipeline():
rpmsClone = steps.Git(
repourl="{{ source_rpm_repo_url }}",
branch=util.Interpolate("%(prop:rpmspec_override:-%(prop:branch)s)s"),
alwaysUseLatest=True,
shallow=True,
mode="full",
method="clobber",
flunkOnFailure=True,
haltOnFailure=True,
name="Cloning rpm packaging configs")
rpmsVersion = steps.SetPropertyFromCommand(
command="git rev-parse HEAD",
property="rpm_script_rev",
flunkOnFailure=True,
warnOnFailure=True,
haltOnFailure=True,
workdir="build",
name="Get rpm script revision")
rpmsFullVersion = steps.SetProperty(
property="rpm_version",
value=util.Interpolate("%(prop:pkg_major_version)s.git%(prop:short_revision)s-%(prop:buildnumber)s"))
rpmsSetup = common.shellSequence(
commands=[
common.shellArg(
# We're using a string here rather than an arg array since we need the shell functions
command='echo -e "%_topdir `pwd`" > ~/.rpmmacros',
logname="rpmdev-setup"),
],
workdir="build/rpmbuild",
name="Fetch built artifacts and build prep")
rpmsFetch = common.syncAWS(
pathFrom="s3://{{ s3_public_bucket }}/builds/{{ builds_fragment }}",
pathTo="rpmbuild/SOURCES",
name="Fetch build from S3")
rpmsPrep = common.shellSequence(
commands=[
common.shellArg(
command=[
'sed',
'-i',
util.Interpolate('s/define srcversion .*$/define srcversion %(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/g'),
util.Interpolate('opencast.spec')
],
logname='version'),
common.shellArg(
command=[
'rpmdev-bumpspec',
'-u', '"Buildbot <*****@*****.**>"',
'-c',
util.Interpolate(
'Opencast revision %(prop:got_revision)s, packaged with RPM scripts version %(prop:rpm_script_rev)s'
),
util.Interpolate('opencast.spec')
],
logname='rpmdev-bumpspec'),
common.shellArg(
command=[
'sed',
'-i',
util.Interpolate("s/\(Version: *\) .*/\\1 %(prop:pkg_major_version)s.git%(prop:short_revision)s/"),
util.Interpolate('opencast.spec')
],
logname='version'),
common.shellArg(
command=[
'sed',
'-i',
util.Interpolate('s/2%%{?dist}/%(prop:buildnumber)s%%{?dist}/g'),
util.Interpolate('opencast.spec')
],
logname='buildnumber'),
common.shellArg(
command=['rm', '-f', 'BUILD/opencast/build/revision.txt'],
logname="cleanup")
],
workdir="build/rpmbuild/SPECS",
name="Prepping rpms")
rpmsBuild = common.shellSequence(
commands=getRPMBuilds,
workdir="build/rpmbuild/SPECS",
name="Build rpms")
# Note: We're using a string here because using the array disables shell globbing!
rpmsUpload = common.syncAWS(
pathFrom="rpmbuild/RPMS/noarch",
pathTo="s3://{{ s3_public_bucket }}/repo/rpms/unstable/el/%(prop:el_version)s/noarch/",
name="Upload rpms to S3")
rpmsPrune = common.shellCommand(
command=util.Interpolate("ls -t /builder/s3/repo/rpms/unstable/el/%(prop:el_version)s/noarch | grep allinone | tail -n +6 | cut -f 4 -d '-' | while read version; do rm -f /builder/s3/repo/rpms/unstable/el/%(prop:el_version)s/noarch/*$version; done"),
name=util.Interpolate("Pruning %(prop:pkg_major_version)s unstable repository"))
repoMetadata = common.shellCommand(
command=['createrepo', '.'],
workdir=util.Interpolate("/builder/s3/repo/rpms/unstable/el/%(prop:el_version)s/noarch"),
name="Building repository")
f_package_rpms = util.BuildFactory()
f_package_rpms.addStep(common.getPreflightChecks())
f_package_rpms.addStep(rpmsClone)
f_package_rpms.addStep(rpmsVersion)
f_package_rpms.addStep(common.getLatestBuildRevision())
f_package_rpms.addStep(common.getShortBuildRevision())
f_package_rpms.addStep(rpmsFullVersion)
f_package_rpms.addStep(rpmsSetup)
f_package_rpms.addStep(rpmsFetch)
f_package_rpms.addStep(rpmsPrep)
f_package_rpms.addStep(common.loadSigningKey())
f_package_rpms.addStep(rpmsBuild)
f_package_rpms.addStep(common.unloadSigningKey())
f_package_rpms.addStep(rpmsUpload)
f_package_rpms.addStep(common.deployS3fsSecrets())
f_package_rpms.addStep(common.mountS3fs())
f_package_rpms.addStep(rpmsPrune)
f_package_rpms.addStep(repoMetadata)
f_package_rpms.addStep(common.unmountS3fs())
f_package_rpms.addStep(common.cleanupS3Secrets())
f_package_rpms.addStep(common.getClean())
return f_package_rpms
def getBuildPipeline():
debsClone = steps.Git(repourl="{{ source_deb_repo_url }}",
branch=util.Property('branch'),
alwaysUseLatest=True,
mode="full",
method="fresh",
flunkOnFailure=True,
haltOnFailure=True,
name="Cloning deb packaging configs")
debsVersion = steps.SetPropertyFromCommand(
command="git rev-parse HEAD",
property="deb_script_rev",
flunkOnFailure=True,
haltOnFailure=True,
workdir="build",
name="Get Debian script revision")
removeSymlinks = common.shellCommand(
command=['rm', '-rf', 'binaries', 'outputs'],
alwaysRun=True,
name="Prep cloned repo for CI use")
debsFetch = common.syncAWS(
pathFrom="s3://{{ s3_public_bucket }}/builds/{{ builds_fragment }}",
pathTo=
"binaries/%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/",
name="Fetch build from S3")
debsBuild = common.shellSequence(commands=[
common.shellArg(command=[
'dch', '--changelog', 'opencast/debian/changelog', '--newversion',
util.Interpolate(
'%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s-%(prop:buildnumber)s-%(prop:short_revision)s'
), '-b', '-D', 'unstable', '-u', 'low', '--empty',
util.Interpolate(
'Opencast revision %(prop:got_revision)s, packaged with Debian scripts version %(prop:deb_script_rev)s'
)
],
logname='dch'),
common.shellArg(command=[
'rm', '-f',
util.Interpolate(
"binaries/%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/revision.txt"
)
],
logname='cleanup'),
common.shellArg(command=util.Interpolate(
'echo "source library.sh\ndoOpencast %(prop:pkg_major_version)s.%(prop:pkg_minor_version)s %(prop:branch)s %(prop:got_revision)s" | tee build.sh'
),
logname='write'),
common.shellArg(command=util.Interpolate(
'ln -s opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s.orig.tar.xz opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s-%(prop:buildnumber)s.orig.tar.xz'
),
logname='link'),
common.shellArg(command=['bash', 'build.sh'], logname='build'),
common.shellArg(command=util.Interpolate(
'echo "Opencast version %(prop:got_revision)s packaged with version %(prop:deb_script_rev)s" | tee outputs/%(prop:oc_commit)s/revision.txt'
),
logname='revision')
],
env={
"NAME":
"Buildbot",
"EMAIL":
"buildbot@{{ groups['master'][0] }}",
"SIGNING_KEY":
util.Interpolate(
"%(prop:signing_key)s")
},
name="Build debs")
debRepoClone = steps.Git(repourl="{{ source_deb_packaging_repo_url }}",
branch="{{ deb_packaging_repo_branch }}",
alwaysUseLatest=True,
mode="full",
method="fresh",
flunkOnFailure=True,
haltOnFailure=True,
name="Cloning deb repo configs")
debRepoLoadKeys = common.shellCommand(command=['./build-keys'],
name="Loading signing keys")
debRepoCreate = common.shellCommand(
command=[
'./create-branch',
util.Interpolate("%(prop:pkg_major_version)s.x")
],
name=util.Interpolate(
"Ensuring %(prop:pkg_major_version)s.x repos exist"))
debRepoIngest = common.shellCommand(
command=[
'./include-binaries',
util.Interpolate("%(prop:pkg_major_version)s.x"),
util.Interpolate("%(prop:repo_component)s"),
util.Interpolate(
"outputs/%(prop:revision)s/opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.x-%(prop:buildnumber)s-%(prop:short_revision)s_amd64.changes"
)
],
name=util.Interpolate(
f"Adding build to %(prop:pkg_major_version)s.x-%(prop:repo_component)s"
))
debRepoPrune = common.shellCommand(
command=[
'./clean-unstable-repo',
util.Interpolate("%(prop:pkg_major_version)s.x")
],
name=util.Interpolate(
f"Pruning %(prop:pkg_major_version)s.x unstable repository"))
debRepoPublish = common.shellCommand(
command=[
"./publish-branch",
util.Interpolate("%(prop:pkg_major_version)s.x"),
util.Interpolate("%(prop:signing_key)s")
],
name=util.Interpolate("Publishing %(prop:pkg_major_version)s.x"),
env={
"AWS_ACCESS_KEY_ID": util.Secret("s3.public_access_key"),
"AWS_SECRET_ACCESS_KEY": util.Secret("s3.public_secret_key")
})
f_package_debs = util.BuildFactory()
f_package_debs.addStep(common.getPreflightChecks())
f_package_debs.addStep(debsClone)
f_package_debs.addStep(debsVersion)
f_package_debs.addStep(common.getLatestBuildRevision())
f_package_debs.addStep(common.getShortBuildRevision())
f_package_debs.addStep(removeSymlinks)
f_package_debs.addStep(debsFetch)
f_package_debs.addStep(common.loadSigningKey())
f_package_debs.addStep(debsBuild)
f_package_debs.addStep(debRepoClone)
f_package_debs.addStep(debRepoLoadKeys)
f_package_debs.addStep(common.deployS3fsSecrets())
f_package_debs.addStep(common.mountS3fs())
f_package_debs.addStep(debRepoCreate)
f_package_debs.addStep(debRepoIngest)
f_package_debs.addStep(debRepoPrune)
f_package_debs.addStep(debRepoPublish)
f_package_debs.addStep(common.unloadSigningKey())
f_package_debs.addStep(common.unmountS3fs())
f_package_debs.addStep(common.cleanupS3Secrets())
f_package_debs.addStep(common.getClean())
return f_package_debs