def getBuildPipeline():
compress = GenerateCompressionCommands(
command='ls -d */site',
name="Determining available docs for compression",
workdir="build/docs/guides",
haltOnFailure=True,
flunkOnFailure=True)
upload = GenerateS3Commands(command='ls -d */site',
name="Determining available docs for upload",
workdir="build/docs/guides",
haltOnFailure=True,
flunkOnFailure=True)
updateMarkdown = steps.MasterShellCommand(command=util.Interpolate(
"rm -f {{ deployed_markdown_symlink }} && ln -s {{ deployed_markdown }} {{ deployed_markdown_symlink }}"
),
flunkOnFailure=True,
name="Deploy Markdown")
f_build = __getBasePipeline()
f_build.addStep(compress)
f_build.addStep(upload)
#f_build.addStep(updateMarkdown)
f_build.addStep(common.getClean())
return f_build
def __getBasePipeline():
f_build = util.BuildFactory()
f_build.addStep(common.getClone())
f_build.addStep(generateDBTestStep("maria", "3307"))
f_build.addStep(generateDBUpgradeStep("maria", "3307"))
f_build.addStep(generateDBTestStep("mysql5.6", "3308"))
f_build.addStep(generateDBUpgradeStep("mysql5.6", "3308"))
f_build.addStep(generateDBTestStep("mysql5.7", "3309"))
f_build.addStep(generateDBUpgradeStep("mysql5.7", "3309"))
f_build.addStep(common.getClean())
return f_build
def getBuildPipeline():
github_release = GenerateGithubRelease(
release_tag=util.Interpolate("%(prop:branch)s"),
release_name=util.Interpolate("Opencast %(prop:branch)s"),
release_message=util.Interpolate("Changelog available at #TODO"),
haltOnFailure=True,
flunkOnFailure=True)
f_build = __getBasePipeline()
f_build.addStep(common.getWorkerPrep())
f_build.addStep(common.loadMavenSettings())
f_build.addStep(common.loadSigningKey())
f_build.addStep(common.getBuild(override=['install', 'nexus-staging:deploy', 'nexus-staging:release', '-P', 'release,none', '-s', 'settings.xml', '-DstagingProgressTimeoutMinutes=10'], timeout=600))
f_build.addStep(common.unloadSigningKey())
f_build.addStep(common.getTarballs())
#f_build.addStep(github_release)
f_build.addStep(common.unloadMavenSettings())
f_build.addStep(common.getClean())
return f_build
def getBuildPipeline():
compressSite = common.compressDir(dirToCompress="target/staging",
outputFile="target/site.tar.bz2")
compressCoverage = common.compressDir(
dirToCompress="target/site/cobertura",
outputFile="target/coverage.tar.bz2")
uploadSite = common.copyAWS(
pathFrom="target/site.tar.bz2",
pathTo="s3://{{ s3_public_bucket }}/builds/{{ reports_fragment }}",
name="Upload site report to S3")
uploadCoverage = common.copyAWS(
pathFrom="target/coverage.tar.bz2",
pathTo="s3://{{ s3_public_bucket }}/builds/{{ coverage_fragment }}",
name="Upload coverage report to S3")
updateSite = steps.MasterShellCommand(command=util.Interpolate(
"ln -s {{ deployed_reports }}/apidocs {{ deployed_javadocs }} && \
ln -s {{ deployed_reports }}/cobertura {{ deployed_coverage }} && \
rm -f {{ deployed_reports_symlink }} {{ deployed_javadocs_symlink }} {{ deployed_coverage_symlink }} && \
ln -s {{ deployed_reports }} {{ deployed_reports_symlink }} && \
ln -s {{ deployed_javadocs }} {{ deployed_javadocs_symlink }} && \
ln -s {{ deployed_coverage }} {{ deployed_coverage_symlink }}"),
flunkOnFailure=True,
name="Deploy Reports")
f_build = __getBasePipeline()
f_build.addStep(compressSite)
f_build.addStep(compressCoverage)
f_build.addStep(uploadSite)
f_build.addStep(uploadCoverage)
#f_build.addStep(updateSite)
f_build.addStep(common.getClean())
return f_build
def getPullRequestPipeline():
f_build = __getBasePipeline()
f_build.addStep(common.getClean())
return f_build
f_build.addStep(common.setTimezone())
return f_build
def getPullRequestPipeline():
f_build = __getBasePipeline()
f_build.addStep(common.getWorkerPrep())
f_build.addStep(common.getBuild())
f_build.addStep(getBuildSize)
{% if push_prs | default(False) %}
f_build.addStep(common.getTarballs())
f_build.addStep(uploadTarballs)
{% endif %}
f_build.addStep(common.getClean())
return f_build
def getBuildPipeline():
stampVersion = common.shellCommand(
command=util.Interpolate("echo '%(prop:got_revision)s' | tee revision.txt"),
name="Stamping the build")
updateBuild = common.copyAWS(
pathFrom="revision.txt",
pathTo="s3://{{ s3_public_bucket }}/builds/%(prop:branch_pretty)s/latest.txt",
name="Update latest build marker in S3")
def getBuildPipeline():
rpmsClone = steps.Git(
repourl="{{ source_rpm_repo_url }}",
branch=util.Interpolate("%(prop:rpmspec_override:-%(prop:branch)s)s"),
alwaysUseLatest=True,
shallow=True,
mode="full",
method="clobber",
flunkOnFailure=True,
haltOnFailure=True,
name="Cloning rpm packaging configs")
rpmsVersion = steps.SetPropertyFromCommand(
command="git rev-parse HEAD",
property="rpm_script_rev",
flunkOnFailure=True,
warnOnFailure=True,
haltOnFailure=True,
workdir="build",
name="Get rpm script revision")
rpmsFullVersion = steps.SetProperty(
property="rpm_version",
value=util.Interpolate("%(prop:pkg_major_version)s.git%(prop:short_revision)s-%(prop:buildnumber)s"))
rpmsSetup = common.shellSequence(
commands=[
common.shellArg(
# We're using a string here rather than an arg array since we need the shell functions
command='echo -e "%_topdir `pwd`" > ~/.rpmmacros',
logname="rpmdev-setup"),
],
workdir="build/rpmbuild",
name="Fetch built artifacts and build prep")
rpmsFetch = common.syncAWS(
pathFrom="s3://{{ s3_public_bucket }}/builds/{{ builds_fragment }}",
pathTo="rpmbuild/SOURCES",
name="Fetch build from S3")
rpmsPrep = common.shellSequence(
commands=[
common.shellArg(
command=[
'sed',
'-i',
util.Interpolate('s/define srcversion .*$/define srcversion %(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/g'),
util.Interpolate('opencast.spec')
],
logname='version'),
common.shellArg(
command=[
'rpmdev-bumpspec',
'-u', '"Buildbot <*****@*****.**>"',
'-c',
util.Interpolate(
'Opencast revision %(prop:got_revision)s, packaged with RPM scripts version %(prop:rpm_script_rev)s'
),
util.Interpolate('opencast.spec')
],
logname='rpmdev-bumpspec'),
common.shellArg(
command=[
'sed',
'-i',
util.Interpolate("s/\(Version: *\) .*/\\1 %(prop:pkg_major_version)s.git%(prop:short_revision)s/"),
util.Interpolate('opencast.spec')
],
logname='version'),
common.shellArg(
command=[
'sed',
'-i',
util.Interpolate('s/2%%{?dist}/%(prop:buildnumber)s%%{?dist}/g'),
util.Interpolate('opencast.spec')
],
logname='buildnumber'),
common.shellArg(
command=['rm', '-f', 'BUILD/opencast/build/revision.txt'],
logname="cleanup")
],
workdir="build/rpmbuild/SPECS",
name="Prepping rpms")
rpmsBuild = common.shellSequence(
commands=getRPMBuilds,
workdir="build/rpmbuild/SPECS",
name="Build rpms")
# Note: We're using a string here because using the array disables shell globbing!
rpmsUpload = common.syncAWS(
pathFrom="rpmbuild/RPMS/noarch",
pathTo="s3://{{ s3_public_bucket }}/repo/rpms/unstable/el/%(prop:el_version)s/noarch/",
name="Upload rpms to S3")
rpmsPrune = common.shellCommand(
command=util.Interpolate("ls -t /builder/s3/repo/rpms/unstable/el/%(prop:el_version)s/noarch | grep allinone | tail -n +6 | cut -f 4 -d '-' | while read version; do rm -f /builder/s3/repo/rpms/unstable/el/%(prop:el_version)s/noarch/*$version; done"),
name=util.Interpolate("Pruning %(prop:pkg_major_version)s unstable repository"))
repoMetadata = common.shellCommand(
command=['createrepo', '.'],
workdir=util.Interpolate("/builder/s3/repo/rpms/unstable/el/%(prop:el_version)s/noarch"),
name="Building repository")
f_package_rpms = util.BuildFactory()
f_package_rpms.addStep(common.getPreflightChecks())
f_package_rpms.addStep(rpmsClone)
f_package_rpms.addStep(rpmsVersion)
f_package_rpms.addStep(common.getLatestBuildRevision())
f_package_rpms.addStep(common.getShortBuildRevision())
f_package_rpms.addStep(rpmsFullVersion)
f_package_rpms.addStep(rpmsSetup)
f_package_rpms.addStep(rpmsFetch)
f_package_rpms.addStep(rpmsPrep)
f_package_rpms.addStep(common.loadSigningKey())
f_package_rpms.addStep(rpmsBuild)
f_package_rpms.addStep(common.unloadSigningKey())
f_package_rpms.addStep(rpmsUpload)
f_package_rpms.addStep(common.deployS3fsSecrets())
f_package_rpms.addStep(common.mountS3fs())
f_package_rpms.addStep(rpmsPrune)
f_package_rpms.addStep(repoMetadata)
f_package_rpms.addStep(common.unmountS3fs())
f_package_rpms.addStep(common.cleanupS3Secrets())
f_package_rpms.addStep(common.getClean())
return f_package_rpms
def getBuildPipeline():
debsClone = steps.Git(repourl="{{ source_deb_repo_url }}",
branch=util.Property('branch'),
alwaysUseLatest=True,
mode="full",
method="fresh",
flunkOnFailure=True,
haltOnFailure=True,
name="Cloning deb packaging configs")
debsVersion = steps.SetPropertyFromCommand(
command="git rev-parse HEAD",
property="deb_script_rev",
flunkOnFailure=True,
haltOnFailure=True,
workdir="build",
name="Get Debian script revision")
removeSymlinks = common.shellCommand(
command=['rm', '-rf', 'binaries', 'outputs'],
alwaysRun=True,
name="Prep cloned repo for CI use")
debsFetch = common.syncAWS(
pathFrom="s3://{{ s3_public_bucket }}/builds/{{ builds_fragment }}",
pathTo=
"binaries/%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/",
name="Fetch build from S3")
debsBuild = common.shellSequence(commands=[
common.shellArg(command=[
'dch', '--changelog', 'opencast/debian/changelog', '--newversion',
util.Interpolate(
'%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s-%(prop:buildnumber)s-%(prop:short_revision)s'
), '-b', '-D', 'unstable', '-u', 'low', '--empty',
util.Interpolate(
'Opencast revision %(prop:got_revision)s, packaged with Debian scripts version %(prop:deb_script_rev)s'
)
],
logname='dch'),
common.shellArg(command=[
'rm', '-f',
util.Interpolate(
"binaries/%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/revision.txt"
)
],
logname='cleanup'),
common.shellArg(command=util.Interpolate(
'echo "source library.sh\ndoOpencast %(prop:pkg_major_version)s.%(prop:pkg_minor_version)s %(prop:branch)s %(prop:got_revision)s" | tee build.sh'
),
logname='write'),
common.shellArg(command=util.Interpolate(
'ln -s opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s.orig.tar.xz opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s-%(prop:buildnumber)s.orig.tar.xz'
),
logname='link'),
common.shellArg(command=['bash', 'build.sh'], logname='build'),
common.shellArg(command=util.Interpolate(
'echo "Opencast version %(prop:got_revision)s packaged with version %(prop:deb_script_rev)s" | tee outputs/%(prop:oc_commit)s/revision.txt'
),
logname='revision')
],
env={
"NAME":
"Buildbot",
"EMAIL":
"buildbot@{{ groups['master'][0] }}",
"SIGNING_KEY":
util.Interpolate(
"%(prop:signing_key)s")
},
name="Build debs")
debRepoClone = steps.Git(repourl="{{ source_deb_packaging_repo_url }}",
branch="{{ deb_packaging_repo_branch }}",
alwaysUseLatest=True,
mode="full",
method="fresh",
flunkOnFailure=True,
haltOnFailure=True,
name="Cloning deb repo configs")
debRepoLoadKeys = common.shellCommand(command=['./build-keys'],
name="Loading signing keys")
debRepoCreate = common.shellCommand(
command=[
'./create-branch',
util.Interpolate("%(prop:pkg_major_version)s.x")
],
name=util.Interpolate(
"Ensuring %(prop:pkg_major_version)s.x repos exist"))
debRepoIngest = common.shellCommand(
command=[
'./include-binaries',
util.Interpolate("%(prop:pkg_major_version)s.x"),
util.Interpolate("%(prop:repo_component)s"),
util.Interpolate(
"outputs/%(prop:revision)s/opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.x-%(prop:buildnumber)s-%(prop:short_revision)s_amd64.changes"
)
],
name=util.Interpolate(
f"Adding build to %(prop:pkg_major_version)s.x-%(prop:repo_component)s"
))
debRepoPrune = common.shellCommand(
command=[
'./clean-unstable-repo',
util.Interpolate("%(prop:pkg_major_version)s.x")
],
name=util.Interpolate(
f"Pruning %(prop:pkg_major_version)s.x unstable repository"))
debRepoPublish = common.shellCommand(
command=[
"./publish-branch",
util.Interpolate("%(prop:pkg_major_version)s.x"),
util.Interpolate("%(prop:signing_key)s")
],
name=util.Interpolate("Publishing %(prop:pkg_major_version)s.x"),
env={
"AWS_ACCESS_KEY_ID": util.Secret("s3.public_access_key"),
"AWS_SECRET_ACCESS_KEY": util.Secret("s3.public_secret_key")
})
f_package_debs = util.BuildFactory()
f_package_debs.addStep(common.getPreflightChecks())
f_package_debs.addStep(debsClone)
f_package_debs.addStep(debsVersion)
f_package_debs.addStep(common.getLatestBuildRevision())
f_package_debs.addStep(common.getShortBuildRevision())
f_package_debs.addStep(removeSymlinks)
f_package_debs.addStep(debsFetch)
f_package_debs.addStep(common.loadSigningKey())
f_package_debs.addStep(debsBuild)
f_package_debs.addStep(debRepoClone)
f_package_debs.addStep(debRepoLoadKeys)
f_package_debs.addStep(common.deployS3fsSecrets())
f_package_debs.addStep(common.mountS3fs())
f_package_debs.addStep(debRepoCreate)
f_package_debs.addStep(debRepoIngest)
f_package_debs.addStep(debRepoPrune)
f_package_debs.addStep(debRepoPublish)
f_package_debs.addStep(common.unloadSigningKey())
f_package_debs.addStep(common.unmountS3fs())
f_package_debs.addStep(common.cleanupS3Secrets())
f_package_debs.addStep(common.getClean())
return f_package_debs