def generateDBTestStep(dbname, dbport):
mysqlString = "mysql -u root -h 127.0.0.1 -P " + dbport
return common.shellSequence(
commands=[
common.shellArg(command='echo "select version()" | ' + mysqlString,
haltOnFailure=False,
logname='version'),
common.shellArg(command=util.Interpolate(
'echo "create database opencast%(prop:buildnumber)s;" | ' +
mysqlString),
haltOnFailure=False,
logname='createdb'),
common.shellArg(command=util.Interpolate(
mysqlString +
' opencast%(prop:buildnumber)s < docs/scripts/ddl/mysql5.sql'),
haltOnFailure=False,
logname='newdb'),
common.shellArg(command=util.Interpolate(
'echo "drop database opencast%(prop:buildnumber)s;" | ' +
mysqlString),
haltOnFailure=False,
logname='dropdb'),
],
workdir="build/",
name="Test database generation script against " + dbname,
haltOnFailure=False,
flunkOnFailure=True,
doStepIf=lambda step: int(step.getProperty("pkg_major_version")) < 9)
def __getBasePipeline():
checkSpaces = common.shellSequence(commands=[
common.shellArg(command=util.Interpolate(
"(! grep -rnP '\t' modules assemblies pom.xml etc --include=pom.xml)"
),
haltOnFailure=False,
logname='Tab Check'),
common.shellArg(command=util.Interpolate(
"(! grep -rn ' $' modules assemblies pom.xml etc --include=pom.xml)"
),
haltOnFailure=False,
logname='End Of Line Space Check')
],
workdir="build/docs/guides",
name="Formatting checks")
reports = [
'site', 'site:stage', '-Daggregate=true', '-Dcheckstyle.skip=true',
'-P', 'none,!frontend'
]
site = common.getBuild(override=reports, name="Build site report")
f_build = util.BuildFactory()
f_build.addStep(common.getPreflightChecks())
f_build.addStep(common.getClone())
f_build.addStep(common.getWorkerPrep())
f_build.addStep(common.setTimezone())
f_build.addStep(common.setLocale())
f_build.addStep(common.getBuild())
f_build.addStep(checkSpaces)
f_build.addStep(site)
return f_build
def __getBasePipeline():
npm_install = common.shellSequence(commands=[
common.shellArg(command=['npm', 'install'], logname='npm_install'),
],
workdir="build/docs/guides",
name="Running npm install",
haltOnFailure=True)
npmCheck = common.shellSequence(
commands=[
common.shellArg(command=['npm', 'test'],
haltOnFailure=False,
logname='markdown-cli'),
],
workdir="build/docs/guides",
name="Check Markdown doc formatting with markdown-cli",
haltOnFailure=False)
pip_install = common.shellSequence(commands=[
common.shellArg(command=[
'python3', '-m', 'pip', 'install', '-r', 'requirements.txt'
],
haltOnFailure=False,
logname='markdown-cli'),
],
workdir="build/docs/guides",
name="Running pip install",
haltOnFailure=True)
build = common.shellCommand(command=['./.style-and-markdown-build.sh'],
name="Running tests and building docs",
env={
"LC_ALL": "en_US.UTF-8",
"LANG": "en_US.UTF-8",
"OC_CTYPE": "en_US.UTF-8",
"PATH": "/builder/.local/bin:${PATH}"
},
haltOnFailure=False,
flunkOnFailure=True)
markdown = GenerateMarkdownCommands(command='ls -d */',
name="Determining available docs",
workdir="build/docs/guides",
haltOnFailure=True,
flunkOnFailure=True)
f_build = util.BuildFactory()
f_build.addStep(common.getClone())
f_build.addStep(npm_install)
f_build.addStep(npmCheck)
f_build.addStep(pip_install)
f_build.addStep(build)
f_build.addStep(markdown)
return f_build
def getRPMBuilds(props):
builds = []
for profile in profiles[props.getProperty('branch_pretty')]:
builds.append(common.shellArg(
command=[
'rpmbuild',
'--define', 'ocdist ' + profile,
'--define', util.Interpolate('tarversion %(prop:pkg_major_version)s-SNAPSHOT'),
'-bb', '--noclean',
'opencast.spec'
],
logname=profile))
builds.append(common.shellArg(
command=[
'rpmsign',
'--addsign',
'--key-id', util.Interpolate("%(prop:signing_key)s"),
util.Interpolate(
"../RPMS/noarch/opencast-" + profile + "-%(prop:rpm_version)s.el%(prop:el_version)s.noarch.rpm")
],
logname=profile + " signing"))
return builds
def generateDBUpgradeStep(dbname, dbport):
mysqlString = "mysql -u root -h 127.0.0.1 -P " + dbport
return common.shellSequence(commands=[
common.shellArg(command='bash docs/upgrade/.test.sh ' + dbport,
haltOnFailure=False,
logname=dbname),
],
workdir="build/",
name="Test database upgrade scripts against " +
dbname,
haltOnFailure=False,
flunkOnFailure=True)
def getBuildPipeline():
rpmsClone = steps.Git(
repourl="{{ source_rpm_repo_url }}",
branch=util.Interpolate("%(prop:rpmspec_override:-%(prop:branch)s)s"),
alwaysUseLatest=True,
shallow=True,
mode="full",
method="clobber",
flunkOnFailure=True,
haltOnFailure=True,
name="Cloning rpm packaging configs")
rpmsVersion = steps.SetPropertyFromCommand(
command="git rev-parse HEAD",
property="rpm_script_rev",
flunkOnFailure=True,
warnOnFailure=True,
haltOnFailure=True,
workdir="build",
name="Get rpm script revision")
rpmsFullVersion = steps.SetProperty(
property="rpm_version",
value=util.Interpolate("%(prop:pkg_major_version)s.git%(prop:short_revision)s-%(prop:buildnumber)s"))
rpmsSetup = common.shellSequence(
commands=[
common.shellArg(
# We're using a string here rather than an arg array since we need the shell functions
command='echo -e "%_topdir `pwd`" > ~/.rpmmacros',
logname="rpmdev-setup"),
],
workdir="build/rpmbuild",
name="Fetch built artifacts and build prep")
rpmsFetch = common.syncAWS(
pathFrom="s3://{{ s3_public_bucket }}/builds/{{ builds_fragment }}",
pathTo="rpmbuild/SOURCES",
name="Fetch build from S3")
rpmsPrep = common.shellSequence(
commands=[
common.shellArg(
command=[
'sed',
'-i',
util.Interpolate('s/define srcversion .*$/define srcversion %(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/g'),
util.Interpolate('opencast.spec')
],
logname='version'),
common.shellArg(
command=[
'rpmdev-bumpspec',
'-u', '"Buildbot <*****@*****.**>"',
'-c',
util.Interpolate(
'Opencast revision %(prop:got_revision)s, packaged with RPM scripts version %(prop:rpm_script_rev)s'
),
util.Interpolate('opencast.spec')
],
logname='rpmdev-bumpspec'),
common.shellArg(
command=[
'sed',
'-i',
util.Interpolate("s/\(Version: *\) .*/\\1 %(prop:pkg_major_version)s.git%(prop:short_revision)s/"),
util.Interpolate('opencast.spec')
],
logname='version'),
common.shellArg(
command=[
'sed',
'-i',
util.Interpolate('s/2%%{?dist}/%(prop:buildnumber)s%%{?dist}/g'),
util.Interpolate('opencast.spec')
],
logname='buildnumber'),
common.shellArg(
command=['rm', '-f', 'BUILD/opencast/build/revision.txt'],
logname="cleanup")
],
workdir="build/rpmbuild/SPECS",
name="Prepping rpms")
rpmsBuild = common.shellSequence(
commands=getRPMBuilds,
workdir="build/rpmbuild/SPECS",
name="Build rpms")
# Note: We're using a string here because using the array disables shell globbing!
rpmsUpload = common.syncAWS(
pathFrom="rpmbuild/RPMS/noarch",
pathTo="s3://{{ s3_public_bucket }}/repo/rpms/unstable/el/%(prop:el_version)s/noarch/",
name="Upload rpms to S3")
rpmsPrune = common.shellCommand(
command=util.Interpolate("ls -t /builder/s3/repo/rpms/unstable/el/%(prop:el_version)s/noarch | grep allinone | tail -n +6 | cut -f 4 -d '-' | while read version; do rm -f /builder/s3/repo/rpms/unstable/el/%(prop:el_version)s/noarch/*$version; done"),
name=util.Interpolate("Pruning %(prop:pkg_major_version)s unstable repository"))
repoMetadata = common.shellCommand(
command=['createrepo', '.'],
workdir=util.Interpolate("/builder/s3/repo/rpms/unstable/el/%(prop:el_version)s/noarch"),
name="Building repository")
f_package_rpms = util.BuildFactory()
f_package_rpms.addStep(common.getPreflightChecks())
f_package_rpms.addStep(rpmsClone)
f_package_rpms.addStep(rpmsVersion)
f_package_rpms.addStep(common.getLatestBuildRevision())
f_package_rpms.addStep(common.getShortBuildRevision())
f_package_rpms.addStep(rpmsFullVersion)
f_package_rpms.addStep(rpmsSetup)
f_package_rpms.addStep(rpmsFetch)
f_package_rpms.addStep(rpmsPrep)
f_package_rpms.addStep(common.loadSigningKey())
f_package_rpms.addStep(rpmsBuild)
f_package_rpms.addStep(common.unloadSigningKey())
f_package_rpms.addStep(rpmsUpload)
f_package_rpms.addStep(common.deployS3fsSecrets())
f_package_rpms.addStep(common.mountS3fs())
f_package_rpms.addStep(rpmsPrune)
f_package_rpms.addStep(repoMetadata)
f_package_rpms.addStep(common.unmountS3fs())
f_package_rpms.addStep(common.cleanupS3Secrets())
f_package_rpms.addStep(common.getClean())
return f_package_rpms
def getBuildPipeline():
debsClone = steps.Git(repourl="{{ source_deb_repo_url }}",
branch=util.Property('branch'),
alwaysUseLatest=True,
mode="full",
method="fresh",
flunkOnFailure=True,
haltOnFailure=True,
name="Cloning deb packaging configs")
debsVersion = steps.SetPropertyFromCommand(
command="git rev-parse HEAD",
property="deb_script_rev",
flunkOnFailure=True,
haltOnFailure=True,
workdir="build",
name="Get Debian script revision")
removeSymlinks = common.shellCommand(
command=['rm', '-rf', 'binaries', 'outputs'],
alwaysRun=True,
name="Prep cloned repo for CI use")
debsFetch = common.syncAWS(
pathFrom="s3://{{ s3_public_bucket }}/builds/{{ builds_fragment }}",
pathTo=
"binaries/%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/",
name="Fetch build from S3")
debsBuild = common.shellSequence(commands=[
common.shellArg(command=[
'dch', '--changelog', 'opencast/debian/changelog', '--newversion',
util.Interpolate(
'%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s-%(prop:buildnumber)s-%(prop:short_revision)s'
), '-b', '-D', 'unstable', '-u', 'low', '--empty',
util.Interpolate(
'Opencast revision %(prop:got_revision)s, packaged with Debian scripts version %(prop:deb_script_rev)s'
)
],
logname='dch'),
common.shellArg(command=[
'rm', '-f',
util.Interpolate(
"binaries/%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/revision.txt"
)
],
logname='cleanup'),
common.shellArg(command=util.Interpolate(
'echo "source library.sh\ndoOpencast %(prop:pkg_major_version)s.%(prop:pkg_minor_version)s %(prop:branch)s %(prop:got_revision)s" | tee build.sh'
),
logname='write'),
common.shellArg(command=util.Interpolate(
'ln -s opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s.orig.tar.xz opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s-%(prop:buildnumber)s.orig.tar.xz'
),
logname='link'),
common.shellArg(command=['bash', 'build.sh'], logname='build'),
common.shellArg(command=util.Interpolate(
'echo "Opencast version %(prop:got_revision)s packaged with version %(prop:deb_script_rev)s" | tee outputs/%(prop:oc_commit)s/revision.txt'
),
logname='revision')
],
env={
"NAME":
"Buildbot",
"EMAIL":
"buildbot@{{ groups['master'][0] }}",
"SIGNING_KEY":
util.Interpolate(
"%(prop:signing_key)s")
},
name="Build debs")
debRepoClone = steps.Git(repourl="{{ source_deb_packaging_repo_url }}",
branch="{{ deb_packaging_repo_branch }}",
alwaysUseLatest=True,
mode="full",
method="fresh",
flunkOnFailure=True,
haltOnFailure=True,
name="Cloning deb repo configs")
debRepoLoadKeys = common.shellCommand(command=['./build-keys'],
name="Loading signing keys")
debRepoCreate = common.shellCommand(
command=[
'./create-branch',
util.Interpolate("%(prop:pkg_major_version)s.x")
],
name=util.Interpolate(
"Ensuring %(prop:pkg_major_version)s.x repos exist"))
debRepoIngest = common.shellCommand(
command=[
'./include-binaries',
util.Interpolate("%(prop:pkg_major_version)s.x"),
util.Interpolate("%(prop:repo_component)s"),
util.Interpolate(
"outputs/%(prop:revision)s/opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.x-%(prop:buildnumber)s-%(prop:short_revision)s_amd64.changes"
)
],
name=util.Interpolate(
f"Adding build to %(prop:pkg_major_version)s.x-%(prop:repo_component)s"
))
debRepoPrune = common.shellCommand(
command=[
'./clean-unstable-repo',
util.Interpolate("%(prop:pkg_major_version)s.x")
],
name=util.Interpolate(
f"Pruning %(prop:pkg_major_version)s.x unstable repository"))
debRepoPublish = common.shellCommand(
command=[
"./publish-branch",
util.Interpolate("%(prop:pkg_major_version)s.x"),
util.Interpolate("%(prop:signing_key)s")
],
name=util.Interpolate("Publishing %(prop:pkg_major_version)s.x"),
env={
"AWS_ACCESS_KEY_ID": util.Secret("s3.public_access_key"),
"AWS_SECRET_ACCESS_KEY": util.Secret("s3.public_secret_key")
})
f_package_debs = util.BuildFactory()
f_package_debs.addStep(common.getPreflightChecks())
f_package_debs.addStep(debsClone)
f_package_debs.addStep(debsVersion)
f_package_debs.addStep(common.getLatestBuildRevision())
f_package_debs.addStep(common.getShortBuildRevision())
f_package_debs.addStep(removeSymlinks)
f_package_debs.addStep(debsFetch)
f_package_debs.addStep(common.loadSigningKey())
f_package_debs.addStep(debsBuild)
f_package_debs.addStep(debRepoClone)
f_package_debs.addStep(debRepoLoadKeys)
f_package_debs.addStep(common.deployS3fsSecrets())
f_package_debs.addStep(common.mountS3fs())
f_package_debs.addStep(debRepoCreate)
f_package_debs.addStep(debRepoIngest)
f_package_debs.addStep(debRepoPrune)
f_package_debs.addStep(debRepoPublish)
f_package_debs.addStep(common.unloadSigningKey())
f_package_debs.addStep(common.unmountS3fs())
f_package_debs.addStep(common.cleanupS3Secrets())
f_package_debs.addStep(common.getClean())
return f_package_debs