def generateDBTestStep(dbname, dbport): mysqlString = "mysql -u root -h 127.0.0.1 -P " + dbport return common.shellSequence( commands=[ common.shellArg(command='echo "select version()" | ' + mysqlString, haltOnFailure=False, logname='version'), common.shellArg(command=util.Interpolate( 'echo "create database opencast%(prop:buildnumber)s;" | ' + mysqlString), haltOnFailure=False, logname='createdb'), common.shellArg(command=util.Interpolate( mysqlString + ' opencast%(prop:buildnumber)s < docs/scripts/ddl/mysql5.sql'), haltOnFailure=False, logname='newdb'), common.shellArg(command=util.Interpolate( 'echo "drop database opencast%(prop:buildnumber)s;" | ' + mysqlString), haltOnFailure=False, logname='dropdb'), ], workdir="build/", name="Test database generation script against " + dbname, haltOnFailure=False, flunkOnFailure=True, doStepIf=lambda step: int(step.getProperty("pkg_major_version")) < 9)
def __getBasePipeline(): checkSpaces = common.shellSequence(commands=[ common.shellArg(command=util.Interpolate( "(! grep -rnP '\t' modules assemblies pom.xml etc --include=pom.xml)" ), haltOnFailure=False, logname='Tab Check'), common.shellArg(command=util.Interpolate( "(! grep -rn ' $' modules assemblies pom.xml etc --include=pom.xml)" ), haltOnFailure=False, logname='End Of Line Space Check') ], workdir="build/docs/guides", name="Formatting checks") reports = [ 'site', 'site:stage', '-Daggregate=true', '-Dcheckstyle.skip=true', '-P', 'none,!frontend' ] site = common.getBuild(override=reports, name="Build site report") f_build = util.BuildFactory() f_build.addStep(common.getPreflightChecks()) f_build.addStep(common.getClone()) f_build.addStep(common.getWorkerPrep()) f_build.addStep(common.setTimezone()) f_build.addStep(common.setLocale()) f_build.addStep(common.getBuild()) f_build.addStep(checkSpaces) f_build.addStep(site) return f_build
def __getBasePipeline(): npm_install = common.shellSequence(commands=[ common.shellArg(command=['npm', 'install'], logname='npm_install'), ], workdir="build/docs/guides", name="Running npm install", haltOnFailure=True) npmCheck = common.shellSequence( commands=[ common.shellArg(command=['npm', 'test'], haltOnFailure=False, logname='markdown-cli'), ], workdir="build/docs/guides", name="Check Markdown doc formatting with markdown-cli", haltOnFailure=False) pip_install = common.shellSequence(commands=[ common.shellArg(command=[ 'python3', '-m', 'pip', 'install', '-r', 'requirements.txt' ], haltOnFailure=False, logname='markdown-cli'), ], workdir="build/docs/guides", name="Running pip install", haltOnFailure=True) build = common.shellCommand(command=['./.style-and-markdown-build.sh'], name="Running tests and building docs", env={ "LC_ALL": "en_US.UTF-8", "LANG": "en_US.UTF-8", "OC_CTYPE": "en_US.UTF-8", "PATH": "/builder/.local/bin:${PATH}" }, haltOnFailure=False, flunkOnFailure=True) markdown = GenerateMarkdownCommands(command='ls -d */', name="Determining available docs", workdir="build/docs/guides", haltOnFailure=True, flunkOnFailure=True) f_build = util.BuildFactory() f_build.addStep(common.getClone()) f_build.addStep(npm_install) f_build.addStep(npmCheck) f_build.addStep(pip_install) f_build.addStep(build) f_build.addStep(markdown) return f_build
def getRPMBuilds(props): builds = [] for profile in profiles[props.getProperty('branch_pretty')]: builds.append(common.shellArg( command=[ 'rpmbuild', '--define', 'ocdist ' + profile, '--define', util.Interpolate('tarversion %(prop:pkg_major_version)s-SNAPSHOT'), '-bb', '--noclean', 'opencast.spec' ], logname=profile)) builds.append(common.shellArg( command=[ 'rpmsign', '--addsign', '--key-id', util.Interpolate("%(prop:signing_key)s"), util.Interpolate( "../RPMS/noarch/opencast-" + profile + "-%(prop:rpm_version)s.el%(prop:el_version)s.noarch.rpm") ], logname=profile + " signing")) return builds
def generateDBUpgradeStep(dbname, dbport): mysqlString = "mysql -u root -h 127.0.0.1 -P " + dbport return common.shellSequence(commands=[ common.shellArg(command='bash docs/upgrade/.test.sh ' + dbport, haltOnFailure=False, logname=dbname), ], workdir="build/", name="Test database upgrade scripts against " + dbname, haltOnFailure=False, flunkOnFailure=True)
def getBuildPipeline(): rpmsClone = steps.Git( repourl="{{ source_rpm_repo_url }}", branch=util.Interpolate("%(prop:rpmspec_override:-%(prop:branch)s)s"), alwaysUseLatest=True, shallow=True, mode="full", method="clobber", flunkOnFailure=True, haltOnFailure=True, name="Cloning rpm packaging configs") rpmsVersion = steps.SetPropertyFromCommand( command="git rev-parse HEAD", property="rpm_script_rev", flunkOnFailure=True, warnOnFailure=True, haltOnFailure=True, workdir="build", name="Get rpm script revision") rpmsFullVersion = steps.SetProperty( property="rpm_version", value=util.Interpolate("%(prop:pkg_major_version)s.git%(prop:short_revision)s-%(prop:buildnumber)s")) rpmsSetup = common.shellSequence( commands=[ common.shellArg( # We're using a string here rather than an arg array since we need the shell functions command='echo -e "%_topdir `pwd`" > ~/.rpmmacros', logname="rpmdev-setup"), ], workdir="build/rpmbuild", name="Fetch built artifacts and build prep") rpmsFetch = common.syncAWS( pathFrom="s3://{{ s3_public_bucket }}/builds/{{ builds_fragment }}", pathTo="rpmbuild/SOURCES", name="Fetch build from S3") rpmsPrep = common.shellSequence( commands=[ common.shellArg( command=[ 'sed', '-i', util.Interpolate('s/define srcversion .*$/define srcversion %(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/g'), util.Interpolate('opencast.spec') ], logname='version'), common.shellArg( command=[ 'rpmdev-bumpspec', '-u', '"Buildbot <*****@*****.**>"', '-c', util.Interpolate( 'Opencast revision %(prop:got_revision)s, packaged with RPM scripts version %(prop:rpm_script_rev)s' ), util.Interpolate('opencast.spec') ], logname='rpmdev-bumpspec'), common.shellArg( command=[ 'sed', '-i', util.Interpolate("s/\(Version: *\) .*/\\1 %(prop:pkg_major_version)s.git%(prop:short_revision)s/"), util.Interpolate('opencast.spec') ], logname='version'), common.shellArg( command=[ 'sed', '-i', util.Interpolate('s/2%%{?dist}/%(prop:buildnumber)s%%{?dist}/g'), util.Interpolate('opencast.spec') ], logname='buildnumber'), common.shellArg( command=['rm', '-f', 'BUILD/opencast/build/revision.txt'], logname="cleanup") ], workdir="build/rpmbuild/SPECS", name="Prepping rpms") rpmsBuild = common.shellSequence( commands=getRPMBuilds, workdir="build/rpmbuild/SPECS", name="Build rpms") # Note: We're using a string here because using the array disables shell globbing! rpmsUpload = common.syncAWS( pathFrom="rpmbuild/RPMS/noarch", pathTo="s3://{{ s3_public_bucket }}/repo/rpms/unstable/el/%(prop:el_version)s/noarch/", name="Upload rpms to S3") rpmsPrune = common.shellCommand( command=util.Interpolate("ls -t /builder/s3/repo/rpms/unstable/el/%(prop:el_version)s/noarch | grep allinone | tail -n +6 | cut -f 4 -d '-' | while read version; do rm -f /builder/s3/repo/rpms/unstable/el/%(prop:el_version)s/noarch/*$version; done"), name=util.Interpolate("Pruning %(prop:pkg_major_version)s unstable repository")) repoMetadata = common.shellCommand( command=['createrepo', '.'], workdir=util.Interpolate("/builder/s3/repo/rpms/unstable/el/%(prop:el_version)s/noarch"), name="Building repository") f_package_rpms = util.BuildFactory() f_package_rpms.addStep(common.getPreflightChecks()) f_package_rpms.addStep(rpmsClone) f_package_rpms.addStep(rpmsVersion) f_package_rpms.addStep(common.getLatestBuildRevision()) f_package_rpms.addStep(common.getShortBuildRevision()) f_package_rpms.addStep(rpmsFullVersion) f_package_rpms.addStep(rpmsSetup) f_package_rpms.addStep(rpmsFetch) f_package_rpms.addStep(rpmsPrep) f_package_rpms.addStep(common.loadSigningKey()) f_package_rpms.addStep(rpmsBuild) f_package_rpms.addStep(common.unloadSigningKey()) f_package_rpms.addStep(rpmsUpload) f_package_rpms.addStep(common.deployS3fsSecrets()) f_package_rpms.addStep(common.mountS3fs()) f_package_rpms.addStep(rpmsPrune) f_package_rpms.addStep(repoMetadata) f_package_rpms.addStep(common.unmountS3fs()) f_package_rpms.addStep(common.cleanupS3Secrets()) f_package_rpms.addStep(common.getClean()) return f_package_rpms
def getBuildPipeline(): debsClone = steps.Git(repourl="{{ source_deb_repo_url }}", branch=util.Property('branch'), alwaysUseLatest=True, mode="full", method="fresh", flunkOnFailure=True, haltOnFailure=True, name="Cloning deb packaging configs") debsVersion = steps.SetPropertyFromCommand( command="git rev-parse HEAD", property="deb_script_rev", flunkOnFailure=True, haltOnFailure=True, workdir="build", name="Get Debian script revision") removeSymlinks = common.shellCommand( command=['rm', '-rf', 'binaries', 'outputs'], alwaysRun=True, name="Prep cloned repo for CI use") debsFetch = common.syncAWS( pathFrom="s3://{{ s3_public_bucket }}/builds/{{ builds_fragment }}", pathTo= "binaries/%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/", name="Fetch build from S3") debsBuild = common.shellSequence(commands=[ common.shellArg(command=[ 'dch', '--changelog', 'opencast/debian/changelog', '--newversion', util.Interpolate( '%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s-%(prop:buildnumber)s-%(prop:short_revision)s' ), '-b', '-D', 'unstable', '-u', 'low', '--empty', util.Interpolate( 'Opencast revision %(prop:got_revision)s, packaged with Debian scripts version %(prop:deb_script_rev)s' ) ], logname='dch'), common.shellArg(command=[ 'rm', '-f', util.Interpolate( "binaries/%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s/revision.txt" ) ], logname='cleanup'), common.shellArg(command=util.Interpolate( 'echo "source library.sh\ndoOpencast %(prop:pkg_major_version)s.%(prop:pkg_minor_version)s %(prop:branch)s %(prop:got_revision)s" | tee build.sh' ), logname='write'), common.shellArg(command=util.Interpolate( 'ln -s opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s.orig.tar.xz opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.%(prop:pkg_minor_version)s-%(prop:buildnumber)s.orig.tar.xz' ), logname='link'), common.shellArg(command=['bash', 'build.sh'], logname='build'), common.shellArg(command=util.Interpolate( 'echo "Opencast version %(prop:got_revision)s packaged with version %(prop:deb_script_rev)s" | tee outputs/%(prop:oc_commit)s/revision.txt' ), logname='revision') ], env={ "NAME": "Buildbot", "EMAIL": "buildbot@{{ groups['master'][0] }}", "SIGNING_KEY": util.Interpolate( "%(prop:signing_key)s") }, name="Build debs") debRepoClone = steps.Git(repourl="{{ source_deb_packaging_repo_url }}", branch="{{ deb_packaging_repo_branch }}", alwaysUseLatest=True, mode="full", method="fresh", flunkOnFailure=True, haltOnFailure=True, name="Cloning deb repo configs") debRepoLoadKeys = common.shellCommand(command=['./build-keys'], name="Loading signing keys") debRepoCreate = common.shellCommand( command=[ './create-branch', util.Interpolate("%(prop:pkg_major_version)s.x") ], name=util.Interpolate( "Ensuring %(prop:pkg_major_version)s.x repos exist")) debRepoIngest = common.shellCommand( command=[ './include-binaries', util.Interpolate("%(prop:pkg_major_version)s.x"), util.Interpolate("%(prop:repo_component)s"), util.Interpolate( "outputs/%(prop:revision)s/opencast-%(prop:pkg_major_version)s_%(prop:pkg_major_version)s.x-%(prop:buildnumber)s-%(prop:short_revision)s_amd64.changes" ) ], name=util.Interpolate( f"Adding build to %(prop:pkg_major_version)s.x-%(prop:repo_component)s" )) debRepoPrune = common.shellCommand( command=[ './clean-unstable-repo', util.Interpolate("%(prop:pkg_major_version)s.x") ], name=util.Interpolate( f"Pruning %(prop:pkg_major_version)s.x unstable repository")) debRepoPublish = common.shellCommand( command=[ "./publish-branch", util.Interpolate("%(prop:pkg_major_version)s.x"), util.Interpolate("%(prop:signing_key)s") ], name=util.Interpolate("Publishing %(prop:pkg_major_version)s.x"), env={ "AWS_ACCESS_KEY_ID": util.Secret("s3.public_access_key"), "AWS_SECRET_ACCESS_KEY": util.Secret("s3.public_secret_key") }) f_package_debs = util.BuildFactory() f_package_debs.addStep(common.getPreflightChecks()) f_package_debs.addStep(debsClone) f_package_debs.addStep(debsVersion) f_package_debs.addStep(common.getLatestBuildRevision()) f_package_debs.addStep(common.getShortBuildRevision()) f_package_debs.addStep(removeSymlinks) f_package_debs.addStep(debsFetch) f_package_debs.addStep(common.loadSigningKey()) f_package_debs.addStep(debsBuild) f_package_debs.addStep(debRepoClone) f_package_debs.addStep(debRepoLoadKeys) f_package_debs.addStep(common.deployS3fsSecrets()) f_package_debs.addStep(common.mountS3fs()) f_package_debs.addStep(debRepoCreate) f_package_debs.addStep(debRepoIngest) f_package_debs.addStep(debRepoPrune) f_package_debs.addStep(debRepoPublish) f_package_debs.addStep(common.unloadSigningKey()) f_package_debs.addStep(common.unmountS3fs()) f_package_debs.addStep(common.cleanupS3Secrets()) f_package_debs.addStep(common.getClean()) return f_package_debs