def addSimpleBuilder(self, name, buildername, category, repourl,
builderconfig, sos, sbranch, bparams):
"""Private.
Add a single builder on the given OS type for the given repo and branch.
"""
factory = BuildFactory()
factory.addStep(
Git(repourl=repourl,
mode='full',
submodules=True,
method='copy',
branch=sbranch,
getDescription={'tags': True}))
if "tag" in builderconfig and not (builderconfig["tag"] is None):
stag = builderconfig["tag"].encode('ascii', 'ignore')
factory.addStep(
ShellCommand(command=['git', 'checkout', stag],
workdir="build",
description="checkout tag"))
# Delegate all knowlege of how to build to a script called buildshim in the project
# directory. Call this script in nine standardize steps. It should ignore steps that it doesn't need.
# Pass the step name as the first arg, and if params was given in the json for this builder, pass that as the
# second arg.
for step in [
"patch", "install_deps", "configure", "compile", "check",
"package", "upload", "compile_extra", "uninstall_deps"
]:
factory.addStep(
ShellCommand(command=["./buildshim", step, bparams],
description=step,
haltOnFailure=True))
self['builders'].append(
BuilderConfig(name=buildername,
slavenames=self._os2slaves[sos],
factory=factory,
category=category))
return factory
def get_clone_step():
from buildbot.steps.source.git import Git
from buildbot.steps.master import SetProperty
from buildbot.process.properties import Interpolate
return [
Git(description=['cloning'],
descriptionDone=['clone'],
hideStepIf=skipped_or_success,
repourl=Interpolate('%(prop:repository)s'),
mode='full',
shallow=16,
method='clobber',
getDescription={'tags': True},
retry=(1, 120),
progress=True,
env={
'GIT_CURL_VERBOSE': '1',
'GIT_TRACE': '1'
},
logEnviron=False)
]
def _make_factory_step_generator(project_name, project_git_uri, make_command=None, workdir="/srv/buildbot"):
make_factory_steps = [
Git(
name = "Executing %s content fetch" % project_name,
repourl=project_git_uri,
mode='incremental'
),
ShellCommand(
name = "Executing %s: 'make %s'" % ( project_name, make_command ),
command = [
"make",
make_command
]
),
DirectoryUpload(
slavesrc="build",
masterdest=Interpolate(
"/srv/output/%(kw:project_name)s/%(src::branch)s",
)
)
]
return make_factory_steps
def installTwistedTrunk():
steps = []
steps.append(
Git(
repourl=TWISTED_GIT,
mode='full',
method='fresh',
codebase='twisted',
workdir='Twisted',
alwaysUseLatest=True,
))
steps.append(
ShellCommand(name='install-twisted-trunk',
description=['installing', 'twisted', 'trunk'],
descriptionDone=['install', 'twisted', 'trunk'],
command=[
virtualenvBinary('pip'), "install", "--no-index",
'--use-wheel', "-f",
"http://data.hybridcluster.net/python/", "."
],
workdir='Twisted',
haltOnFailure=True))
return steps
def getFactory(codebase, useSubmodules=True, mergeForward=False):
factory = BuildFactory()
repourl = GITHUB + b"/" + codebase
# check out the source
factory.addStep(
Git(repourl=repourl,
submodules=useSubmodules,
mode='full',
method='fresh',
codebase=codebase))
if mergeForward:
factory.addStep(MergeForward(repourl=repourl, codebase=codebase))
if useSubmodules:
# Work around http://trac.buildbot.net/ticket/2155
factory.addStep(
ShellCommand(command=["git", "submodule", "update", "--init"],
description=["updating", "git", "submodules"],
descriptionDone=["update", "git", "submodules"],
name="update-git-submodules"))
return factory
def bootstrap_builder(workers):
bootstrap_factory = BuildFactory()
bootstrap_factory.addStep(
steps.EveProperty(name='set the bootstrap build number',
property='bootstrap',
hideStepIf=util.hideStepIfSuccess,
value=Property('buildnumber')))
bootstrap_factory.addStep(
ShellCommand(name='check index.lock',
command='test ! -f .git/index.lock',
hideStepIf=util.hideStepIfSuccess,
haltOnFailure=True,
logEnviron=False))
bootstrap_factory.addStep(
Git(name='checkout git branch',
repourl=util.env.GIT_REPO,
retry=(60, 10),
submodules=True,
branch=Property('branch'),
mode='full',
method='fresh',
hideStepIf=util.hideStepIfSuccess,
haltOnFailure=True,
logEnviron=False))
bootstrap_factory.addStep(
steps.CancelNonTipBuild(
name='cancel builds for commits that are not branch tips',
logEnviron=False))
bootstrap_factory.addStep(
steps.EveProperty(name='set the master_builddir property',
property='master_builddir',
hideStepIf=util.hideStepIfSuccess,
value=Property('builddir')))
yaml_dirpath = dirname(util.env.PROJECT_YAML)
bootstrap_factory.addStep(
steps.EvePropertyFromCommand(
name='get the product version',
command=('./{}/get_product_version.sh 2> /dev/null'
' || echo 0.0.0'.format(yaml_dirpath)),
hideStepIf=util.hideStepIfSuccess,
property='product_version',
logEnviron=False))
bootstrap_factory.addStep(
ShellCommand(name='check the product version',
command=Interpolate(
r'echo %(prop:product_version)s |'
r' grep -qE "^[0-9]+(\.[0-9]+){1,3}$"'),
hideStepIf=util.hideStepIfSuccess,
haltOnFailure=True,
logEnviron=False))
# Read conf from yaml file
bootstrap_factory.addStep(
steps.ReadConfFromYaml(name='read %s' % util.env.PROJECT_YAML,
yaml=util.env.PROJECT_YAML))
return BuilderConfig(name=util.env.BOOTSTRAP_BUILDER_NAME,
workernames=[lw.name for lw in workers],
factory=bootstrap_factory,
canStartBuild=eve_canStartBuild,
nextBuild=util.nextBootstrapBuild,
properties={
'git_host': util.env.GIT_HOST,
'git_owner': util.env.GIT_OWNER,
'git_slug': util.env.GIT_SLUG,
'max_step_duration': util.env.MAX_STEP_DURATION,
'stage_name': 'bootstrap',
})
def ros_testbuild(c, job_name, url, branch, distro, arch, rosdistro, machines, othermirror, keys):
# Change source is simply a GitPoller
# TODO: make this configurable for svn/etc
c['change_source'].append(
NamedGitPoller(
repourl = url,
name = rosdistro,
branch = branch,
project = job_name+'_'+rosdistro+'_testbuild'
)
)
c['schedulers'].append(
basic.SingleBranchScheduler(
name = job_name+'_'+rosdistro+'_testbuild',
builderNames = [job_name+'_'+rosdistro+'_testbuild',],
change_filter = ChangeFilter(project = job_name+'_'+rosdistro+'_testbuild')
)
)
# Directory which will be bind-mounted
binddir = '/tmp/'+job_name+'_'+rosdistro+'_testbuild'
f = BuildFactory()
# Remove any old crud in /tmp folder
f.addStep(
ShellCommand(
command = ['rm', '-rf', binddir],
hideStepIf = success
)
)
# Check out repository (to /tmp)
f.addStep(
Git(
repourl = url,
branch = branch,
alwaysUseLatest = True,
mode = 'full',
workdir = binddir+'/src/'+job_name
)
)
# Download testbuild.py script from master
f.addStep(
FileDownload(
name = job_name+'-grab-script',
mastersrc = 'scripts/testbuild.py',
slavedest = Interpolate('%(prop:workdir)s/testbuild.py'),
hideStepIf = success
)
)
# Update the cowbuilder
f.addStep(
ShellCommand(
command = ['cowbuilder-update.py', distro, arch] + keys,
hideStepIf = success
)
)
# Make and run tests in a cowbuilder
f.addStep(
TestBuild(
name = job_name+'-build',
command = ['cowbuilder', '--execute', Interpolate('%(prop:workdir)s/testbuild.py'),
'--distribution', distro, '--architecture', arch,
'--bindmounts', binddir,
'--basepath', '/var/cache/pbuilder/base-'+distro+'-'+arch+'.cow',
'--override-config', '--othermirror', othermirror,
'--', binddir, rosdistro],
logfiles = {'tests' : binddir+'/testresults' },
env = {'DIST': distro},
descriptionDone = ['make and test', job_name]
)
)
c['builders'].append(
BuilderConfig(
name = job_name+'_'+rosdistro+'_testbuild',
slavenames = machines,
factory = f
)
)
# return the name of the job created
return job_name+'_'+rosdistro+'_testbuild'
"-v", Interpolate("%(prop:builddir)s/build/:/build/:rw"),
"-w", "/build/",
"-e", Interpolate("PYTHONPATH=/build/%s", common.outputdir),
"-e", Interpolate("LD_LIBRARY_PATH=/build/%s/lib", common.outputdir),
"-e", Interpolate("PATH=/build/%s/bin", common.outputdir),
Interpolate("%(prop:suite)s-%(prop:arch)s"),
setarch,
"/usr/bin/python3", "tests/build_samples.py"
]
changelog_msg = Interpolate("Automatic build %(prop:buildnumber)s by builder %(prop:buildername)s")
# Build steps shared by all builders.
build_steps = [
Git(config.git_url, getDescription={'match': 'v*'}),
# Patch a bug in 1.10.7
ShellCommand(command=["sed", "-i", "s/\"version\": SDK\\[\"PYTHONVERSION\"\\]\\[6:\\],/\"version\": SDK[\"PYTHONVERSION\"][6:].rstrip('dmu'),/", "makepanda/makepandacore.py"]),
# Decode the version number from the dtool/PandaVersion.pp file.
SetPropertyFromCommand("version", command=[
"python3", "makepanda/getversion.py", buildtype_flag],
haltOnFailure=True),
# Delete the built dir, if requested.
ShellCommand(name="clean", command=get_clean_command(),
haltOnFailure=False, doStepIf=lambda step:step.getProperty("clean", False)),
# These steps fill in properties used to determine upstream_version.
] + whl_version_steps + [
def ros_testbuild(c,
job_name,
url,
branch,
distro,
arch,
rosdistro,
machines,
othermirror,
keys,
token=None):
# Change source is either GitPoller or GitPRPoller
# TODO: make this configurable for svn/etc
project_name = ''
if token:
project_name = '_'.join([job_name, rosdistro, 'prtestbuild'])
c['change_source'].append(
GitPRPoller(
name=rosdistro + "_pr_poller",
repourl=url, # this may pose some problems
project=project_name,
token=token,
pollInterval=15))
# parse repo_url git@github:author/repo.git to repoOwner, repoName
r_owner, r_name = (url.split(':')[1])[:-4].split('/')
c['status'].append(
status.GitHubStatus(token=token,
repoOwner=r_owner,
repoName=r_name))
else:
project_name = '_'.join([job_name, rosdistro, 'testbuild'])
c['change_source'].append(
NamedGitPoller(repourl=url,
name=rosdistro,
branch=branch,
project=project_name))
c['schedulers'].append(
basic.SingleBranchScheduler(
name=project_name,
builderNames=[
project_name,
],
change_filter=ChangeFilter(project=project_name)))
# Directory which will be bind-mounted
binddir = '/tmp/' + project_name
f = BuildFactory()
# Remove any old crud in /tmp folder
f.addStep(ShellCommand(command=['rm', '-rf', binddir], hideStepIf=success))
# Check out repository (to /tmp)
f.addStep(
Git(repourl=util.Property('repository', default=url),
branch=util.Property('branch', default=branch),
alwaysUseLatest=True,
mode='full',
workdir=binddir + '/src/' + job_name))
# Download testbuild.py script from master
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='scripts/testbuild.py',
slavedest=Interpolate('%(prop:workdir)s/testbuild.py'),
hideStepIf=success))
# Update the cowbuilder
f.addStep(
ShellCommand(command=['cowbuilder-update.py', distro, arch] + keys,
hideStepIf=success))
# Make and run tests in a cowbuilder
f.addStep(
TestBuild(name=job_name + '-build',
command=[
'sudo', 'cowbuilder', '--execute',
Interpolate('%(prop:workdir)s/testbuild.py'),
'--distribution', distro, '--architecture', arch,
'--bindmounts', binddir, '--basepath',
'/var/cache/pbuilder/base-' + distro + '-' + arch +
'.cow', '--override-config', '--othermirror',
othermirror, '--', binddir, rosdistro
],
logfiles={'tests': binddir + '/testresults'},
descriptionDone=['make and test', job_name]))
c['builders'].append(
BuilderConfig(name=project_name, slavenames=machines, factory=f))
# return the name of the job created
return project_name
def ros_docbuild(c,
job_name,
url,
branch,
rosdistro,
machines,
trigger_pkgs=None):
# Directory which will be bind-mounted
binddir = job_name + '_' + rosdistro + '_docbuild'
f = BuildFactory()
# Remove any old crud in /tmp folder
f.addStep(ShellCommand(command=['rm', '-rf', binddir], hideStepIf=success))
# Check out repository (to /tmp)
f.addStep(
Git(repourl=url,
branch=branch,
alwaysUseLatest=True,
mode='full'
#workdir = binddir+'/src/'+job_name+'/'
))
# Download script from master
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='scripts/docbuild.py',
workerdest=Interpolate('%(prop:builddir)s/docbuild.py'),
hideStepIf=success))
f.addStep(
FileDownload(
name=job_name + '-grab-script',
mastersrc='scripts/unique_docker_doc.py',
workerdest=Interpolate('%(prop:builddir)s/unique_docker_doc.py'),
hideStepIf=success))
f.addStep(
FileDownload(
name=job_name + '-grab-script',
mastersrc='docker_components/Dockerfile_doc',
workerdest=Interpolate('%(prop:builddir)s/Dockerfile_doc'),
hideStepIf=success))
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='docker_components/docker-compose-doc.yaml',
workerdest=Interpolate(
'%(prop:builddir)s/docker-compose-doc.yaml'),
hideStepIf=success))
# reedit docker-compose-doc.yaml
f.addStep(
ShellCommand(
haltOnFailure=True,
name=job_name + '-reedit-docker-compose',
command=[
'python', 'unique_docker_doc.py',
Interpolate('%(prop:builddir)s/docker-compose-doc.yaml'),
Interpolate(job_name)
],
workdir=Interpolate('%(prop:builddir)s'),
descriptionDone=['reedit docker-compose', job_name]))
# Build docker image for creating doc
f.addStep(
ShellCommand(
# haltOnFailure = True,
name=job_name + '-create_docker',
command=[
'docker-compose', '-f',
Interpolate('%(prop:builddir)s/docker-compose-doc.yaml'),
'build'
],
workdir=Interpolate('%(prop:builddir)s'),
descriptionDone=['create_doc', job_name]))
# creating doc in docker
f.addStep(
ShellCommand(
# haltOnFailure=True,
name=job_name + '-create_doc',
command=[
'docker',
'run',
# '-v', 'ros-repository-docker_deb_repository:/home/package',
'--name',
Interpolate('doc_' + job_name),
Interpolate('scalable-doc:' + job_name),
'python',
'/root/docbuild.py',
'/tmp/',
rosdistro
],
descriptionDone=['create doc', job_name]))
f.addStep(
ShellCommand(name=job_name + '-copydocs',
command=[
'docker', 'cp',
Interpolate('doc_' + job_name + ':' + '/tmp/docs'),
'/docs'
],
workdir=Interpolate('%(prop:builddir)s'),
descriptionDone=['copydocs', job_name]))
# rm container
f.addStep(
ShellCommand(name=job_name + '-rm_container',
command=['docker', 'rm',
Interpolate('doc_' + job_name)],
descriptionDone=['remove docker container', job_name]))
# rm image
f.addStep(
ShellCommand(name=job_name + '-rm_image',
command=[
'docker', 'image', 'rm',
Interpolate('scalable-doc:' + job_name)
],
descriptionDone=['remove docker image', job_name]))
# Trigger if needed
if trigger_pkgs != None:
f.addStep(
Trigger(schedulerNames=[
t.replace('_', '-') + '-' + rosdistro + '-doctrigger'
for t in trigger_pkgs
],
waitForFinish=False,
alwaysRun=True))
# Create trigger
c['schedulers'].append(
triggerable.Triggerable(name=job_name.replace('_', '-') + '-' +
rosdistro + '-doctrigger',
builderNames=[
job_name + '_' + rosdistro + '_docbuild',
]))
# Add builder config
c['builders'].append(
BuilderConfig(name=job_name + '_' + rosdistro + '_docbuild',
workernames=machines,
factory=f))
# return the name of the job created
return job_name + '_' + rosdistro + '_docbuild'
def ros_docbuild(c, job_name, url, branch, distro, arch, rosdistro, machines, othermirror, keys, trigger_pkgs = None):
# Directory which will be bind-mounted
binddir = '/tmp/'+job_name+'_'+rosdistro+'_docbuild'
f = BuildFactory()
# Remove any old crud in /tmp folder
f.addStep(
ShellCommand(
command = ['rm', '-rf', binddir],
hideStepIf = success
)
)
# Check out repository (to /tmp)
f.addStep(
Git(
repourl = url,
branch = branch,
alwaysUseLatest = True,
mode = 'full',
workdir = binddir+'/src/'+job_name+'/'
)
)
# Download testbuild.py script from master
f.addStep(
FileDownload(
name = job_name+'-grab-script',
mastersrc = 'scripts/docbuild.py',
slavedest = Interpolate('%(prop:workdir)s/docbuild.py'),
hideStepIf = success
)
)
# Update the cowbuilder
f.addStep(
ShellCommand(
command = ['cowbuilder-update.py', distro, arch] + keys,
hideStepIf = success
)
)
# Build docs in a cowbuilder
f.addStep(
ShellCommand(
haltOnFailure = True,
name = job_name+'-docbuild',
command = ['cowbuilder', '--execute', Interpolate('%(prop:workdir)s/docbuild.py'),
'--distribution', distro, '--architecture', arch,
'--bindmounts', binddir,
'--basepath', '/var/cache/pbuilder/base-'+distro+'-'+arch+'.cow',
'--override-config', '--othermirror', othermirror,
'--', binddir, rosdistro],
env = {'DIST': distro},
descriptionDone = ['built docs', ]
)
)
# Upload docs to master
f.addStep(
DirectoryUpload(
name = job_name+'-upload',
slavesrc = binddir+'/docs',
masterdest = 'docs/' + rosdistro,
hideStepIf = success
)
)
# Trigger if needed
if trigger_pkgs != None:
f.addStep(
Trigger(
schedulerNames = [t.replace('_','-')+'-'+rosdistro+'-doctrigger' for t in trigger_pkgs],
waitForFinish = False,
alwaysRun=True
)
)
# Create trigger
c['schedulers'].append(
triggerable.Triggerable(
name = job_name.replace('_','-')+'-'+rosdistro+'-doctrigger',
builderNames = [job_name+'_'+rosdistro+'_docbuild',]
)
)
# Add builder config
c['builders'].append(
BuilderConfig(
name = job_name+'_'+rosdistro+'_docbuild',
slavenames = machines,
factory = f
)
)
# return the name of the job created
return job_name+'_'+rosdistro+'_docbuild'
# This little dictionary is passed to the ShellCommand constructor to tell the
# factory how it should handle different return codes
rc_dict = {
0 : SUCCESS,
2 : WARNINGS,
1 : FAILURE
}
# builders map directly to targets, but since targets don't have everything
for project in project_list.projects():
for target in project.targets():
fact = BuildFactory()
fact.addStep(Git(repourl = project.url(),
branch = target.branch(),
mode = "full",
method = "clobber"
)
)
for step in target.steps():
fact.addStep(ShellCommand(command=step['command'],
name="{0}".format(step['name']) if 'name' in step else 'shell',
alwaysRun = True if 'alwaysRun' in step else False,
haltOnFailure = True,
flunkOnWarnings = True if 'flunkOnWarnings' in step else False,
warnOnWarnings = True if 'warnOnWarnings' in step else False,
decodeRC = rc_dict,
timeout=(10 * 60 * 60)
)
)
def ros_debbuild(c,
job_name,
packages,
url,
distro,
arch,
rosdistro,
version,
machines,
othermirror,
keys,
trigger_pkgs=None):
gbp_args = [
'-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist=' + distro, '--git-arch=' + arch
]
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name=job_name + '-clean',
dir=Interpolate('%(prop:workdir)s'),
hideStepIf=success,
))
# Check out the repository master branch, since releases are tagged and not branched
f.addStep(
Git(
repourl=url,
branch='master',
alwaysUseLatest=
True, # this avoids broken builds when schedulers send wrong tag/rev
mode='full' # clean out old versions
))
# Update the cowbuilder
f.addStep(
ShellCommand(command=['cowbuilder-update.py', distro, arch] + keys,
hideStepIf=success))
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-' + rosdistro + '-' + package.replace(
'_', '-') # debian package name (ros-groovy-foo)
branch_name = 'debian/' + debian_pkg + '_%(prop:release_version)s_' + distro # release branch from bloom
deb_name = debian_pkg + '_%(prop:release_version)s' + distro
final_name = debian_pkg + '_%(prop:release_version)s-%(prop:datestamp)s' + distro + '_' + arch + '.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(haltOnFailure=True,
name=package + '-checkout',
command=[
'git', 'checkout',
Interpolate(branch_name), '--force'
],
hideStepIf=success))
# Download script for building the source deb
f.addStep(
FileDownload(
name=job_name + '-grab-build-source-deb-script',
mastersrc='scripts/build_source_deb.py',
slavedest=Interpolate('%(prop:workdir)s/build_source_deb.py'),
mode=0755,
hideStepIf=success))
def ros_debbuild(c,
job_name,
packages,
url,
distro,
arch,
rosdistro,
version,
machines,
othermirror,
keys,
trigger_pkgs=None):
gbp_args = [
'-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist=' + distro, '--git-arch=' + arch
]
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name=job_name + '-clean',
dir=Interpolate('%(prop:workdir)s'),
hideStepIf=success,
))
# Check out the repository master branch, since releases are tagged and not branched
f.addStep(
Git(
repourl=url,
branch='master',
alwaysUseLatest=
True, # this avoids broken builds when schedulers send wrong tag/rev
mode='full' # clean out old versions
))
# Update the cowbuilder
f.addStep(
ShellCommand(command=['cowbuilder-update.py', distro, arch] + keys,
hideStepIf=success))
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-' + rosdistro + '-' + package.replace(
'_', '-') # debian package name (ros-groovy-foo)
branch_name = 'debian/' + debian_pkg + '_%(prop:release_version)s_' + distro # release branch from bloom
deb_name = debian_pkg + '_%(prop:release_version)s' + distro
final_name = debian_pkg + '_%(prop:release_version)s-%(prop:datestamp)s' + distro + '_' + arch + '.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(haltOnFailure=True,
name=package + '-checkout',
command=[
'git', 'checkout',
Interpolate(branch_name), '--force'
],
hideStepIf=success))
# Build the source deb
f.addStep(
ShellCommand(haltOnFailure=True,
name=package + '-buildsource',
command=['git-buildpackage', '-S'] + gbp_args,
descriptionDone=['sourcedeb', package]))
# Upload sourcedeb to master (currently we are not actually syncing these with a public repo)
f.addStep(
FileUpload(
name=package + '-uploadsource',
slavesrc=Interpolate('%(prop:workdir)s/' + deb_name + '.dsc'),
masterdest=Interpolate('sourcedebs/' + deb_name + '.dsc'),
hideStepIf=success))
# Stamp the changelog, in a similar fashion to the ROS buildfarm
f.addStep(
SetPropertyFromCommand(command="date +%Y%m%d-%H%M-%z",
property="datestamp",
name=package + '-getstamp',
hideStepIf=success))
f.addStep(
ShellCommand(
haltOnFailure=True,
name=package + '-stampdeb',
command=[
'git-dch', '-a', '--ignore-branch', '--verbose', '-N',
Interpolate('%(prop:release_version)s-%(prop:datestamp)s' +
distro)
],
descriptionDone=[
'stamped changelog',
Interpolate('%(prop:release_version)s'),
Interpolate('%(prop:datestamp)s')
]))
# download hooks
f.addStep(
FileDownload(
name=package + '-grab-hooks',
mastersrc='hooks/D05deps',
slavedest=Interpolate('%(prop:workdir)s/hooks/D05deps'),
hideStepIf=success,
mode=0777 # make this executable for the cowbuilder
))
# build the binary from the git working copy
f.addStep(
ShellCommand(
haltOnFailure=True,
name=package + '-buildbinary',
command=[
'git-buildpackage', '--git-pbuilder', '--git-export=WC',
Interpolate('--git-export-dir=%(prop:workdir)s')
] + gbp_args,
env={
'DIST':
distro,
'GIT_PBUILDER_OPTIONS':
Interpolate(
'--hookdir %(prop:workdir)s/hooks --override-config'),
'OTHERMIRROR':
othermirror
},
descriptionDone=['binarydeb', package]))
# Upload binarydeb to master
f.addStep(
FileUpload(name=package + '-uploadbinary',
slavesrc=Interpolate('%(prop:workdir)s/' + final_name),
masterdest=Interpolate('binarydebs/' + final_name),
hideStepIf=success))
# Add the binarydeb using reprepro updater script on master
f.addStep(
MasterShellCommand(name=package + 'includedeb',
command=[
'reprepro-include.bash', debian_pkg,
Interpolate(final_name), distro, arch
],
descriptionDone=['updated in apt', package]))
# Trigger if needed
if trigger_pkgs != None:
f.addStep(
Trigger(schedulerNames=[
t.replace('_', '-') + '-' + rosdistro + '-' + distro + '-' +
arch + '-debtrigger' for t in trigger_pkgs
],
waitForFinish=False,
alwaysRun=True))
# Create trigger
c['schedulers'].append(
triggerable.Triggerable(
name=job_name.replace('_', '-') + '-' + rosdistro + '-' + distro +
'-' + arch + '-debtrigger',
builderNames=[
job_name + '_' + rosdistro + '_' + distro + '_' + arch +
'_debbuild',
]))
# Add to builders
c['builders'].append(
BuilderConfig(name=job_name + '_' + rosdistro + '_' + distro + '_' +
arch + '_debbuild',
properties={'release_version': version},
slavenames=machines,
factory=f))
# return name of builder created
return job_name + '_' + rosdistro + '_' + distro + '_' + arch + '_debbuild'
def getGlobalBuilders(self):
ret = list()
f = factory.BuildFactory()
f.useProgress = False
f.addStep(
Git(
mode="incremental",
workdir=".",
repourl=self.giturl,
branch=self.branch,
locks=[self.lock_src.access("exclusive")],
))
if len(self.PATCHES):
f.addStep(
steps.Patch(
patches=self.PATCHES,
workdir=".",
locks=[self.lock_src.access("exclusive")],
))
if self.nightly is not None:
# Trigger nightly scheduler to let it know the source stamp
f.addStep(
Trigger(name="Updating source stamp",
hideStepIf=(lambda r, s: r == results.SUCCESS),
schedulerNames=["nightly-{0}".format(self.name)]))
f.addStep(
Trigger(name="Building all platforms",
schedulerNames=[self.name],
copy_properties=['got_revision', 'clean', 'package'],
updateSourceStamp=True,
waitForFinish=True))
ret.append(
BuilderConfig(
name="fetch-{0}".format(self.name),
# This is specific
workername='fetcher',
workerbuilddir="/data/src/{0}".format(self.name),
factory=f,
tags=["fetch"],
))
if self.nightly is not None:
f = factory.BuildFactory()
f.addStep(
Trigger(schedulerNames=[self.name],
copy_properties=['got_revision'],
updateSourceStamp=True,
waitForFinish=True,
set_properties={
'clean': True,
'package': True
}))
ret.append(
BuilderConfig(
name="nightly-{0}".format(self.name),
# TODO: Fix this
workername='fetcher',
workerbuilddir="/data/triggers/nightly-{0}".format(
self.name),
factory=f,
tags=["nightly"],
locks=[self.lock_src.access("counting")]))
return ret
dayOfWeek=config.WEEKLY_TIME[2],
hour=config.WEEKLY_TIME[0],
minute=config.WEEKLY_TIME[1],
onlyIfChanged=True))
####### BUILDERS
from buildbot.process.factory import BuildFactory
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
from buildbot.steps import shell
from time import gmtime, strftime
from buildbot.process.properties import WithProperties, Interpolate
git_fetch_step = Git(repourl=config.REPOSITORY,
mode='incremental',
submodules=True,
timeout=120)
export_sdk_cmd = 'export ANDROID_HOME={};'.format(config.ANDROID_HOME)
export_sdk_step = ShellCommand(command=export_sdk_cmd)
make_step = ShellCommand(command=export_sdk_cmd + config.PRE_MAKE_STEP +
config.MAKE_STEP)
test_step = ShellCommand(command=export_sdk_cmd + config.TEST_STEP)
update_rev_step = shell.SetProperty(command='git rev-parse HEAD',
property='gitlastrev')
update_tag_step = shell.SetProperty(
command='git describe --tags | awk -F\'-\' \'{print $1}\'',
property='gitversion')
def ros_testbuild(c, job_name, url, branch, distro, arch, rosdistro, machines,
othermirror, keys, source=True, locks=[]):
# Create a Job for Source
if source:
project_name = '_'.join([job_name, rosdistro, 'source_build'])
c['change_source'].append(
GitPoller(
repourl=url,
name=url,
branch=branch,
category=project_name,
pollAtLaunch=True,
)
)
c['schedulers'].append(
schedulers.SingleBranchScheduler(
name=project_name,
builderNames=[project_name,],
change_filter=util.ChangeFilter(category=project_name)
)
)
c['schedulers'].append(
schedulers.Nightly(
name = project_name+'-nightly-master',
codebases = {url:{'repository':url,'branch':'master'}},
builderNames = [project_name,],
hour=3,
minute=0,
)
)
c['schedulers'].append(
schedulers.Nightly(
name = project_name+'-nightly-develop',
codebases = {url:{'repository':url,'branch':'develop'}},
builderNames = [project_name,],
hour=5,
minute=0,
)
)
c['schedulers'].append(
schedulers.ForceScheduler(
name=project_name+'-force',
codebases = [util.CodebaseParameter("",
branch=util.ChoiceStringParameter(
name="branch",
choices=["master", "devel"],
default="master"),
repository=util.FixedParameter(name="repository", default=url),
)],
builderNames=[project_name,],
)
)
else:
r_owner, r_name = (url.split(':')[1])[:-4].split('/')
project_name = '_'.join([job_name, rosdistro, 'pr_build'])
c['change_source'].append(
GitPRPoller(
owner=r_owner,
repo=r_name,
category=project_name,
branches=[branch],
pollInterval=10*60,
pollAtLaunch=True,
token=util.Secret("OathToken"),
repository_type='ssh'
)
)
c['schedulers'].append(
schedulers.SingleBranchScheduler(
name=project_name,
builderNames=[project_name,],
change_filter=util.ChangeFilter(category=project_name)
)
)
# Directory which will be bind-mounted
binddir = '/tmp/'+project_name
dockerworkdir = '/tmp/test/'
f = BuildFactory()
# Remove any old crud in build/src folder
f.addStep(
ShellCommand(
name='rm src',
command=['rm', '-rf', 'build/src'],
hideStepIf=success,
workdir=Interpolate('%(prop:builddir)s')
)
)
# Check out repository (to /build/src)
f.addStep(
Git(
repourl=util.Property('repository', default=url),
branch=util.Property('branch', default=branch),
alwaysUseLatest=True,
mode='full',
workdir=Interpolate('%(prop:builddir)s/build/src')
)
)
# Download testbuild_docker.py script from master
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='scripts/testbuild_docker.py',
workerdest=('testbuild_docker.py'),
hideStepIf=success
)
)
# Download Dockerfile_test script from master
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='docker_components/Dockerfile_test',
workerdest=('Dockerfile_test'),
hideStepIf=success
)
)
# Download docker-compose.py script from master
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='docker_components/docker-compose-test.yaml',
workerdest=('docker-compose-test.yaml'),
hideStepIf=success
)
)
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='docker_components/rosdep_private.yaml',
workerdest=('rosdep_private.yaml'),
hideStepIf=success
)
)
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='scripts/docker-container.py',
workerdest=('docker-container.py'),
hideStepIf=success
)
)
# create docker work environment
f.addStep(
ShellCommand(
command=['python','docker-container.py', job_name],
hideStepIf=success,
workdir=Interpolate('%(prop:builddir)s/build/')
)
)
# Make and run tests in a docker container
f.addStep(
ShellCommand(
name=job_name+'-build',
command=['docker', 'run',
'-v', 'ros-buildbot-docker_deb_repository:/home/package',
'--name='+project_name,
'scalable-env:'+job_name,
'python', '/tmp/build/testbuild_docker.py', binddir,
rosdistro],
descriptionDone=['make and test', job_name]
)
)
f.addStep(
ShellCommand(
name=job_name+'-copytestresults',
command=['docker', 'cp', project_name + ':' +binddir + '/testresults',
'testresults'],
logfiles={'tests': 'testresults'},
descriptionDone=['testresults', job_name]
)
)
f.addStep(
ShellCommand(
name=job_name+'-rm_container',
command=['docker', 'rm', project_name],
descriptionDone=['remove docker container', job_name]
)
)
f.addStep(
ShellCommand(
name=job_name+'-rm_image',
command=['docker', 'image', 'rm', 'scalable-env:'+job_name],
descriptionDone=['remove docker image', job_name]
)
)
c['builders'].append(
BuilderConfig(
name=project_name,
workernames=machines,
factory=f,
locks=locks
)
)
# return the name of the job created
return project_name
name='download customization configuration',
haltOnFailure=True,
command=[
'wget',
util.Interpolate(
'http://192.168.250.160:6543/build_order/%(prop:build_order_id)s/configuration'
), '-O',
util.Interpolate('%(prop:workdir)s\\customization_config.zip')
],
workdir=util.Interpolate('%(prop:workdir)s'))
# 1. check out the source
ngq_get_src_bld_step = Git(name='checkout ngq',
repourl=ngq_repourl,
branch=ngq_branch_for_everyday,
mode='full',
method='clobber',
submodules=True,
workdir='ngq_src',
timeout=1800)
ngq_get_src_rel_step = Git(
name='checkout ngq',
repourl=ngq_repourl,
branch=util.Interpolate('%(prop:release_ngq_branch)s'),
mode='full',
method='clobber',
submodules=True,
workdir='ngq_src',
timeout=1800)
installer_get_step = Git(name='checkout installer',
repourl=installer_repourl,
branch='4ngq_autobuild',
from buildbot.steps.source.git import Git
from buildbot.steps.shell import Compile, Test, ShellCommand
from buildbot.steps.slave import RemoveDirectory, MakeDirectory
from buildbot.steps.transfer import FileDownload
from buildbot.steps.python_twisted import Trial
from buildbot.steps.python import PyFlakes
from metabbotcfg.common import GIT_URL
from metabbotcfg.slaves import slaves, get_slaves, names
_PACKAGE_STASH = 'http://ftp.buildbot.net/pub/metabuildbot/python-packages/'
builders = []
# slaves seem to have a hard time fetching from github, so retry
gitStep = Git(repourl=GIT_URL, mode='full', method='fresh', retryFetch=True)
####### Custom Steps
# only allow one VirtualenvSetup to run on a slave at a time. This prevents
# collisions in the shared package cache.
veLock = locks.SlaveLock('veLock')
class VirtualenvSetup(ShellCommand):
def __init__(self, virtualenv_dir='sandbox', virtualenv_python='python',
virtualenv_packages=[], no_site_packages=False, **kwargs):
kwargs['locks'] = kwargs.get('locks', []) + [veLock.access('exclusive')]
ShellCommand.__init__(self, **kwargs)
self.virtualenv_dir = virtualenv_dir
self.virtualenv_python = virtualenv_python
# what steps, and which slaves can execute them. Note that any particular build will
# only take place on one slave.
from buildbot.process.factory import BuildFactory
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand, Configure, Compile
from buildbot.steps.master import MasterShellCommand
from buildbot.steps.transfer import DirectoryUpload
from buildbot.config import BuilderConfig
c['builders'] = []
f = BuildFactory()
# Check out the source.
f.addStep(Git(repourl="git://github.com/Forkk/forkk.net", mode="incremental"))
# f.addStep(ShellCommand(name="init-sandbox",
# command=["cabal", "sandbox", "init"],
# description=["initializing", "sandbox"],
# descriptionDone=["initialize", "sandbox"]))
# Build the site.
f.addStep(Compile(
name="compile",
command=["ghc", "--make", "-threaded", "site.hs"],
description=["compiling", "site", "builder"],
descriptionDone=["compile", "site", "builder"]
))
f.addStep(Compile(
name="build-site",
def createWindowsDevFactory():
f = BuildFactory()
f.addStep(
Git(
description="fetching sources",
descriptionDone="sources",
haltOnFailure=True,
repourl=repositoryUri,
mode='full',
method='clobber',
))
f.addStep(
ShellCommand(description="fetching packages",
descriptionDone="packages",
haltOnFailure=True,
command=["paket.exe", "restore"],
workdir=workingDirectory))
f.addStep(
SetPropertyFromCommand(description="setting version",
descriptionDone="version",
haltOnFailure=True,
command=[
"racket",
"c:\\build-tools\\patch-version.rkt", "-p",
"windows", "-v", "0.1.4", "-b",
Property("buildnumber")
],
property="buildPostfix",
workdir=workingDirectory))
f.addStep(
ShellCommand(description="building",
descriptionDone="build",
haltOnFailure=True,
command=["msbuild", "CorvusAlba.ToyFactory.Windows.sln"],
workdir=workingDirectory))
f.addStep(
ShellCommand(
description="archiving",
descriptionDone="archive",
haltOnFailure=True,
command=[
"tar", "-zcvf",
Interpolate("toy-factory-%(prop:buildPostfix)s.tar.gz"),
"../bin"
],
workdir=workingDirectory))
f.addStep(
FileUpload(
description="uploading",
descriptionDone="upload",
haltOnFailure=True,
mode=0644,
slavesrc=Interpolate("toy-factory-%(prop:buildPostfix)s.tar.gz"),
masterdest=Interpolate(
"~/builds/toy-factory-%(prop:buildPostfix)s.tar.gz"),
workdir=workingDirectory))
def createPoclFactory( environ={},
repository='https://github.com/pocl/pocl.git',
branch='master',
buildICD=True,
llvm_dir='/usr/',
icd_dir='/usr/',
tests_dir=None,
config_opts='',
pedantic=True,
tcedir='',
f=None,
cmake=False
):
"""
Create a buildbot factory object that builds pocl.
environ Dictionary: The environment variables to append to the build. PATH and
LD_LIBRARY_PATH will be added from llvm_dir (if given).
repository String: the repo to build from. defaults to pocl on github
branch String: the branch in 'repository' to build from. default to master
buildICD Bool: if false, the ICD extension is not built.
llvm_dir String: LLVM installation dir. I.e. without the 'bin/' or 'lib/'.
icd_dir String: ICD loader installation dir. We expect here to be a ICD loader that
understand the OCL_ICD_VENDORS parameter, i.e. ocl-icd or patched
Khronos loader.
tests_dir String: Path where the extenral testsuite packages can be copied from.
('cp' is used, so they need to be on the same filesystem).
NOTE: currently only a placeholder - not tested on the public buildbot
config_opts String: extra options to pass to ./configure
cmake String: use CMake instead of autotools to build pocl
"""
#multiple slaves that pend on lock seem to pend after they modified environ.
myenviron = environ.copy()
if 'PATH' in myenviron.keys():
myenviron['PATH'] = llvm_dir+"/bin/:"+myenviron['PATH']+":${PATH}"
else:
myenviron['PATH'] = llvm_dir+"/bin/:${PATH}"
if 'LD_LIBRARY_PATH' in myenviron.keys():
myenviron['LD_LIBRARY_PATH'] = llvm_dir+"/lib/:"+myenviron['PATH']+":${LD_LIBRARY_PATH}"
else:
myenviron['LD_LIBRARY_PATH'] = llvm_dir+"/lib/:${LD_LIBRARY_PATH}"
if tcedir:
myenviron['PATH'] = tcedir+"/bin/:"+myenviron['PATH']
myenviron['LD_LIBRARY_PATH'] = tcedir+"/lib/:"+myenviron['LD_LIBRARY_PATH']
if f==None:
f = factory.BuildFactory()
f.addStep(
Git(
repourl=repository,
mode=Property('git_mode'),
branch=branch )
)
if not cmake:
f.addStep(ShellCommand(
command=["./autogen.sh"],
haltOnFailure=True,
name="autoconfig",
env=myenviron,
description="autoconfiging",
descriptionDone="autoconf"))
if tests_dir!=None:
f.addStep(ShellCommand(
haltOnFailure=True,
command=["cp", "-u", tests_dir+AMD_test_pkg,
"examples/AMD/"+AMD_test_pkg],
name="copy AMD",
description="copying",
descriptionDone="copied AMD",
#kludge around 'cp' always complaining if source is missing
decodeRC={0:SUCCESS,1:SUCCESS}
))
f.addStep(ShellCommand(
haltOnFailure=False,
command=["cp", "-u", tests_dir+ViennaCL_test_pkg,
"examples/ViennaCL/"+ViennaCL_test_pkg],
name="copy ViennaCL",
description="copying",
descriptionDone="copied ViennaCL",
decodeRC={0:SUCCESS,1:SUCCESS}
))
if cmake:
f.addStep(
ShellCommand(
command=["cmake", "."],
env=myenviron,
haltOnFailure=True,
name="CMake",
description="cmaking",
descriptionDone="cmade"))
else:
configOpts=config_opts.split(' ')
if pedantic==True:
configOpts = configOpts + ['--enable-pedantic']
if buildICD==False:
configOpts = configOpts + ['--disable-icd']
f.addStep(ShellCommand(
command=["./configure"] + configOpts,
haltOnFailure=True,
name="configure pocl",
env=myenviron,
description="configureing",
descriptionDone="configure"))
f.addStep(Compile(env=myenviron ))
if tests_dir!=None and not cmake:
f.addStep(ShellCommand(command=["make", "prepare-examples"],
haltOnFailure=True,
name="prepare examples",
env=myenviron,
description="preparing",
descriptionDone="prepare"))
if tcedir:
f.addStep(ShellCommand(command=["./tools/scripts/run_tta_tests"],
haltOnFailure=True,
name="checks",
env=myenviron,
description="testing",
descriptionDone="tests",
logfiles={"test.log": "tests/testsuite.log"},
timeout=60*60))
else:
f.addStep(ShellCommand(command=["make", "check"],
haltOnFailure=True,
name="checks",
env=myenviron,
description="testing",
descriptionDone="tests",
logfiles={"test.log": "tests/testsuite.log"},
#blas3 alone takes 15-20 min.
timeout=60*60))
return f
#######
## LLVM/clang builder
##
# srcdir - LLVM source diectory
# builddir - LLVM build dir
# installdir - final LLVM install directory
# test_install_dir - the LLVM install dir pocl_build tests against
def createLLVMFactory(srcdir, builddir, installdir, test_install_dir):
f = factory.BuildFactory()
f.addStep(
SVN(
name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=srcdir))
f.addStep(
SVN(
name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % srcdir))
f.addStep(
ShellCommand(
command=[
'%s/configure' % srcdir,
'--prefix=' + installdir,
'--enable-optimized',
'--enable-targets=host',
'--enable-shared'],
workdir=builddir,
haltOnFailure=True,
name="configure",
descriptionDone='configure',
description='configuring'))
f.addStep(
ShellCommand(
command=['make', '-j', '4'],
workdir=builddir,
haltOnFailure=True,
name = "compile",
descriptionDone = 'compile',
description='compiling'))
f.addStep(
ShellCommand(
command=['make', 'check'],
workdir=builddir,
name='check',
descriptionDone='check',
haltOnFailure=True,
description='checking'))
f.addStep(
ShellCommand(
command=['make', 'install'],
env={'DESTDIR':test_install_dir},
workdir=builddir,
haltOnFailure=True,
name = 'install',
descriptionDone='install',
description='installing'))
f=createPoclFactory(
llvm_dir=test_install_dir+installdir,
f=f)
f.addStep(
ShellCommand(
command=['make', 'install'],
workdir=builddir,
haltOnFailure=True,
name = 'install',
descriptionDone='install',
description='installing'))
return f
def ros_branch_build(c, job_name, packages, url, branch, distro, arch, rosdistro, machines, othermirror, keys):
gbp_args = ['-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist='+distro, '--git-arch='+arch]
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name = job_name+'-clean',
dir = Interpolate('%(prop:workdir)s'),
hideStepIf = success,
)
)
# Pulling the repo
f.addStep(
Git(
repourl = url,
branch = 'HEAD',
alwaysUseLatest = True, # this avoids broken builds when schedulers send wrong tag/rev
mode = 'full', # clean out old versions
getDescription={'tags': True}
)
)
# Check out the repository branch/commit/tag
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'checkout: ' + branch,
command = ['git', 'checkout', branch],
)
)
# get the short commit hash
f.addStep(
SetPropertyFromCommand(
command="git rev-parse --short HEAD", property="commit_hash",
name = 'commit-short-hash',
hideStepIf = success
)
)
# get the time stamp
f.addStep(
SetPropertyFromCommand(
command='date +%Y%M%d%H', property='date_stamp',
name = 'date-stamp',
hideStepIf = success
)
)
# Update the cowbuilder
f.addStep(
ShellCommand(
command = ['cowbuilder-update.py', distro, arch] + keys,
hideStepIf = success
)
)
# Generate the changelog for the package
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'catkin_generate_changelog',
command= ['catkin_generate_changelog', '-y'],
descriptionDone = ['catkin_generate_changelog']
)
)
# Add all files including untracked ones
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'add_changelogs',
command= ['git', 'add', '.'],
descriptionDone = ['add_changelogs']
)
)
# Commit the changelog after updating it
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'update_changelogs',
command= ['git', 'commit', '-m', '\"Updated changelogs\"'],
descriptionDone = ['update_changelogs']
)
)
# Prepare the release without pushing it
# Set very big number to avoid conflicts with available tags
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'catkin_prepare_release',
command= ['catkin_prepare_release', '--version', '100.0.0', '--no-push', '-y'],
descriptionDone = ['catkin_prepare_release']
)
)
#
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'git_bloom_generate_release',
command = ['git-bloom-generate', '-y', 'rosrelease', rosdistro],
)
)
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'git_bloom_generate_debian',
command = ['git-bloom-generate', '-y', 'rosdebian', '-a', '-p', 'release', rosdistro],
)
)
# Get the tag number for the lastest commit
f.addStep(
SetPropertyFromCommand(
command="git describe --tags", property="release_version",
name = 'latest_tag',
)
)
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-'+rosdistro+'-'+package.replace('_','-') # debian package name (ros-groovy-foo)
branch_name = 'debian/'+debian_pkg+'_%(prop:release_version)s-0_'+distro
deb_name = debian_pkg+'_%(prop:release_version)s-0'+distro
final_name = debian_pkg+'_%(prop:release_version)s-%(prop:date_stamp)s-%(prop:commit_hash)s-'+distro+'_'+arch+'.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(
haltOnFailure = True,
name = package+'-checkout',
command = ['git', 'checkout', Interpolate(branch_name), '--force'],
hideStepIf = success
)
)
# Stamp the changelog
f.addStep(
ShellCommand(
haltOnFailure = True,
name = package+'-stampdeb',
command = ['gbp', 'dch', '-a', '--ignore-branch', '--verbose',
'-N', Interpolate('%(prop:release_version)s-%(prop:date_stamp)s-%(prop:commit_hash)s-'+distro)],
descriptionDone = ['stamped changelog', Interpolate('%(prop:release_version)s'),
Interpolate('%(prop:date_stamp)s'), Interpolate('%(prop:commit_hash)s')]
)
)
# download hooks
f.addStep(
FileDownload(
name = package+'-grab-hooks',
mastersrc = 'hooks/D05deps',
slavedest = Interpolate('%(prop:workdir)s/hooks/D05deps'),
hideStepIf = success,
mode = 0777 # make this executable for the cowbuilder
)
)
# Download script for building the binary deb
f.addStep(
FileDownload(
name = job_name+'-grab-build-binary-deb-script',
mastersrc = 'scripts/build_binary_deb.py',
slavedest = Interpolate('%(prop:workdir)s/build_binary_deb.py'),
mode = 0755,
hideStepIf = success
)
)
def ros_branch_build(c,
job_name,
packages,
url,
branch,
distro,
arch,
rosdistro,
machines,
othermirror,
keys,
trigger_pkgs=None):
gbp_args = [
'-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist=' + distro, '--git-arch=' + arch
]
with open(os.path.dirname(os.path.realpath(__file__)) +
"/spec.yaml") as file:
spec_list = yaml.full_load(file)
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name=job_name + '-clean',
dir=Interpolate('%(prop:workdir)s'),
hideStepIf=success,
))
# Check out the repository master branch, since releases are tagged and not branched
f.addStep(
Git(
repourl=url,
branch=branch,
alwaysUseLatest=
True, # this avoids broken builds when schedulers send wrong tag/rev
mode='full', # clean out old versions
getDescription={'tags': True}))
# Update the cowbuilder
f.addStep(
ShellCommand(command=['cowbuilder-update.py', distro, arch] + keys,
hideStepIf=success))
# Generate the changelog for the package
f.addStep(
ShellCommand(haltOnFailure=True,
name='catkin_generate_changelog',
command=['catkin_generate_changelog', '-y'],
descriptionDone=['catkin_generate_changelog']))
# Add all files including untracked ones
f.addStep(
ShellCommand(haltOnFailure=True,
name='add_changelogs',
command=['git', 'add', '.'],
descriptionDone=['add_changelogs']))
# Commit the changelog after updating it
f.addStep(
ShellCommand(haltOnFailure=True,
name='update_changelogs',
command=['git', 'commit', '-m', '\"Updated changelogs\"'],
descriptionDone=['update_changelogs']))
# Prepare the release without pushing it
f.addStep(
ShellCommand(haltOnFailure=True,
name='catkin_prepare_release',
command=[
'catkin_prepare_release', '--bump', 'minor',
'--no-push', '-y'
],
descriptionDone=['catkin_prepare_release']))
#
f.addStep(
ShellCommand(
haltOnFailure=True,
name='git_bloom_generate_release',
command=['git-bloom-generate', '-y', 'rosrelease', rosdistro],
))
f.addStep(
ShellCommand(
haltOnFailure=True,
name='git_bloom_generate_debian',
command=[
'git-bloom-generate', '-y', 'rosdebian', '-a', '-p', 'release',
rosdistro
],
))
# Get the tag number for the lastest commit
f.addStep(
SetPropertyFromCommand(
command="git describe --tags",
property="release_version",
name='latest_tag',
))
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-' + rosdistro + '-' + package.replace(
'_', '-') # debian package name (ros-groovy-foo)
branch_name = 'debian/' + debian_pkg + '_%(prop:release_version)s-0_' + distro
deb_name = debian_pkg + '_%(prop:release_version)s-0' + distro
final_name = debian_pkg + '_%(prop:release_version)s-%(prop:datestamp)s' + distro + '_' + arch + '.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(haltOnFailure=True,
name=package + '-checkout',
command=[
'git', 'checkout',
Interpolate(branch_name), '--force'
],
hideStepIf=success))
# A hack for generating the debian folder so we could build the lastest commit of the specified branch
# f.addStep(
# ShellCommand(
# haltOnFailure = True,
# name = package+'-bloom_generate',
# command= ['bloom-generate', 'rosdebian'],
# descriptionDone = ['bloom_generate', package]
# )
# )
# Download script for building the source deb
f.addStep(
FileDownload(
name=job_name + '-grab-build-source-deb-script',
mastersrc='scripts/build_source_deb.py',
slavedest=Interpolate('%(prop:workdir)s/build_source_deb.py'),
mode=0755,
hideStepIf=success))
def ros_debbuild(c,
job_name,
packages,
url,
distro,
arch,
rosdistro,
version,
machines,
othermirror,
keys,
trigger_pkgs=None,
locks=[]):
gbp_args = [
'-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist=' + distro, '--git-arch=' + arch
]
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name=job_name + '-clean',
dir=Interpolate('%(prop:builddir)s'),
hideStepIf=success,
))
# Check out the repository master branch, since releases are tagged and not branched
f.addStep(
Git(
repourl=url,
branch='master',
alwaysUseLatest=
True, # this avoids broken builds when schedulers send wrong tag/rev
mode='full' # clean out old versions
))
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-' + rosdistro + '-' + package.replace(
'_', '-') # debian package name (ros-groovy-foo)
branch_name = 'debian/' + debian_pkg + '_%(prop:release_version)s_' + distro # release branch from bloom debian/ros-groovy-foo_0.0.1_kinetic
deb_name = debian_pkg + '_%(prop:release_version)s' + distro
final_name = debian_pkg + '_%(prop:release_version)s' + distro + '_' + arch + '.deb'
# final_name = debian_pkg+'_%(prop:release_version)s-%(prop:datestamp)s'+distro+'_'+arch+'.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(haltOnFailure=True,
name=package + '-checkout',
command=[
'git', 'checkout',
Interpolate(branch_name), '--force'
],
hideStepIf=success))
# Download script for building the source deb
f.addStep(
FileDownload(name=job_name + '-grab-docker-compose-debian',
mastersrc='docker_components/docker-compose-deb.yaml',
workerdest=Interpolate(
'%(prop:builddir)s/docker-compose-deb.yaml'),
mode=0o755,
hideStepIf=success))
f.addStep(
FileDownload(
name=job_name + '-grab-dockerfile-debian',
mastersrc='docker_components/Dockerfile_deb',
workerdest=Interpolate('%(prop:builddir)s/Dockerfile_deb'),
mode=0o755,
hideStepIf=success))
f.addStep(
FileDownload(
name=job_name + '-grab-build-deb-shell',
mastersrc='shell/builddebian.sh',
workerdest=Interpolate('%(prop:builddir)s/builddebian.sh'),
mode=0o755,
hideStepIf=success))
f.addStep(
FileDownload(name=job_name + '-grab-rosdep-private',
mastersrc='docker_components/rosdep_private.yaml',
workerdest=Interpolate(
'%(prop:builddir)s/rosdep_private.yaml'),
mode=0o755,
hideStepIf=success))
f.addStep(
FileDownload(name=job_name + '-grab-rosdep-private',
mastersrc='scripts/unique_docker_deb.py',
workerdest=Interpolate(
'%(prop:builddir)s/unique_docker_deb.py'),
mode=0o755,
hideStepIf=success))
# reedit docker-compose-deb.yaml
f.addStep(
ShellCommand(
haltOnFailure=True,
name=package + '-reedit-docker-compose',
command=[
'python', 'unique_docker_deb.py',
Interpolate('%(prop:builddir)s/docker-compose-deb.yaml'),
Interpolate(package)
],
workdir=Interpolate('%(prop:builddir)s'),
descriptionDone=['reedit docker-compose', package]))
# Build docker image for creating debian
f.addStep(
ShellCommand(
#haltOnFailure = True,
name=package + '-buildsource',
command=[
'docker-compose', '-f',
Interpolate('%(prop:builddir)s/docker-compose-deb.yaml'),
'build'
],
workdir=Interpolate('%(prop:builddir)s'),
descriptionDone=['sourcedeb', package]))
# build debian package
f.addStep(
ShellCommand(
#haltOnFailure=True,
name=job_name + '-build',
command=[
'docker', 'run', '-v',
'ros-buildbot-docker_deb_repository:/home/package',
'--name',
Interpolate(package),
Interpolate('scalable-deb:' + package), 'bash',
'/usr/local/sbin/builddeb.sh'
],
descriptionDone=['build debian package', job_name]))
# update to local repository
f.addStep(
ShellCommand(name=job_name + '-upload',
command=[
'docker', 'exec', '-e',
Interpolate('package=' + debian_pkg + '*'),
'local-repository', 'bash',
'/tmp/debian-upload.sh'
],
descriptionDone=['release package', job_name]))
# rm container
f.addStep(
ShellCommand(name=job_name + '-rm_container',
command=['docker', 'rm',
Interpolate(package)],
descriptionDone=['remove docker container',
job_name]))
# rm image
f.addStep(
ShellCommand(name=job_name + '-rm_image',
command=[
'docker', 'image', 'rm',
Interpolate('scalable-deb:' + package)
],
descriptionDone=['remove docker image', job_name]))
# Trigger if needed
if trigger_pkgs != None:
f.addStep(
Trigger(schedulerNames=[
t.replace('_', '-') + '-' + rosdistro + '-' + distro + '-' +
arch + '-debtrigger' for t in trigger_pkgs
],
waitForFinish=False,
alwaysRun=True))
# Create trigger
c['schedulers'].append(
triggerable.Triggerable(
name=job_name.replace('_', '-') + '-' + rosdistro + '-' + distro +
'-' + arch + '-debtrigger',
builderNames=[
job_name + '_' + rosdistro + '_' + distro + '_' + arch +
'_debbuild',
]))
# Add to builders
c['builders'].append(
BuilderConfig(name=job_name + '_' + rosdistro + '_' + distro + '_' +
arch + '_debbuild',
properties={'release_version': version},
workernames=machines,
factory=f,
locks=locks))
# return name of builder created
return job_name + '_' + rosdistro + '_' + distro + '_' + arch + '_debbuild'
