def hg_tag_buildout(self, options, cfg_tokens, manifest_dir):
"""Steps to retrieve the buildout dir as a Mercurial tag.
Useful for release/packaging oriented builds.
The tag name is read from build properties.
The clone is made outside of the main build/ directory, that must
stay pristine to test the produced packages.
See module docstring for signature and return values.
"""
if len(cfg_tokens) != 2:
raise ValueError("Wrong hgtag buildout specification: %r" % cfg_tokens)
url, conf_path = cfg_tokens
tag = Property('buildout-tag')
return conf_path, (FileDownload(mastersrc=os.path.join(
BUILD_UTILS_PATH, 'buildout_hg_dl.py'),
slavedest='buildout_hg_dl.py',
workdir='src',
haltOnFailure=True),
ShellCommand(
command=[
'python', 'buildout_hg_dl.py', '-t', 'tag', url,
tag
],
workdir='./src',
description=(
"Retrieve buildout",
"tag",
tag,
"from hg",
),
haltOnFailure=True,
))
def hg_buildout(self, options, cfg_tokens, manifest_dir):
"""Steps to retrieve the buildout using Mercurial.
See module docstring for signature and return values.
manifest_dir is not used in this downloader.
"""
if len(cfg_tokens) != 3:
raise ValueError("Wrong standalong buildout specification: %r" %
cfg_tokens)
url, branch, conf_path = cfg_tokens
return conf_path, (FileDownload(mastersrc=os.path.join(
BUILD_UTILS_PATH, 'buildout_hg_dl.py'),
slavedest='buildout_hg_dl.py',
haltOnFailure=True),
ShellCommand(
command=[
'python', 'buildout_hg_dl.py', url, branch
],
description=(
"Retrieve buildout",
"from hg",
),
haltOnFailure=True,
))
def mkdocsfactory():
f = factory.BuildFactory()
f.addSteps([
gitStep,
FileDownload(mastersrc="virtualenv.py",
slavedest="virtualenv.py",
flunkOnFailure=True),
# run docs tools in their own virtualenv, otherwise we end up documenting
# the version of Buildbot running the metabuildbot!
VirtualenvSetup(name='virtualenv setup',
virtualenv_packages=[
'sphinx==1.2.2', 'Pygments==2.0.1',
'--editable=master[docs]', '--editable=worker'
],
virtualenv_dir='sandbox',
haltOnFailure=True),
# manual
ShellCommand(command=Interpolate(
textwrap.dedent("""\
export VERSION=latest &&
. sandbox/bin/activate &&
gmake docs
""")),
name="create docs"),
])
return f
def steps_odoo_port_reservation(configurator, options, environ=()):
"""Return steps for port reservation.
The chosen port is stored in ``openerp_port`` property.
Available manifest file options:
:odoo.http-port-min: minimal value for the HTTP port (defaults to 6069)
:odoo.http-port-max: maximal value for the HTTP port (defaults to 7069)
:odoo.http-port-step: increment value for the HTTP port (defaults to 5)
"""
return (
FileDownload(
mastersrc=os.path.join(BUILD_UTILS_PATH, 'port_reserve.py'),
slavedest='port_reserve.py'),
SetPropertyFromCommand(
property='openerp_port',
description=['Port', 'reservation'],
locks=[port_lock.access('exclusive')],
command=[
'python', 'port_reserve.py',
'--port-min=' + options.get('odoo.http-port-min', '6069'),
'--port-max=' + options.get('odoo.http-port-max', '7068'),
'--step=' + options.get('odoo.http-port-step', '5'),
])
)
def upload_benchmarker(self):
self.addStep(FileDownload(Interpolate('%s/benchmarker.tar.gz' % MASTERWORKDIR), 'benchmarker.tar.gz', workdir='.'))
self.addStep(ShellCommand(name='md5', command=['md5sum', 'benchmarker.tar.gz'], workdir='.'))
self.addStep(ShellCommand(name='unpack_benchmarker', command=['tar', 'xf', 'benchmarker.tar.gz'], workdir='.'))
self.addStep(ShellCommand(name='debug2', command=['ls', '-lha', 'benchmarker'], workdir='.'))
self.addStep(MasterShellCommand(name="cleanup", command=['rm', '-rf', Interpolate(MASTERWORKDIR)]))
def bzr_buildout(self, options, cfg_tokens, manifest_dir, subdir=None):
"""Steps to retrieve the buildout using Bazaar.
See module docstring for signature and return values.
manifest_dir is not used in this downloader.
:param subdir: if not ``None``, then branch will be set aside and
the default workdir, 'build' will be set as a link
to the specified subdir in branch.
"""
def conf_error(cfg_tokens):
raise ValueError("Wrong bzr buildout specification: %r" % cfg_tokens)
subdir = None
if len(cfg_tokens) > 2:
options = cfg_tokens[2:]
for opt in options:
split = opt.split('=')
if split[0].strip() == 'subdir':
subdir = split[1].strip()
else:
conf_error(cfg_tokens)
if len(cfg_tokens) < 2:
conf_error(cfg_tokens)
url, conf_path = cfg_tokens[:2]
steps = [
FileDownload(mastersrc=os.path.join(BUILD_UTILS_PATH,
'buildout_bzr_dl.py'),
slavedest='buildout_bzr_dl.py',
haltOnFailure=True),
]
if subdir is None:
steps.append(
ShellCommand(
command=['python', 'buildout_bzr_dl.py', url],
description=(
"Retrieve buildout",
"from bzr",
),
haltOnFailure=True,
))
else:
steps.append(
ShellCommand(
command=[
'python', 'build/buildout_bzr_dl.py', url, '--subdir',
subdir, '--subdir-target', 'build', '--force-remove-subdir'
],
description=(
"Retrieve buildout",
"from bzr",
),
haltOnFailure=True,
workdir='.',
))
return conf_path, steps
def steps_unibootstrap(self,
buildout_slave_path,
options,
eggs_cache,
dump_options_to=None,
**step_kw):
"""return a list of steps for buildout bootstrap, using uniform script.
The uniform script is ``unibootstrap.py``. For now it ships with
build_utils and is downloaded from the buildmaster.
options prefixed with 'bootstrap-' are applied
:param dump_options_to: kept for backwards compatibility,
(unibootstrap will dump them in all cases).
:param step_kw: will be passed to the step constructor. Known use-case:
change workdir in packaging step.
"""
boot_opts = {}
if options.get('virtualenv', 'true').strip().lower() == 'true':
boot_opts['--python'] = Interpolate(
'%(prop:cap_python_venv:-~/openerp-env)s'
'/bin/python')
bv = options.get('bootstrap-version')
if bv is not None:
boot_opts['--buildout-version'] = bv.strip()
command = [
Property('cap_python_bin',
default='python'), 'unibootstrap.py', '--dists-directory',
WithProperties(eggs_cache), '--buildout-config',
buildout_slave_path
]
if dump_options_to is None:
command.append('--no-output-bootstrap-config')
else:
boot_opts['--output-bootstrap-config'] = dump_options_to
for o, v in boot_opts.items():
command.extend((o, v))
command.append('.')
return [
FileDownload(mastersrc=os.path.join(BUILD_UTILS_PATH,
'unibootstrap.py'),
slavedest='unibootstrap.py',
name="download",
description=['download', 'unibootstrap'],
**step_kw),
ShellCommand(command=command,
name='bootstrap',
description="bootstrapping",
descriptionDone="bootstrapped",
locks=[buildout_caches_lock.access('exclusive')],
haltOnFailure=True,
**step_kw)
]
def upload_sgen_grep_binprot(self):
self.addStep(
FileDownload(MONO_SGEN_GREP_BINPROT_FILENAME,
'sgen-grep-binprot',
workdir='benchmarker'))
self.addStep(
ShellCommand(name='chmod',
command=['chmod', 'a+x', 'sgen-grep-binprot'],
workdir='benchmarker'))
def ros_debbuild(c, job_name, packages, url, distro, arch, rosdistro, version, machines, othermirror, keys, trigger_pkgs = None):
gbp_args = ['-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist='+distro, '--git-arch='+arch]
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name = job_name+'-clean',
dir = Interpolate('%(prop:workdir)s'),
hideStepIf = success,
)
)
# Check out the repository master branch, since releases are tagged and not branched
f.addStep(
Git(
repourl = url,
branch = 'master',
alwaysUseLatest = True, # this avoids broken builds when schedulers send wrong tag/rev
mode = 'full' # clean out old versions
)
)
# Update the cowbuilder
f.addStep(
ShellCommand(
command = ['cowbuilder-update.py', distro, arch] + keys,
hideStepIf = success
)
)
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-'+rosdistro+'-'+package.replace('_','-') # debian package name (ros-groovy-foo)
branch_name = 'debian/'+debian_pkg+'_%(prop:release_version)s_'+distro # release branch from bloom
deb_name = debian_pkg+'_%(prop:release_version)s'+distro
final_name = debian_pkg+'_%(prop:release_version)s-%(prop:datestamp)s'+distro+'_'+arch+'.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(
haltOnFailure = True,
name = package+'-checkout',
command = ['git', 'checkout', Interpolate(branch_name), '--force'],
hideStepIf = success
)
)
# Download script for building the source deb
f.addStep(
FileDownload(
name = job_name+'-grab-build-source-deb-script',
mastersrc = 'scripts/build_source_deb.py',
slavedest = Interpolate('%(prop:workdir)s/build_source_deb.py'),
mode = 0755,
hideStepIf = success
)
)
def upload_benchmarker(self):
self.addStep(
FileDownload(Interpolate('%s/benchmarker.tar.gz' % MASTERWORKDIR),
'benchmarker.tar.gz',
workdir='.'))
self.addStep(
ShellCommand(name='md5',
command=['md5sum', 'benchmarker.tar.gz'],
workdir='.'))
self.addStep(
ShellCommand(name='unpack_benchmarker',
command=['tar', 'xf', 'benchmarker.tar.gz'],
workdir='.'))
def standalone_buildout(configurator, options, cfg_tokens, manifest_dir):
"""Simple download from master of a self-contained buildout conf file.
See module docstring for signature and return values.
"""
if len(cfg_tokens) != 1:
raise ValueError("Wrong standalong buildout specification: %r" %
cfg_tokens)
if 'bootstrap-script' in options:
warnings.warn(
"The option 'boostrap-script' is now ignored, all "
"bootstraps are done with unibootstrap.py", DeprecationWarning)
conf_path = cfg_tokens[0]
conf_name = os.path.split(conf_path)[-1]
conf_path = os.path.join(manifest_dir, conf_path)
return conf_name, (FileDownload(
mastersrc=conf_path,
slavedest=conf_name,
name="download",
description=["Download", "buildout", "conf"],
haltOnFailure=True), )
def initFactory(self,arch):
if arch != "linux-64": return None
f = factory.BuildFactory()
f.addStep(FileDownload(mastersrc="/var/www/pips/get-pips4u.sh",
slavedest="get-pips4u.sh"))
f.addStep(Cleaning(command=["rm","-rf","/tmp/pipsget"]))
f.addStep(GetPipsTarball(
command=["sh","get-pips4u.sh","--devel",
"--srcdir", "/tmp/pipsget/src",
"--prefix", "/tmp/pipsget/root",
"--force",
"--debug"])
)
f.addStep(Cleaning(command=["rm","-rf","/tmp/pipsget"]))
f.addStep(GetPipsSvn(
command=["sh","get-pips4u.sh",
"--srcdir", "/tmp/pipsget/src",
"--prefix", "/tmp/pipsget/root",
"--force",
"--debug"])
)
return f
def disable_intel_turbo_steps():
steps = []
steps.append(
ShellCommand(
name="disableintelturbo",
command=[
'bash', '-c',
'(echo 0 | sudo /usr/bin/tee /sys/devices/system/cpu/cpufreq/boost) || (echo "only supported on Intel CPUs" && exit 1)'
],
haltOnFailure=True))
class AlwaysSuccessShellCommand(ShellCommand):
def __init__(self, *args, **kwargs):
ShellCommand.__init__(self, *args, **kwargs)
def finished(self, _):
ShellCommand.finished(self, SUCCESS)
# cf. http://pm-blog.yarda.eu/2011/10/deeper-c-states-and-increased-latency.html
# by keeping the file descriptor alive, we make sure that this setting is used.
# after closing the file descriptor, the old setting will be restored by the
# kernel module.
steps.append(FileDownload('forcec0state.sh', 'forcec0state.sh'))
# `setsid' is used in to escape the process group, otherwise it will be
# killed by the timeout logic of AlwaysSuccessShellCommand. since the
# parent process gets killed by it, we always force it to be
# successful. (I wish there would be a nicer way to do it).
steps.append(
AlwaysSuccessShellCommand(name="forceC0state",
command=[
'sudo', '-b', '/bin/bash', '-c',
'setsid bash -x ./forcec0state.sh'
],
haltOnFailure=False,
flunkOnFailure=False,
timeout=5))
return steps
def archive_buildout(self, options, cfg_tokens, manifest_dir):
"""Steps to retrieve an archive (tarball, zip...) buildout from the master.
Currently only .tar.bz2 is supported.
The path of the archive to retrieve is made of:
- a base directory from the same option as upload options for
packaging subfactory (``packaging.upload_dir``)
- a subdir and an archive name property, both specified as tokens in
the buildout option. Archive name MUST NOT contain the archive type
suffix (e.g, '.tar.bz2')
Therefore, this is meant to work on a wide range of archives, not tied
to a particular project
Typically this would be for a triggered build that would do some
further packaging or testing.
For example, one could use this for a generic binary builder that produces
a docker image based on debian:7.7 for any archive produced by this master.
"""
archive_type = '.tar.bz2'
subdir_prop, archive_prop, conf_name = cfg_tokens
master_path = os.path.join(options['packaging.root-dir'],
'%%(prop:%s)s' % subdir_prop,
'%%(prop:%s)s' % archive_prop + archive_type)
slave_name_unpacked = '%%(prop:%s)s' % archive_prop
slave_fname = slave_name_unpacked + archive_type
slave_path = '../' + slave_fname
return conf_name, [
ShellCommand(command=[
'find', '.', '-maxdepth', '1', '-name', '*' + archive_type + '*',
'-delete'
],
workdir='.',
name='clean_arch',
description=['remove', 'prev', 'downloads']),
FileDownload(slavedest=Interpolate(slave_path),
mastersrc=Interpolate(master_path)),
FileDownload(slavedest=Interpolate(slave_path + '.md5'),
mastersrc=Interpolate(master_path + '.md5')),
ShellCommand(
command=['md5sum', '-c',
Interpolate(slave_fname + '.md5')],
workdir='.',
name="checksum",
haltOnFailure=True),
ShellCommand(command=['tar', 'xjf',
Interpolate(slave_fname)],
workdir='.',
name="untar",
description=['unpacking', 'archive'],
descriptionDone=['unpacked', 'archive'],
haltOnFailure=True),
ShellCommand(command=['rm', '-rf', 'build'],
workdir='.',
name='clean',
description=['removing', 'previous', 'build'],
descriptionDone=['removed', 'previous', 'build']),
ShellCommand(command=['mv',
Interpolate(slave_name_unpacked), 'build'],
workdir='.',
name='mv',
description=['setting', 'at', 'build/'])
]
descriptionDone=["clobbered"]))
# force the build/ directory to exist so we can download into it
self.addStep(MakeDirectory(dir="build/build", haltOnFailure=True))
self.addStep(
FileDownload(mastersrc="mingw-makefile",
slavedest="mingw-makefile",
maxsize=102400,
mode=0600,
haltOnFailure=True))
self.addStep(
FileDownload(name="src-download-script",
mastersrc="scripts/download-src.py",
slavedest="download-src.py",
maxsize=102400,
mode=0755,
haltOnFailure=True))
self.addStep(
ShellCommand(name="src-download-web",
description=["source", "download", "web"],
doStepIf=lambda step:
("src_url" in step.build.getProperties()) and
(step.getProperty("src_url") != ""),
command=[
"python", "download-src.py",
WithProperties("%(src_url)s"),
WithProperties("%(src_archive)s")
],
def __init__(self, **kwargs):
factory.BuildFactory.__init__(self, **kwargs)
# set properties about this builder
self.addStep(
SetProperty(
property="basedir",
command=[
"bash", "-c",
"""pwd | sed 's@\\\\\\\\@/@g; s@^\\\\(.\\\\):@/\\\\1@ '"""
],
haltOnFailure=True))
self.addStep(
SetProperty(property="host_cpu", command=["echo", self.host_cpu]))
self.addStep(
SetProperty(property="host_os", command=["echo", self.host_os]))
target_os = "unknown"
for os_mask in [{
"expr": "x86_64",
"value": "w64"
}, {
"expr": "i.86",
"value": "w32"
}]:
if re.match(os_mask["expr"], self.target) is not None:
target_os = os_mask["value"]
break
self.addStep(
SetProperty(property="target-os", command=["echo", target_os]))
self.addStep(
SetProperty(property="target_arch", command=["echo", self.target]))
self.addStep(
SetProperty(property="gcc_config_args",
command=[
"echo",
Property("gcc_config_args", default=""),
self.gccConfigExtraArgs
]))
self.addStep(
SetProperty(property="mingw_config_args",
command=[
"echo",
Property("mingw_config_args", default=""),
self.crtConfigExtraArgs
]))
self.addStep(
SetProperty(property="binutils_config_args",
command=[
"echo",
Property("binutils_config_args", default=""),
self.binutilsConfigExtraArgs
]))
self.addStep(
SetProperty(property="gmp_config_args",
command=[
"echo",
Property("gmp_config_args", default=""),
self.gmpConfigExtraArgs
]))
self.addStep(
SetProperty(property="mpfr_config_args",
command=[
"echo",
Property("mpfr_config_args", default=""),
self.mpfrConfigExtraArgs
]))
self.addStep(
SetProperty(property="mpc_config_args",
command=[
"echo",
Property("mpc_config_args", default=""),
self.mpcConfigExtraArgs
]))
self.addStep(
SetProperty(command=["echo", self.file_extension],
property="file_extension"))
self._step_AdditionalProperties()
self.addStep(
SetProperty(
property="filename_format",
command=[
"echo",
"mingw-%(target-os)s-bin_%(host_cpu)s-%(host_os)s.%(file_extension)s"
],
doStepIf=lambda step:
(not step.build.getProperties().has_key("filename_format"))))
self.addStep(
SetProperty(property="filename",
command=[
"echo",
WithPropertiesRecursive(
WithProperties("%(filename_format)s"))
],
doStepIf=lambda step:
(not step.build.getProperties().has_key("filename"))))
self.addStep(
SetProperty(
property="destname_format",
command=[
"echo",
"mingw-%(target-os)s-bin_%(host_cpu)s-%(host_os)s%(host-extra:-)s%(datestamp:-)s.%(file_extension)s"
],
doStepIf=lambda step:
(not step.build.getProperties().has_key("destname_format"))))
self.addStep(
SetProperty(property="destname",
command=[
"echo",
WithPropertiesRecursive(
WithProperties("%(destname_format)s"))
],
doStepIf=lambda step:
(not step.build.getProperties().has_key("destname"))))
# dump info about this slave
self.addStep(
ShellCommand(name="sys-info",
workdir=".",
command=[
"bash", "-c", r"""true
; echo -e "\n----------\nuname:\n----------"
; uname -a
; echo -e "\n----------\ndistro:\n----------"
; cat /etc/issue
; echo -e "\n----------\ncompiler version:\n----------"
; gcc --version
; echo -e "\n----------\nlinker version:\n----------"
; ld --version
; echo -e "\n----------\nassembler version:\n----------"
; as --version
; echo -e "\n----------\nflex version:\n----------"
; flex --version
; echo -e "\n----------\nbison version:\n----------"
; bison --version
; echo -e "\n----------\nmake version:\n----------"
; make --version
; true
""".replace("\n", "")
],
haltOnFailure=False,
description=["system info"],
descriptionDone=["system info"]))
# set up build root
if self.clobber:
self.addStep(
ShellCommand(name="clobber",
workdir=".",
command=[
"bash", "-c", " ".join("""
junction -d build/build/root/mingw || true ;
rm -rfv build""".split())
],
haltOnFailure=False,
description=["clobber all"],
descriptionDone=["clobbered"]))
# force the build/ directory to exist so we can download into it
self.addStep(MakeDirectory(dir="build/build", haltOnFailure=True))
self.addStep(
FileDownload(mastersrc="mingw-makefile",
slavedest="mingw-makefile",
maxsize=102400,
mode=0600,
haltOnFailure=True))
def launchpad_debbuild(c, package, version, binaries, url, distro, arch, machines, othermirror, keys, trigger_names = None):
f = BuildFactory()
# Grab the source package
f.addStep(
ShellCommand(
haltOnFailure = True,
name = package+'-getsourcedeb',
command = ['dget', '--allow-unauthenticated', url]
)
)
# download hooks
f.addStep(
FileDownload(
name = package+'-grab-hooks',
mastersrc = 'hooks/D05deps',
slavedest = Interpolate('%(prop:workdir)s/hooks/D05deps'),
hideStepIf = success,
mode = 0777 # make this executable for the cowbuilder
)
)
# Update the cowbuilder
f.addStep(
ShellCommand(
command = ['cowbuilder-update.py', distro, arch] + keys,
hideStepIf = success
)
)
# Build it
f.addStep(
ShellCommand(
haltOnFailure = True,
name = package+'-build',
command = ['cowbuilder',
'--build', package+'_'+version+'.dsc',
'--distribution', distro, '--architecture', arch,
'--basepath', '/var/cache/pbuilder/base-'+distro+'-'+arch+'.cow',
'--buildresult', Interpolate('%(prop:workdir)s'),
'--hookdir', Interpolate('%(prop:workdir)s/hooks'),
'--othermirror', othermirror,
'--override-config'],
env = {'DIST': distro},
descriptionDone = ['built binary debs', ]
)
)
# Upload debs
for deb_arch in binaries.keys():
for deb_name in binaries[deb_arch]:
debian_pkg = deb_name+'_'+version+'_'+deb_arch+'.deb'
f.addStep(
FileUpload(
name = deb_name+'-upload',
slavesrc = Interpolate('%(prop:workdir)s/'+debian_pkg),
masterdest = Interpolate('binarydebs/'+debian_pkg),
hideStepIf = success
)
)
# Add the binarydeb using reprepro updater script on master
f.addStep(
MasterShellCommand(
name = deb_name+'-include',
command = ['reprepro-include.bash', deb_name, Interpolate(debian_pkg), distro, deb_arch],
descriptionDone = ['updated in apt', debian_pkg]
)
)
# Trigger if needed
if trigger_names != None:
f.addStep( Trigger(schedulerNames = trigger_names, waitForFinish = False) )
# Add to builders
c['builders'].append(
BuilderConfig(
name = package+'_'+distro+'_'+arch+'_debbuild',
slavenames = machines,
factory = f
)
)
# return name of builder created
return package+'_'+distro+'_'+arch+'_debbuild'
def ros_debbuild(c,
job_name,
packages,
url,
distro,
arch,
rosdistro,
version,
machines,
othermirror,
keys,
trigger_pkgs=None,
locks=[]):
gbp_args = [
'-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist=' + distro, '--git-arch=' + arch
]
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name=job_name + '-clean',
dir=Interpolate('%(prop:builddir)s'),
hideStepIf=success,
))
# Check out the repository master branch, since releases are tagged and not branched
f.addStep(
Git(
repourl=url,
branch='master',
alwaysUseLatest=
True, # this avoids broken builds when schedulers send wrong tag/rev
mode='full' # clean out old versions
))
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-' + rosdistro + '-' + package.replace(
'_', '-') # debian package name (ros-groovy-foo)
branch_name = 'debian/' + debian_pkg + '_%(prop:release_version)s_' + distro # release branch from bloom debian/ros-groovy-foo_0.0.1_kinetic
deb_name = debian_pkg + '_%(prop:release_version)s' + distro
final_name = debian_pkg + '_%(prop:release_version)s' + distro + '_' + arch + '.deb'
# final_name = debian_pkg+'_%(prop:release_version)s-%(prop:datestamp)s'+distro+'_'+arch+'.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(haltOnFailure=True,
name=package + '-checkout',
command=[
'git', 'checkout',
Interpolate(branch_name), '--force'
],
hideStepIf=success))
# Download script for building the source deb
f.addStep(
FileDownload(name=job_name + '-grab-docker-compose-debian',
mastersrc='docker_components/docker-compose-deb.yaml',
workerdest=Interpolate(
'%(prop:builddir)s/docker-compose-deb.yaml'),
mode=0o755,
hideStepIf=success))
f.addStep(
FileDownload(
name=job_name + '-grab-dockerfile-debian',
mastersrc='docker_components/Dockerfile_deb',
workerdest=Interpolate('%(prop:builddir)s/Dockerfile_deb'),
mode=0o755,
hideStepIf=success))
f.addStep(
FileDownload(
name=job_name + '-grab-build-deb-shell',
mastersrc='shell/builddebian.sh',
workerdest=Interpolate('%(prop:builddir)s/builddebian.sh'),
mode=0o755,
hideStepIf=success))
f.addStep(
FileDownload(name=job_name + '-grab-rosdep-private',
mastersrc='docker_components/rosdep_private.yaml',
workerdest=Interpolate(
'%(prop:builddir)s/rosdep_private.yaml'),
mode=0o755,
hideStepIf=success))
f.addStep(
FileDownload(name=job_name + '-grab-rosdep-private',
mastersrc='scripts/unique_docker_deb.py',
workerdest=Interpolate(
'%(prop:builddir)s/unique_docker_deb.py'),
mode=0o755,
hideStepIf=success))
# reedit docker-compose-deb.yaml
f.addStep(
ShellCommand(
haltOnFailure=True,
name=package + '-reedit-docker-compose',
command=[
'python', 'unique_docker_deb.py',
Interpolate('%(prop:builddir)s/docker-compose-deb.yaml'),
Interpolate(package)
],
workdir=Interpolate('%(prop:builddir)s'),
descriptionDone=['reedit docker-compose', package]))
# Build docker image for creating debian
f.addStep(
ShellCommand(
#haltOnFailure = True,
name=package + '-buildsource',
command=[
'docker-compose', '-f',
Interpolate('%(prop:builddir)s/docker-compose-deb.yaml'),
'build'
],
workdir=Interpolate('%(prop:builddir)s'),
descriptionDone=['sourcedeb', package]))
# build debian package
f.addStep(
ShellCommand(
#haltOnFailure=True,
name=job_name + '-build',
command=[
'docker', 'run', '-v',
'ros-buildbot-docker_deb_repository:/home/package',
'--name',
Interpolate(package),
Interpolate('scalable-deb:' + package), 'bash',
'/usr/local/sbin/builddeb.sh'
],
descriptionDone=['build debian package', job_name]))
# update to local repository
f.addStep(
ShellCommand(name=job_name + '-upload',
command=[
'docker', 'exec', '-e',
Interpolate('package=' + debian_pkg + '*'),
'local-repository', 'bash',
'/tmp/debian-upload.sh'
],
descriptionDone=['release package', job_name]))
# rm container
f.addStep(
ShellCommand(name=job_name + '-rm_container',
command=['docker', 'rm',
Interpolate(package)],
descriptionDone=['remove docker container',
job_name]))
# rm image
f.addStep(
ShellCommand(name=job_name + '-rm_image',
command=[
'docker', 'image', 'rm',
Interpolate('scalable-deb:' + package)
],
descriptionDone=['remove docker image', job_name]))
# Trigger if needed
if trigger_pkgs != None:
f.addStep(
Trigger(schedulerNames=[
t.replace('_', '-') + '-' + rosdistro + '-' + distro + '-' +
arch + '-debtrigger' for t in trigger_pkgs
],
waitForFinish=False,
alwaysRun=True))
# Create trigger
c['schedulers'].append(
triggerable.Triggerable(
name=job_name.replace('_', '-') + '-' + rosdistro + '-' + distro +
'-' + arch + '-debtrigger',
builderNames=[
job_name + '_' + rosdistro + '_' + distro + '_' + arch +
'_debbuild',
]))
# Add to builders
c['builders'].append(
BuilderConfig(name=job_name + '_' + rosdistro + '_' + distro + '_' +
arch + '_debbuild',
properties={'release_version': version},
workernames=machines,
factory=f,
locks=locks))
# return name of builder created
return job_name + '_' + rosdistro + '_' + distro + '_' + arch + '_debbuild'
def ros_testbuild(c,
job_name,
url,
branch,
distro,
arch,
rosdistro,
machines,
othermirror,
keys,
token=None):
# Change source is either GitPoller or GitPRPoller
# TODO: make this configurable for svn/etc
project_name = ''
if token:
project_name = '_'.join([job_name, rosdistro, 'prtestbuild'])
c['change_source'].append(
GitPRPoller(
name=rosdistro + "_pr_poller",
repourl=url, # this may pose some problems
project=project_name,
token=token,
pollInterval=15))
# parse repo_url git@github:author/repo.git to repoOwner, repoName
r_owner, r_name = (url.split(':')[1])[:-4].split('/')
c['status'].append(
status.GitHubStatus(token=token,
repoOwner=r_owner,
repoName=r_name))
else:
project_name = '_'.join([job_name, rosdistro, 'testbuild'])
c['change_source'].append(
NamedGitPoller(repourl=url,
name=rosdistro,
branch=branch,
project=project_name))
c['schedulers'].append(
basic.SingleBranchScheduler(
name=project_name,
builderNames=[
project_name,
],
change_filter=ChangeFilter(project=project_name)))
# Directory which will be bind-mounted
binddir = '/tmp/' + project_name
f = BuildFactory()
# Remove any old crud in /tmp folder
f.addStep(ShellCommand(command=['rm', '-rf', binddir], hideStepIf=success))
# Check out repository (to /tmp)
f.addStep(
Git(repourl=util.Property('repository', default=url),
branch=util.Property('branch', default=branch),
alwaysUseLatest=True,
mode='full',
workdir=binddir + '/src/' + job_name))
# Download testbuild.py script from master
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='scripts/testbuild.py',
slavedest=Interpolate('%(prop:workdir)s/testbuild.py'),
hideStepIf=success))
# Update the cowbuilder
f.addStep(
ShellCommand(command=['cowbuilder-update.py', distro, arch] + keys,
hideStepIf=success))
# Make and run tests in a cowbuilder
f.addStep(
TestBuild(name=job_name + '-build',
command=[
'sudo', 'cowbuilder', '--execute',
Interpolate('%(prop:workdir)s/testbuild.py'),
'--distribution', distro, '--architecture', arch,
'--bindmounts', binddir, '--basepath',
'/var/cache/pbuilder/base-' + distro + '-' + arch +
'.cow', '--override-config', '--othermirror',
othermirror, '--', binddir, rosdistro
],
logfiles={'tests': binddir + '/testresults'},
descriptionDone=['make and test', job_name]))
c['builders'].append(
BuilderConfig(name=project_name, slavenames=machines, factory=f))
# return the name of the job created
return project_name
from buildbot.steps.shell import ShellCommand
from buildbot.steps.slave import MakeDirectory, RemoveDirectory
from buildbot.steps.source.git import Git
from buildbot.steps.transfer import FileDownload
from metabbotcfg.common import GIT_URL
from metabbotcfg.slaves import get_slaves, names, slaves
from metabbotcfg.virtualenvsetup import VirtualenvSetup
_PACKAGE_STASH = 'http://ftp.buildbot.net/pub/metabuildbot/python-packages/'
builders = []
# slaves seem to have a hard time fetching from github, so retry
gitStep = Git(repourl=GIT_URL, mode='full', method='fresh', retryFetch=True)
downloadStep = FileDownload(mastersrc="metabbotcfg/virtualenv.whl",
slavedest="virtualenv.whl",
flunkOnFailure=True)
class DatabaseTrial(Trial):
def __init__(self, db, **kwargs):
Trial.__init__(self, **kwargs)
self.db = db
self.addFactoryArguments(db=db)
def setupEnvironment(self, cmd):
Trial.setupEnvironment(self, cmd)
# get the appropriate database configuration from the slave
extra_env = self.buildslave.databases[self.db]
cmd.args['env'].update(extra_env)
command=[
'gbp', 'dch', '-a', '--ignore-branch', '--verbose', '-N',
Interpolate('%(prop:release_version)s-%(prop:datestamp)s' +
distro)
],
descriptionDone=[
'stamped changelog',
Interpolate('%(prop:release_version)s'),
Interpolate('%(prop:datestamp)s')
]))
# download hooks
f.addStep(
FileDownload(
name=package + '-grab-hooks',
mastersrc='hooks/D05deps',
slavedest=Interpolate('%(prop:workdir)s/hooks/D05deps'),
hideStepIf=success,
mode=0777 # make this executable for the cowbuilder
))
# Download script for building the binary deb
f.addStep(
FileDownload(
name=job_name + '-grab-build-binary-deb-script',
mastersrc='scripts/build_binary_deb.py',
slavedest=Interpolate('%(prop:workdir)s/build_binary_deb.py'),
mode=0755,
hideStepIf=success))
# build the binary from the git working copy
f.addStep(
ShellCommand(
haltOnFailure=True,
def upload_credentials(self):
self.addStep(
FileDownload('benchmarkerCredentials',
'benchmarkerCredentials',
workdir='benchmarker'))
def AddDownloadFileStep(self, mastersrc, slavedest):
"""Download a file from master."""
self._factory.addStep(
FileDownload(mastersrc=mastersrc, slavedest=slavedest))
def ros_testbuild(c, job_name, url, branch, distro, arch, rosdistro, machines,
othermirror, keys, source=True, locks=[]):
# Create a Job for Source
if source:
project_name = '_'.join([job_name, rosdistro, 'source_build'])
c['change_source'].append(
GitPoller(
repourl=url,
name=url,
branch=branch,
category=project_name,
pollAtLaunch=True,
)
)
c['schedulers'].append(
schedulers.SingleBranchScheduler(
name=project_name,
builderNames=[project_name,],
change_filter=util.ChangeFilter(category=project_name)
)
)
c['schedulers'].append(
schedulers.Nightly(
name = project_name+'-nightly-master',
codebases = {url:{'repository':url,'branch':'master'}},
builderNames = [project_name,],
hour=3,
minute=0,
)
)
c['schedulers'].append(
schedulers.Nightly(
name = project_name+'-nightly-develop',
codebases = {url:{'repository':url,'branch':'develop'}},
builderNames = [project_name,],
hour=5,
minute=0,
)
)
c['schedulers'].append(
schedulers.ForceScheduler(
name=project_name+'-force',
codebases = [util.CodebaseParameter("",
branch=util.ChoiceStringParameter(
name="branch",
choices=["master", "devel"],
default="master"),
repository=util.FixedParameter(name="repository", default=url),
)],
builderNames=[project_name,],
)
)
else:
r_owner, r_name = (url.split(':')[1])[:-4].split('/')
project_name = '_'.join([job_name, rosdistro, 'pr_build'])
c['change_source'].append(
GitPRPoller(
owner=r_owner,
repo=r_name,
category=project_name,
branches=[branch],
pollInterval=10*60,
pollAtLaunch=True,
token=util.Secret("OathToken"),
repository_type='ssh'
)
)
c['schedulers'].append(
schedulers.SingleBranchScheduler(
name=project_name,
builderNames=[project_name,],
change_filter=util.ChangeFilter(category=project_name)
)
)
# Directory which will be bind-mounted
binddir = '/tmp/'+project_name
dockerworkdir = '/tmp/test/'
f = BuildFactory()
# Remove any old crud in build/src folder
f.addStep(
ShellCommand(
name='rm src',
command=['rm', '-rf', 'build/src'],
hideStepIf=success,
workdir=Interpolate('%(prop:builddir)s')
)
)
# Check out repository (to /build/src)
f.addStep(
Git(
repourl=util.Property('repository', default=url),
branch=util.Property('branch', default=branch),
alwaysUseLatest=True,
mode='full',
workdir=Interpolate('%(prop:builddir)s/build/src')
)
)
# Download testbuild_docker.py script from master
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='scripts/testbuild_docker.py',
workerdest=('testbuild_docker.py'),
hideStepIf=success
)
)
# Download Dockerfile_test script from master
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='docker_components/Dockerfile_test',
workerdest=('Dockerfile_test'),
hideStepIf=success
)
)
# Download docker-compose.py script from master
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='docker_components/docker-compose-test.yaml',
workerdest=('docker-compose-test.yaml'),
hideStepIf=success
)
)
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='docker_components/rosdep_private.yaml',
workerdest=('rosdep_private.yaml'),
hideStepIf=success
)
)
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='scripts/docker-container.py',
workerdest=('docker-container.py'),
hideStepIf=success
)
)
# create docker work environment
f.addStep(
ShellCommand(
command=['python','docker-container.py', job_name],
hideStepIf=success,
workdir=Interpolate('%(prop:builddir)s/build/')
)
)
# Make and run tests in a docker container
f.addStep(
ShellCommand(
name=job_name+'-build',
command=['docker', 'run',
'-v', 'ros-buildbot-docker_deb_repository:/home/package',
'--name='+project_name,
'scalable-env:'+job_name,
'python', '/tmp/build/testbuild_docker.py', binddir,
rosdistro],
descriptionDone=['make and test', job_name]
)
)
f.addStep(
ShellCommand(
name=job_name+'-copytestresults',
command=['docker', 'cp', project_name + ':' +binddir + '/testresults',
'testresults'],
logfiles={'tests': 'testresults'},
descriptionDone=['testresults', job_name]
)
)
f.addStep(
ShellCommand(
name=job_name+'-rm_container',
command=['docker', 'rm', project_name],
descriptionDone=['remove docker container', job_name]
)
)
f.addStep(
ShellCommand(
name=job_name+'-rm_image',
command=['docker', 'image', 'rm', 'scalable-env:'+job_name],
descriptionDone=['remove docker image', job_name]
)
)
c['builders'].append(
BuilderConfig(
name=project_name,
workernames=machines,
factory=f,
locks=locks
)
)
# return the name of the job created
return project_name
def ros_docbuild(c,
job_name,
url,
branch,
rosdistro,
machines,
trigger_pkgs=None):
# Directory which will be bind-mounted
binddir = job_name + '_' + rosdistro + '_docbuild'
f = BuildFactory()
# Remove any old crud in /tmp folder
f.addStep(ShellCommand(command=['rm', '-rf', binddir], hideStepIf=success))
# Check out repository (to /tmp)
f.addStep(
Git(repourl=url,
branch=branch,
alwaysUseLatest=True,
mode='full'
#workdir = binddir+'/src/'+job_name+'/'
))
# Download script from master
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='scripts/docbuild.py',
workerdest=Interpolate('%(prop:builddir)s/docbuild.py'),
hideStepIf=success))
f.addStep(
FileDownload(
name=job_name + '-grab-script',
mastersrc='scripts/unique_docker_doc.py',
workerdest=Interpolate('%(prop:builddir)s/unique_docker_doc.py'),
hideStepIf=success))
f.addStep(
FileDownload(
name=job_name + '-grab-script',
mastersrc='docker_components/Dockerfile_doc',
workerdest=Interpolate('%(prop:builddir)s/Dockerfile_doc'),
hideStepIf=success))
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='docker_components/docker-compose-doc.yaml',
workerdest=Interpolate(
'%(prop:builddir)s/docker-compose-doc.yaml'),
hideStepIf=success))
# reedit docker-compose-doc.yaml
f.addStep(
ShellCommand(
haltOnFailure=True,
name=job_name + '-reedit-docker-compose',
command=[
'python', 'unique_docker_doc.py',
Interpolate('%(prop:builddir)s/docker-compose-doc.yaml'),
Interpolate(job_name)
],
workdir=Interpolate('%(prop:builddir)s'),
descriptionDone=['reedit docker-compose', job_name]))
# Build docker image for creating doc
f.addStep(
ShellCommand(
# haltOnFailure = True,
name=job_name + '-create_docker',
command=[
'docker-compose', '-f',
Interpolate('%(prop:builddir)s/docker-compose-doc.yaml'),
'build'
],
workdir=Interpolate('%(prop:builddir)s'),
descriptionDone=['create_doc', job_name]))
# creating doc in docker
f.addStep(
ShellCommand(
# haltOnFailure=True,
name=job_name + '-create_doc',
command=[
'docker',
'run',
# '-v', 'ros-repository-docker_deb_repository:/home/package',
'--name',
Interpolate('doc_' + job_name),
Interpolate('scalable-doc:' + job_name),
'python',
'/root/docbuild.py',
'/tmp/',
rosdistro
],
descriptionDone=['create doc', job_name]))
f.addStep(
ShellCommand(name=job_name + '-copydocs',
command=[
'docker', 'cp',
Interpolate('doc_' + job_name + ':' + '/tmp/docs'),
'/docs'
],
workdir=Interpolate('%(prop:builddir)s'),
descriptionDone=['copydocs', job_name]))
# rm container
f.addStep(
ShellCommand(name=job_name + '-rm_container',
command=['docker', 'rm',
Interpolate('doc_' + job_name)],
descriptionDone=['remove docker container', job_name]))
# rm image
f.addStep(
ShellCommand(name=job_name + '-rm_image',
command=[
'docker', 'image', 'rm',
Interpolate('scalable-doc:' + job_name)
],
descriptionDone=['remove docker image', job_name]))
# Trigger if needed
if trigger_pkgs != None:
f.addStep(
Trigger(schedulerNames=[
t.replace('_', '-') + '-' + rosdistro + '-doctrigger'
for t in trigger_pkgs
],
waitForFinish=False,
alwaysRun=True))
# Create trigger
c['schedulers'].append(
triggerable.Triggerable(name=job_name.replace('_', '-') + '-' +
rosdistro + '-doctrigger',
builderNames=[
job_name + '_' + rosdistro + '_docbuild',
]))
# Add builder config
c['builders'].append(
BuilderConfig(name=job_name + '_' + rosdistro + '_docbuild',
workernames=machines,
factory=f))
# return the name of the job created
return job_name + '_' + rosdistro + '_docbuild'
def ros_docbuild(c, job_name, url, branch, distro, arch, rosdistro, machines, othermirror, keys, trigger_pkgs = None):
# Directory which will be bind-mounted
binddir = '/tmp/'+job_name+'_'+rosdistro+'_docbuild'
f = BuildFactory()
# Remove any old crud in /tmp folder
f.addStep(
ShellCommand(
command = ['rm', '-rf', binddir],
hideStepIf = success
)
)
# Check out repository (to /tmp)
f.addStep(
Git(
repourl = url,
branch = branch,
alwaysUseLatest = True,
mode = 'full',
workdir = binddir+'/src/'+job_name+'/'
)
)
# Download testbuild.py script from master
f.addStep(
FileDownload(
name = job_name+'-grab-script',
mastersrc = 'scripts/docbuild.py',
slavedest = Interpolate('%(prop:workdir)s/docbuild.py'),
hideStepIf = success
)
)
# Update the cowbuilder
f.addStep(
ShellCommand(
command = ['cowbuilder-update.py', distro, arch] + keys,
hideStepIf = success
)
)
# Build docs in a cowbuilder
f.addStep(
ShellCommand(
haltOnFailure = True,
name = job_name+'-docbuild',
command = ['cowbuilder', '--execute', Interpolate('%(prop:workdir)s/docbuild.py'),
'--distribution', distro, '--architecture', arch,
'--bindmounts', binddir,
'--basepath', '/var/cache/pbuilder/base-'+distro+'-'+arch+'.cow',
'--override-config', '--othermirror', othermirror,
'--', binddir, rosdistro],
env = {'DIST': distro},
descriptionDone = ['built docs', ]
)
)
# Upload docs to master
f.addStep(
DirectoryUpload(
name = job_name+'-upload',
slavesrc = binddir+'/docs',
masterdest = 'docs/' + rosdistro,
hideStepIf = success
)
)
# Trigger if needed
if trigger_pkgs != None:
f.addStep(
Trigger(
schedulerNames = [t.replace('_','-')+'-'+rosdistro+'-doctrigger' for t in trigger_pkgs],
waitForFinish = False,
alwaysRun=True
)
)
# Create trigger
c['schedulers'].append(
triggerable.Triggerable(
name = job_name.replace('_','-')+'-'+rosdistro+'-doctrigger',
builderNames = [job_name+'_'+rosdistro+'_docbuild',]
)
)
# Add builder config
c['builders'].append(
BuilderConfig(
name = job_name+'_'+rosdistro+'_docbuild',
slavenames = machines,
factory = f
)
)
# return the name of the job created
return job_name+'_'+rosdistro+'_docbuild'
ShellCommand(command=["sed", "-i", "s/\"version\": SDK\\[\"PYTHONVERSION\"\\]\\[6:\\],/\"version\": SDK[\"PYTHONVERSION\"][6:].rstrip('dmu'),/", "makepanda/makepandacore.py"]),
# Decode the version number from the dtool/PandaVersion.pp file.
SetPropertyFromCommand("version", command=[
"python3", "makepanda/getversion.py", buildtype_flag],
haltOnFailure=True),
# Delete the built dir, if requested.
ShellCommand(name="clean", command=get_clean_command(),
haltOnFailure=False, doStepIf=lambda step:step.getProperty("clean", False)),
# These steps fill in properties used to determine upstream_version.
] + whl_version_steps + [
# Download the Dockerfile for this distribution.
FileDownload(mastersrc=Interpolate("dockerfiles/%(prop:suite)s-%(prop:arch)s"), workerdest="Dockerfile", workdir="context"),
# Make sure the base distribution is up-to-date.
ShellCommand(command=cloudimg_cmd, workdir="context"),
# Build the Docker image.
ShellCommand(name="setup", command=setup_cmd, workdir="context", haltOnFailure=True),
# Invoke makepanda.
Compile(name="compile py2",
command=get_build_command(2),
haltOnFailure=True,
doStepIf=is_branch("release/1.10.x")),
Compile(name="compile py3",
command=get_build_command(3),
haltOnFailure=True),
def ros_branch_build(c, job_name, packages, url, branch, distro, arch, rosdistro, machines, othermirror, keys):
gbp_args = ['-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist='+distro, '--git-arch='+arch]
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name = job_name+'-clean',
dir = Interpolate('%(prop:workdir)s'),
hideStepIf = success,
)
)
# Pulling the repo
f.addStep(
Git(
repourl = url,
branch = 'HEAD',
alwaysUseLatest = True, # this avoids broken builds when schedulers send wrong tag/rev
mode = 'full', # clean out old versions
getDescription={'tags': True}
)
)
# Check out the repository branch/commit/tag
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'checkout: ' + branch,
command = ['git', 'checkout', branch],
)
)
# get the short commit hash
f.addStep(
SetPropertyFromCommand(
command="git rev-parse --short HEAD", property="commit_hash",
name = 'commit-short-hash',
hideStepIf = success
)
)
# get the time stamp
f.addStep(
SetPropertyFromCommand(
command='date +%Y%M%d%H', property='date_stamp',
name = 'date-stamp',
hideStepIf = success
)
)
# Update the cowbuilder
f.addStep(
ShellCommand(
command = ['cowbuilder-update.py', distro, arch] + keys,
hideStepIf = success
)
)
# Generate the changelog for the package
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'catkin_generate_changelog',
command= ['catkin_generate_changelog', '-y'],
descriptionDone = ['catkin_generate_changelog']
)
)
# Add all files including untracked ones
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'add_changelogs',
command= ['git', 'add', '.'],
descriptionDone = ['add_changelogs']
)
)
# Commit the changelog after updating it
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'update_changelogs',
command= ['git', 'commit', '-m', '\"Updated changelogs\"'],
descriptionDone = ['update_changelogs']
)
)
# Prepare the release without pushing it
# Set very big number to avoid conflicts with available tags
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'catkin_prepare_release',
command= ['catkin_prepare_release', '--version', '100.0.0', '--no-push', '-y'],
descriptionDone = ['catkin_prepare_release']
)
)
#
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'git_bloom_generate_release',
command = ['git-bloom-generate', '-y', 'rosrelease', rosdistro],
)
)
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'git_bloom_generate_debian',
command = ['git-bloom-generate', '-y', 'rosdebian', '-a', '-p', 'release', rosdistro],
)
)
# Get the tag number for the lastest commit
f.addStep(
SetPropertyFromCommand(
command="git describe --tags", property="release_version",
name = 'latest_tag',
)
)
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-'+rosdistro+'-'+package.replace('_','-') # debian package name (ros-groovy-foo)
branch_name = 'debian/'+debian_pkg+'_%(prop:release_version)s-0_'+distro
deb_name = debian_pkg+'_%(prop:release_version)s-0'+distro
final_name = debian_pkg+'_%(prop:release_version)s-%(prop:date_stamp)s-%(prop:commit_hash)s-'+distro+'_'+arch+'.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(
haltOnFailure = True,
name = package+'-checkout',
command = ['git', 'checkout', Interpolate(branch_name), '--force'],
hideStepIf = success
)
)
# Stamp the changelog
f.addStep(
ShellCommand(
haltOnFailure = True,
name = package+'-stampdeb',
command = ['gbp', 'dch', '-a', '--ignore-branch', '--verbose',
'-N', Interpolate('%(prop:release_version)s-%(prop:date_stamp)s-%(prop:commit_hash)s-'+distro)],
descriptionDone = ['stamped changelog', Interpolate('%(prop:release_version)s'),
Interpolate('%(prop:date_stamp)s'), Interpolate('%(prop:commit_hash)s')]
)
)
# download hooks
f.addStep(
FileDownload(
name = package+'-grab-hooks',
mastersrc = 'hooks/D05deps',
slavedest = Interpolate('%(prop:workdir)s/hooks/D05deps'),
hideStepIf = success,
mode = 0777 # make this executable for the cowbuilder
)
)
# Download script for building the binary deb
f.addStep(
FileDownload(
name = job_name+'-grab-build-binary-deb-script',
mastersrc = 'scripts/build_binary_deb.py',
slavedest = Interpolate('%(prop:workdir)s/build_binary_deb.py'),
mode = 0755,
hideStepIf = success
)
)
def ros_branch_build(c,
job_name,
packages,
url,
branch,
distro,
arch,
rosdistro,
machines,
othermirror,
keys,
trigger_pkgs=None):
gbp_args = [
'-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist=' + distro, '--git-arch=' + arch
]
with open(os.path.dirname(os.path.realpath(__file__)) +
"/spec.yaml") as file:
spec_list = yaml.full_load(file)
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name=job_name + '-clean',
dir=Interpolate('%(prop:workdir)s'),
hideStepIf=success,
))
# Check out the repository master branch, since releases are tagged and not branched
f.addStep(
Git(
repourl=url,
branch=branch,
alwaysUseLatest=
True, # this avoids broken builds when schedulers send wrong tag/rev
mode='full', # clean out old versions
getDescription={'tags': True}))
# Update the cowbuilder
f.addStep(
ShellCommand(command=['cowbuilder-update.py', distro, arch] + keys,
hideStepIf=success))
# Generate the changelog for the package
f.addStep(
ShellCommand(haltOnFailure=True,
name='catkin_generate_changelog',
command=['catkin_generate_changelog', '-y'],
descriptionDone=['catkin_generate_changelog']))
# Add all files including untracked ones
f.addStep(
ShellCommand(haltOnFailure=True,
name='add_changelogs',
command=['git', 'add', '.'],
descriptionDone=['add_changelogs']))
# Commit the changelog after updating it
f.addStep(
ShellCommand(haltOnFailure=True,
name='update_changelogs',
command=['git', 'commit', '-m', '\"Updated changelogs\"'],
descriptionDone=['update_changelogs']))
# Prepare the release without pushing it
f.addStep(
ShellCommand(haltOnFailure=True,
name='catkin_prepare_release',
command=[
'catkin_prepare_release', '--bump', 'minor',
'--no-push', '-y'
],
descriptionDone=['catkin_prepare_release']))
#
f.addStep(
ShellCommand(
haltOnFailure=True,
name='git_bloom_generate_release',
command=['git-bloom-generate', '-y', 'rosrelease', rosdistro],
))
f.addStep(
ShellCommand(
haltOnFailure=True,
name='git_bloom_generate_debian',
command=[
'git-bloom-generate', '-y', 'rosdebian', '-a', '-p', 'release',
rosdistro
],
))
# Get the tag number for the lastest commit
f.addStep(
SetPropertyFromCommand(
command="git describe --tags",
property="release_version",
name='latest_tag',
))
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-' + rosdistro + '-' + package.replace(
'_', '-') # debian package name (ros-groovy-foo)
branch_name = 'debian/' + debian_pkg + '_%(prop:release_version)s-0_' + distro
deb_name = debian_pkg + '_%(prop:release_version)s-0' + distro
final_name = debian_pkg + '_%(prop:release_version)s-%(prop:datestamp)s' + distro + '_' + arch + '.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(haltOnFailure=True,
name=package + '-checkout',
command=[
'git', 'checkout',
Interpolate(branch_name), '--force'
],
hideStepIf=success))
# A hack for generating the debian folder so we could build the lastest commit of the specified branch
# f.addStep(
# ShellCommand(
# haltOnFailure = True,
# name = package+'-bloom_generate',
# command= ['bloom-generate', 'rosdebian'],
# descriptionDone = ['bloom_generate', package]
# )
# )
# Download script for building the source deb
f.addStep(
FileDownload(
name=job_name + '-grab-build-source-deb-script',
mastersrc='scripts/build_source_deb.py',
slavedest=Interpolate('%(prop:workdir)s/build_source_deb.py'),
mode=0755,
hideStepIf=success))
]
build_steps = [
Git(config.git_url, getDescription={'match': 'v*'}),
# Decode the version number from the dtool/PandaVersion.pp file.
SetPropertyFromCommand("version",
command=["python", "makepanda/getversion.py"],
haltOnFailure=True),
# Steps to figure out which .whl version to use.
] + whl_version_steps + [
# Download the Dockerfile for this distribution.
FileDownload(mastersrc=Interpolate("dockerfiles/manylinux1-%(prop:arch)s"),
slavedest="Dockerfile",
workdir="context"),
# And the build scripts.
FileDownload(mastersrc="build_scripts/build.sh",
slavedest="build_scripts/build.sh",
workdir="context"),
FileDownload(mastersrc="build_scripts/build_utils.sh",
slavedest="build_scripts/build_utils.sh",
workdir="context"),
# Build the Docker image.
ShellCommand(
name="setup", command=setup_cmd, workdir="context",
haltOnFailure=True),
