def test_setServiceParent_slaveLocks(self):
"""
http://trac.buildbot.net/ticket/2278
"""
master = self.master
bm = botmaster.FakeBotMaster(master)
bm.startService()
lock = locks.SlaveLock('lock')
bs = ConcreteBuildSlave('bot', 'pass', locks=[lock.access("counting")])
bs.setServiceParent(bm)
def test_setServiceParent_slaveLocks(self):
"""
http://trac.buildbot.net/ticket/2278
"""
master = self.master = fakemaster.make_master()
botmaster = FakeBotMaster(master)
botmaster.startService()
lock = locks.SlaveLock('lock')
bs = self.ConcreteBuildSlave('bot', 'pass', locks=[lock])
bs.setServiceParent(botmaster)
def test_setServiceParent_slaveLocks(self):
"""
http://trac.buildbot.net/ticket/2278
"""
master = self.master
bsmanager = master.buildslaves
yield master.startService()
lock = locks.SlaveLock('lock')
bs = ConcreteBuildSlave('bot', 'pass', locks=[lock.access("counting")])
bs.setServiceParent(bsmanager)
def getBuilders():
result = []
oneBuildPerSlave = locks.SlaveLock('one-build-per-slave')
for name, factory, slaves in _builders:
result.append({
'name': name,
'slavenames': slaves,
'builddir': name,
'locks': [oneBuildPerSlave],
'factory': factory()
})
return result
def testGet(self):
# the master.cfg file contains "lock ids", which are instances of
# MasterLock and SlaveLock but which are not actually Locks per se.
# When the build starts, these markers are turned into RealMasterLock
# and RealSlaveLock instances. This insures that any builds running
# on slaves that were unaffected by the config change are still
# referring to the same Lock instance as new builds by builders that
# *were* affected by the change. There have been bugs in the past in
# which this didn't happen, and the Locks were bypassed because half
# the builders were using one incarnation of the lock while the other
# half were using a separate (but equal) incarnation.
#
# Changing the lock id in any way should cause it to be replaced in
# the BotMaster. This will result in a couple of funky artifacts:
# builds in progress might pay attention to a different lock, so we
# might bypass the locking for the duration of a couple builds.
# There's also the problem of old Locks lingering around in
# BotMaster.locks, but they're small and shouldn't really cause a
# problem.
b = master.BotMaster()
l1 = locks.MasterLock("one")
l1a = locks.MasterLock("one")
l2 = locks.MasterLock("one", maxCount=4)
rl1 = b.getLockByID(l1)
rl2 = b.getLockByID(l1a)
self.failUnlessIdentical(rl1, rl2)
rl3 = b.getLockByID(l2)
self.failIfIdentical(rl1, rl3)
s1 = locks.SlaveLock("one")
s1a = locks.SlaveLock("one")
s2 = locks.SlaveLock("one", maxCount=4)
s3 = locks.SlaveLock("one", maxCount=4,
maxCountForSlave={"a":1, "b":2})
s3a = locks.SlaveLock("one", maxCount=4,
maxCountForSlave={"a":1, "b":2})
s4 = locks.SlaveLock("one", maxCount=4,
maxCountForSlave={"a":4, "b":4})
rl1 = b.getLockByID(s1)
rl2 = b.getLockByID(s1a)
self.failUnlessIdentical(rl1, rl2)
rl3 = b.getLockByID(s2)
self.failIfIdentical(rl1, rl3)
rl4 = b.getLockByID(s3)
self.failIfIdentical(rl1, rl4)
self.failIfIdentical(rl3, rl4)
rl5 = b.getLockByID(s3a)
self.failUnlessIdentical(rl4, rl5)
rl6 = b.getLockByID(s4)
self.failIfIdentical(rl5, rl6)
def testSlave(self):
sid1 = locks.SlaveLock("name1")
sid2 = locks.SlaveLock("name1")
sid3 = locks.SlaveLock("name3")
sid4 = locks.SlaveLock("name1", maxCount=3)
mcfs = {"bigslave": 4, "smallslave": 1}
sid5 = locks.SlaveLock("name1", maxCount=3, maxCountForSlave=mcfs)
mcfs2 = {"bigslave": 4, "smallslave": 1}
sid5a = locks.SlaveLock("name1", maxCount=3, maxCountForSlave=mcfs2)
mcfs3 = {"bigslave": 1, "smallslave": 99}
sid5b = locks.SlaveLock("name1", maxCount=3, maxCountForSlave=mcfs3)
self.failUnlessEqual(sid1, sid2)
self.failIfEqual(sid1, sid3)
self.failIfEqual(sid1, sid4)
self.failIfEqual(sid1, sid5)
self.failUnlessEqual(sid5, sid5a)
self.failIfEqual(sid5a, sid5b)
# they should all be hashable
d = {sid1: 1, sid2: 2, sid3: 3, sid4: 4, sid5: 5, sid5a: 6, sid5b: 7}
del d
l1 = self.make(sid1)
self.failUnlessEqual(l1.name, "name1")
self.failUnlessEqual(l1.maxCount, 1)
l1s1 = l1.getLock(slave("slave1"))
self.failIfIdentical(l1s1, l1)
l4 = self.make(sid4)
self.failUnlessEqual(l4.maxCount, 3)
l4s1 = l4.getLock(slave("slave1"))
self.failUnlessEqual(l4s1.maxCount, 3)
l5 = self.make(sid5)
l5s1 = l5.getLock(slave("bigslave"))
l5s2 = l5.getLock(slave("smallslave"))
l5s3 = l5.getLock(slave("unnamedslave"))
self.failUnlessEqual(l5s1.maxCount, 4)
self.failUnlessEqual(l5s2.maxCount, 1)
self.failUnlessEqual(l5s3.maxCount, 3)
import textwrap
from buildbot import locks
from buildbot.steps.shell import ShellCommand
# only allow one VirtualenvSetup to run on a slave at a time. This prevents
# collisions in the shared package cache.
veLock = locks.SlaveLock('veLock')
# we hardcode a decent version of virtualenv here
class VirtualenvSetup(ShellCommand):
def __init__(self,
virtualenv_dir='sandbox',
virtualenv_python='python2.7',
virtualenv_packages=None,
**kwargs):
kwargs['locks'] = kwargs.get('locks',
[]) + [veLock.access('exclusive')]
ShellCommand.__init__(self, **kwargs)
if virtualenv_packages is None:
virtualenv_packages = []
self.virtualenv_dir = virtualenv_dir
self.virtualenv_python = virtualenv_python
self.virtualenv_packages = virtualenv_packages
self.addFactoryArguments(virtualenv_dir=virtualenv_dir,
virtualenv_python=virtualenv_python,
virtualenv_packages=virtualenv_packages)
"nextBuild": myNextBuild,
'category': "experimental"
}
###############################################################################################
#
# RQG and storage engine tests on a Windows machine
from buildbot.steps.slave import RemoveDirectory
from buildbot import locks
# This is a very strong lock, it will be used if the builder cannot proceed without killing mysqld,
# and it will require waiting for all tests in other builder to finish
#kill_mysqld_lock = locks.SlaveLock("mysqld_kill_license")
git_rqg_lock = locks.SlaveLock("git_rqg")
#release_build_lock = locks.SlaveLock("release_build")
#debug_build_lock = locks.SlaveLock("debug_build")
def rqg_win_factory(mtr_build_thread="130", config="Debug"):
if config == 'Debug':
do_debug_steps = True
do_release_steps = False
else:
do_debug_steps = False
do_release_steps = True
f = factory.BuildFactory()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import StringIO
import factory
reload(factory)
from factory import *
from buildbot import locks
integration_lock = locks.SlaveLock("integration", maxCount=1)
class XvfbNoseTest(ShellCommand):
def __init__(self, test_files, min_coverage, reportdir=""):
test_paths = ''
for test_file in test_files:
test_paths += ' ' + test_file + '.py'
self.packages = test_files
cover_packages = '--cover-package=' + ','.join(test_files)
count_packages = len(test_files)
name_packages = test_files[0] + ' test' if count_packages == 1 else 'tests'
if reportdir:
reportdir = "/" + reportdir
self.reportdir = reportdir
command = Interpolate('DISPLAY=:1 xvfb-run -s "-screen 0 1280x1200x8" nosetests -v'
'--with-html --html-file=report%s/index.html --cover-tests '
'--with-coverage ' % reportdir + cover_packages +
' --cover-min-percentage=' + str(min_coverage) +
' --cover-erase --cover-html --cover-html-dir=report%s/coverage' % reportdir + test_paths)
from factories.pyethapp import *
from factories.serpent import *
from factories.debian import *
from factories.debian_backport import *
from factories.poc_servers import *
from factories.integration import *
# ###### BUILDERS
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them. Note that any particular build will
# only take place on one slave.
builders = []
self_lock = locks.SlaveLock("self_update", maxCount=1)
build_lock = locks.SlaveLock("slave_builds", maxCount=2)
# package_lock = locks.SlaveLock("slave_packaging",
# maxCount=4,
# maxCountForSlave = {
# 'slave-cpp-one-deb': 2,
# 'slave-cpp-two-deb': 2,
# 'slave-go-one-deb': 2,
# 'slave-go-two-deb': 2 })
go_lock = locks.SlaveLock("go_builds", maxCount=1)
arm_lock = locks.SlaveLock("arm_builds", maxCount=1)
osx_lock = locks.SlaveLock("osx_builds", maxCount=2)
brew_lock = locks.SlaveLock("brew_builds", maxCount=1)
win_lock = locks.SlaveLock("win_builds", maxCount=2)
win_lock_go = locks.SlaveLock("win_go_builds", maxCount=1)
from buildbot import locks
from buildbot.steps.shell import ShellCommand
from buildbot.steps.shell import SetPropertyFromCommand
from buildbot.steps.python import Sphinx
from buildbot.steps.transfer import FileDownload
from buildbot.steps.transfer import FileUpload
from buildbot.steps.transfer import DirectoryUpload
from buildbot.steps.master import MasterShellCommand
from buildbot.process.properties import WithProperties
from buildbot.process.properties import Property
from ..utils import comma_list_sanitize
from ..utils import bool_opt
from ..utils import BUILD_UTILS_PATH
from ..constants import DEFAULT_BUILDOUT_PART
port_lock = locks.SlaveLock("port-reserve")
def steps_odoo_port_reservation(configurator, options, environ=()):
"""Return steps for port reservation.
The chosen port is stored in ``openerp_port`` property.
Available manifest file options:
:odoo.http-port-min: minimal value for the HTTP port (defaults to 6069)
:odoo.http-port-max: maximal value for the HTTP port (defaults to 7069)
:odoo.http-port-step: increment value for the HTTP port (defaults to 5)
"""
return (
from buildbot import locks
slave_cpu_lock = locks.SlaveLock('slave_cpu_lock', maxCount=1)
master_upload_lock = locks.MasterLock('master_upload_lock')
def skipped_or_success(results, step):
from buildbot.status.results import SKIPPED
from buildbot.status.results import SUCCESS
return ((results == SKIPPED) or (results == SUCCESS))
def skipped(results, step):
from buildbot.status.results import SKIPPED
return (results == SKIPPED)
def get_cmake_step(link, type, options=[]):
from buildbot.process.properties import Interpolate
from buildbot.steps.shell import ShellCommand
build_type = ''
if type == 'debug':
build_type += '-DCMAKE_BUILD_TYPE=Debug'
else:
build_type += '-DCMAKE_BUILD_TYPE=Release'
def __init__(self):
self.slaves = []
self.builders = []
self.auto_builder_names = []
self.local_builder_lock = locks.MasterLock("local", maxCount=2)
self.deps_lock = locks.SlaveLock("deps", maxCount=1)
# Add linux slaves and builders.
for linux_distro, versions in CONFIG['linux'].iteritems():
factory = LINUX_FACTORIES[linux_distro]
for version in versions:
self._AddBuilderAndSlave(linux_distro, version, False, factory)
self._AddBuilderAndSlave(linux_distro, version, True, factory)
# Add Ubuntu PPAs.
for version in CONFIG['linux']['ubuntu']:
self._AddBuilder(name='Ubuntu dev PPA %s' % version.title(),
slave='ubuntu-%s-32' % version,
build_factory=builders.MakePPABuilder(
version, DEV_PPA))
self._AddBuilder(name='Ubuntu official PPA %s' % version.title(),
slave='ubuntu-%s-32' % version,
build_factory=builders.MakePPABuilder(
version, OFFICIAL_PPA),
auto=False)
# Add special slaves.
for name in CONFIG['special_slaves']:
self._AddSlave(name)
# Windows.
self._AddBuilder(name='Windows Dependencies',
slave='mingw',
build_factory=builders.MakeWindowsDepsBuilder(),
auto=False,
deps_lock='exclusive')
self._AddBuilder(name='Windows Release',
slave='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=False, is_portable=False),
deps_lock='counting')
self._AddBuilder(name='Windows Debug',
slave='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=True, is_portable=False),
deps_lock='counting')
self._AddBuilder(name='Windows Portable',
slave='mingw',
build_factory=builders.MakeWindowsBuilder(
is_debug=False, is_portable=True),
deps_lock='counting')
# Mac.
self._AddSlave('mac')
self._AddBuilder(name='Mac Dependencies',
slave='mac',
build_factory=builders.MakeMacDepsBuilder(),
auto=False,
local_lock=False,
deps_lock='exclusive')
self._AddBuilder(name='Mac Release',
slave='mac',
build_factory=builders.MakeMacBuilder(),
local_lock=False,
deps_lock='counting')
self._AddBuilder(name='Mac Cross',
slave='mac-cross',
build_factory=builders.MakeMacCrossBuilder())
# Spotify.
self._AddBuilder(name='Spofify blob 32-bit',
slave='spotify-blob-32',
build_factory=builders.MakeSpotifyBlobBuilder())
self._AddBuilder(name='Spofify blob 64-bit',
slave='spotify-blob-64',
build_factory=builders.MakeSpotifyBlobBuilder())
# Transifex.
self._AddBuilder(name='Transifex POT push',
slave='transifex',
build_factory=builders.MakeTransifexPotPushBuilder())
self._AddBuilder(name='Transifex PO pull',
slave='transifex',
build_factory=builders.MakeTransifexPoPullBuilder(),
auto=False)
self._AddBuilder(
name='Transifex website POT push',
slave='transifex',
build_factory=builders.MakeWebsiteTransifexPotPushBuilder(),
auto=False)
self._AddBuilder(
name='Transifex website PO pull',
slave='transifex',
build_factory=builders.MakeWebsiteTransifexPoPullBuilder(),
auto=False)
self._AddBuilder(
name='Transifex Android PO pull',
slave='transifex',
build_factory=builders.MakeAndroidTransifexPoPullBuilder(),
auto=False)
# Android.
self._AddBuilder(name='Android Remote',
slave='android',
build_factory=builders.MakeAndroidRemoteBuilder(),
auto=False)
# Source.
self._AddBuilder(name='Source',
slave='ubuntu-xenial-64',
build_factory=builders.MakeSourceBuilder())
BUILDSLAVE_KWARGS = { # name -> validating callable
'max_builds': int,
'notify_on_missing': str,
}
BUILDSLAVE_PROPERTIES = { # name -> validating callable (only for non-str)
'slave_priority': float,
}
BUILDSLAVE_REQUIRED = ('password', )
logger = logging.getLogger(__name__)
# Running buildouts in parallel on one slave fails
# if they used shared eggs or downloads area
buildout_caches_lock = locks.SlaveLock("buildout caches")
class BuildoutsConfigurator(object):
"""Populate buildmaster configs from buildouts and external slaves.cfg.
Use the three subfactories of steps from ``subfactories``:
- buildout_download (see ``subfactories.download``)
- post_download (see ``subfactories.postdownload``)
- post_buildout (see ``subfactories.postbuildout``)
"""
capabilities = dict(wkhtmltopdf=dict(
version_prop='wkhtml2pdf_version',
environ={'DISPLAY': '%(cap(display):-:0)s'}),
python=dict(version_prop='py_version',
