def getBaseConfig(self):
return {
'builders': [
BuilderConfig(name="testy",
workernames=["local1", "local2"],
factory=BuildFactory(
[steps.ShellCommand(command='echo hello')])),
],
'workers': [Worker('local' + str(i), 'pass') for i in range(3)],
'schedulers':
[ForceScheduler(name="force", builderNames=["testy"])],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
def loadBuilderConfig(c, use_localhost_worker=False, master_prefix_path='./'):
config = json.load(open(os.path.join(master_prefix_path, 'config.json')))
passwords = json.load(
open(os.path.join(master_prefix_path, 'passwords.json')))
checkWorkersAndBuildersForConsistency(config['workers'],
config['builders'])
c['workers'] = [
Worker(worker['name'], passwords.get(worker['name'], 'password'))
for worker in config['workers']
]
if use_localhost_worker:
c['workers'].append(Worker('local-worker', 'password', max_builds=2))
c['builders'] = []
for builder in config['builders']:
builder['tags'] = getTagsForBuilder(builder)
factory = globals()[builder['factory']]
builder['factory'] = factory()
del builder['platform']
if 'configuration' in builder:
del builder['configuration']
if use_localhost_worker:
builder['workernames'].append("local-worker")
c['builders'].append(builder)
c['schedulers'] = []
for scheduler in config['schedulers']:
schedulerType = globals()[scheduler.pop('type')]
# Python 2.6 can't handle unicode keys as keyword arguments:
# http://bugs.python.org/issue2646. Modern versions of json return
# unicode strings from json.load, so we map all keys to str objects.
scheduler = dict(
map(
lambda key_value_pair:
(str(key_value_pair[0]), key_value_pair[1]),
scheduler.items()))
c['schedulers'].append(schedulerType(**scheduler))
force_scheduler = ForceScheduler(
name='force-{0}'.format(scheduler['name']),
builderNames=scheduler['builderNames'],
properties=[WorkerChoiceParameter()])
c['schedulers'].append(force_scheduler)
def getXmippSchedulers(groupId):
xmippSchedulerNames = [XMIPP_TESTS + groupId]
if groupId == PROD_GROUP_ID:
xmippSchedulerNames += [XMIPP_INSTALL_PREFIX + groupId]
if groupId == SDEVEL_GROUP_ID:
xmippSchedulerNames.append(XMIPP_BUNDLE_TESTS + groupId)
xmippSchedulerNames.append(XMIPP_DOCS_PREFIX + groupId)
schedulers = []
for name in xmippSchedulerNames:
schedulers.append(
triggerable.Triggerable(name=name, builderNames=[name]))
forceScheduler = '%s%s' % (FORCE_BUILDER_PREFIX, name)
schedulers.append(
ForceScheduler(name=forceScheduler, builderNames=[name]))
return schedulers
def test_bad_codebases(self):
# cant specify both codebases and branch/revision/project/repository:
self.assertRaisesConfigError(
"ForceScheduler 'foo': Must either specify 'codebases' or the 'branch/revision/repository/project' parameters:",
lambda: ForceScheduler(name='foo',
builderNames=['bar'],
codebases=['foo'],
branch=StringParameter('name')))
self.assertRaisesConfigError(
"ForceScheduler 'foo': Must either specify 'codebases' or the 'branch/revision/repository/project' parameters:",
lambda: ForceScheduler(name='foo',
builderNames=['bar'],
codebases=['foo'],
revision=StringParameter('name')))
self.assertRaisesConfigError(
"ForceScheduler 'foo': Must either specify 'codebases' or the 'branch/revision/repository/project' parameters:",
lambda: ForceScheduler(name='foo',
builderNames=['bar'],
codebases=['foo'],
project=StringParameter('name')))
self.assertRaisesConfigError(
"ForceScheduler 'foo': Must either specify 'codebases' or the 'branch/revision/repository/project' parameters:",
lambda: ForceScheduler(name='foo',
builderNames=['bar'],
codebases=['foo'],
repository=StringParameter('name')))
# codebases must be a list of either string or BaseParameter types
self.assertRaisesConfigError(
"ForceScheduler 'foo': 'codebases' must be a list of strings or CodebaseParameter objects:",
lambda: ForceScheduler(
name='foo',
builderNames=['bar'],
codebases=[123],
))
self.assertRaisesConfigError(
"ForceScheduler 'foo': 'codebases' must be a list of strings or CodebaseParameter objects:",
lambda: ForceScheduler(
name='foo',
builderNames=['bar'],
codebases=[IntParameter('foo')],
))
# codebases cannot be empty
self.assertRaisesConfigError(
"ForceScheduler 'foo': 'codebases' cannot be empty; use CodebaseParameter(codebase='', hide=True) if needed:",
lambda: ForceScheduler(
name='foo', builderNames=['bar'], codebases=[]))
def get_schedulers() :
from buildbot.schedulers.basic import AnyBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers.forcesched import StringParameter
from buildbot.schedulers.forcesched import FixedParameter
from buildbot.plugins import util
import builders
return [
AnyBranchScheduler(
name = 'default',
reason = 'main repository source code modification',
builderNames = builders.get_builder_names(),
treeStableTimer = 20,
change_filter = util.ChangeFilter(
repository_fn = lambda repository : bool('github.com/SFML/SFML' in repository)
),
properties = {
'trigger' : 'internal'
}
),
AnyBranchScheduler(
name = 'foreign',
builderNames = builders.get_builder_names(),
change_filter = util.ChangeFilter(
repository_fn = lambda repository : bool('github.com/SFML/SFML' not in repository)
),
properties = {
'trigger' : 'external'
}
),
ForceScheduler(
name = 'force',
reason = StringParameter(name = "reason", default = "manual build", size = 100),
builderNames = builders.get_builder_names(),
branch = StringParameter(name = "branch", default = "master", size = 100),
revision = StringParameter(name = "revision", default = "", size = 100),
repository = StringParameter(name = "repository", default = "https://github.com/SFML/SFML.git", regex = r"^https://github.com/[\w-]*/[\w-]*\.git$", size = 100),
project = StringParameter(name = "project", default = "SFML", size = 100),
properties = [
util.FixedParameter(name = "trigger", default = "force")
]
)
]
def loadBuilderConfig(c, use_localhost_worker=False, master_prefix_path='./'):
config = json.load(open(os.path.join(master_prefix_path, 'config.json')))
passwords = json.load(open(os.path.join(master_prefix_path, 'passwords.json')))
checkWorkersAndBuildersForConsistency(config, config['workers'], config['builders'])
checkValidSchedulers(config, config['schedulers'])
c['workers'] = [Worker(worker['name'], passwords.get(worker['name'], 'password')) for worker in config['workers']]
if use_localhost_worker:
c['workers'].append(Worker('local-worker', 'password', max_builds=2))
c['builders'] = []
for builder in config['builders']:
builder['tags'] = getTagsForBuilder(builder)
factory = globals()[builder['factory']]
factorykwargs = {}
for key in ["platform", "configuration", "architectures", "triggers", "additionalArguments"]:
value = builder.pop(key, None)
if value:
factorykwargs[key] = value
builder["factory"] = factory(**factorykwargs)
if use_localhost_worker:
builder['workernames'].append("local-worker")
c['builders'].append(builder)
c['schedulers'] = []
for scheduler in config['schedulers']:
schedulerClassName = scheduler.pop('type')
schedulerClass = globals()[schedulerClassName]
# Python 2.6 can't handle unicode keys as keyword arguments:
# http://bugs.python.org/issue2646. Modern versions of json return
# unicode strings from json.load, so we map all keys to str objects.
scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
if (schedulerClassName == 'Try_Userpass'):
# FIXME: Read the credentials from local file on disk.
scheduler['userpass'] = [('sampleuser', 'samplepass')]
c['schedulers'].append(schedulerClass(**scheduler))
force_scheduler = ForceScheduler(name='force-{0}'.format(scheduler['name']),
builderNames=scheduler['builderNames'],
properties=[WorkerChoiceParameter()])
c['schedulers'].append(force_scheduler)
def makeForceScheduler():
return ForceScheduler(
name="force",
builderNames=["runtests", "runtests2", "runtests1"],
codebases=[
CodebaseParameter('',
branch=StringParameter(name="branch",
label="Branch",
default='master'),
repository=FixedParameter(
name="repository", default='repo'))
],
properties=[
BooleanParameter(name="force_rebuild",
label="Force Rebuild",
default=True),
BooleanParameter(name="force_chain_rebuild",
label="Force Chain Rebuild",
default=False)
])
def __init__(self, name, workers):
self.name = utils.name(name)
self.workers = workers
self.build = BuildFactory()
BuildmasterConfig['builders'].append(BuilderConfig(
name=self.name,
factory=self.build,
slavenames=self.workers))
BuildmasterConfig['schedulers'].append(ForceScheduler(
name=scheduler_name(self, 'force'),
builderNames=[self.name],
branch=FixedParameter(name="reason", default=""),
reason=FixedParameter(name="reason", default="Forced build"),
revision=FixedParameter(name="revision", default=""),
repository=FixedParameter(name="repository", default=""),
project=FixedParameter(name="project", default=""),
properties=[],
username=FixedParameter(name="username", default="WebUI")))
def getSchedulers():
daily = Periodic(name='daily',
builderNames=['clean-old-builds'],
periodicBuildTimer=24 * 3600)
# This is so the zulip reporter gives better message.
daily.codebases = {'maint': {'branch': 'daily'}}
resource_cleanup = Periodic(
name='resource-cleanup',
builderNames=[RESOURCE_CLEANUP_BUILDER],
periodicBuildTimer=60 * 30,
)
# This is so the zulip reporter gives better message.
resource_cleanup.codebases = {'maint': {'branch': 'resource-cleanup'}}
force = ForceScheduler(
name="force-maintenance",
builderNames=["clean-old-builds", RESOURCE_CLEANUP_BUILDER],
)
return [daily, resource_cleanup, force]
def masterConfig():
global num_reconfig
num_reconfig += 1
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.steps.shell import ShellCommand
from buildbot.util.service import BuildbotService
class MyShellCommand(ShellCommand):
def getResultSummary(self):
service = self.master.service_manager.namedServices['myService']
return dict(step=u"num reconfig: %d" % (service.num_reconfig,))
class MyService(BuildbotService):
name = "myService"
def reconfigService(self, num_reconfig):
self.num_reconfig = num_reconfig
return defer.succeed(None)
c['schedulers'] = [
ForceScheduler(
name="force",
builderNames=["testy"])]
f = BuildFactory()
f.addStep(MyShellCommand(command='echo hei'))
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
factory=f)]
c['services'] = [MyService(num_reconfig=num_reconfig)]
if num_reconfig == 3:
c['services'].append(
MyService(name="myService2", num_reconfig=num_reconfig))
return c
def configure(self, config_dict):
steps = []
if self.logHorizon is not None:
steps.append(LogChunksJanitor(logHorizon=self.logHorizon))
if self.build_data_horizon is not None:
steps.append(
BuildDataJanitor(build_data_horizon=self.build_data_horizon))
if not steps:
return
hour = self.hour
kwargs = self.kwargs
super().configure(config_dict)
nightly_kwargs = {}
# we take the defaults of Nightly, except for hour
for arg in ('minute', 'dayOfMonth', 'month', 'dayOfWeek'):
if arg in kwargs:
nightly_kwargs[arg] = kwargs[arg]
self.schedulers.append(
Nightly(name=JANITOR_NAME,
builderNames=[JANITOR_NAME],
hour=hour,
**nightly_kwargs))
self.schedulers.append(
ForceScheduler(name=JANITOR_NAME + "_force",
builderNames=[JANITOR_NAME]))
self.builders.append(
BuilderConfig(name=JANITOR_NAME,
workername=JANITOR_NAME,
factory=BuildFactory(steps=steps)))
self.protocols.setdefault('null', {})
self.workers.append(LocalWorker(JANITOR_NAME))
def getSchedulers():
schedulers = [
ForceScheduler(
name="force-flocker-acceptance",
codebases=[
CodebaseParameter(
"flocker",
branch=StringParameter("branch", default="master",
size=80),
repository=FixedParameter("repository",
default=GITHUB + b"/flocker"),
),
],
properties=[
report_expected_failures_parameter,
],
builderNames=BUILDERS,
),
]
for distribution in OMNIBUS_DISTRIBUTIONS:
builders = [
configuration.builder_name
for configuration in ACCEPTANCE_CONFIGURATIONS
if configuration.distribution == distribution
]
schedulers.append(
Triggerable(
name='trigger/built-packages/%s' % (distribution, ),
builderNames=builders,
codebases={
"flocker": {
"repository": GITHUB + b"/flocker"
},
},
))
return schedulers
def test_compare_properties(self):
self.assertNotEqual(
ForceScheduler(name="testched", builderNames=[],
properties=[]),
ForceScheduler(name="testched", builderNames=[],
properties=[FixedParameter("prop", "thanks for the fish!")]))
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.schedulers.basic import AnyBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.steps.shell import ShellCommand
from buildbot.worker import Worker
c['workers'] = [Worker("local1", "localpw")]
c['protocols'] = {'pb': {'port': 'tcp:0'}}
c['change_source'] = []
c['change_source'] = PBChangeSource()
c['schedulers'] = []
c['schedulers'].append(
AnyBranchScheduler(name="all",
change_filter=ChangeFilter(project_re='^testy/'),
treeStableTimer=1 * 60,
builderNames=[
'testy',
]))
c['schedulers'].append(ForceScheduler(name="force", builderNames=["testy"]))
f1 = BuildFactory()
f1.addStep(ShellCommand(command='echo hi'))
c['builders'] = []
c['builders'].append(
BuilderConfig(name="testy", workernames=["local1"], factory=f1))
c['status'] = []
c['title'] = "test"
c['titleURL'] = "test"
c['buildbotURL'] = "http://localhost:8010/"
c['db'] = {'db_url': "sqlite:///state.sqlite"}
def test_compare_username(self):
self.assertNotEqual(
ForceScheduler(name="testched", builderNames=[]),
ForceScheduler(name="testched", builderNames=[],
username=FixedParameter("username", "The Fisher King <*****@*****.**>")))
def test_listofints_builderNames(self):
self.assertRaisesConfigError("ForceScheduler 'testsched': builderNames must be a list of strings:",
lambda: ForceScheduler(name='testsched', builderNames=[1234],
codebases=['bar'], username="******"))
def test_listofmixed_properties(self):
self.assertRaisesConfigError("ForceScheduler 'testsched': properties must be a list of BaseParameters:",
lambda: ForceScheduler(name='testsched', builderNames=[],
codebases=['bar'], username="******",
properties=[BaseParameter(name="test",),
4567]))
def test_bad_reason(self):
self.assertRaisesConfigError("ForceScheduler 'testsched': reason must be a StringParameter",
lambda: ForceScheduler(name='testsched', builderNames=[],
codebases=['bar'], reason="foo"))
def test_compare_codebases(self):
self.assertNotEqual(
ForceScheduler(name="testched", builderNames=[],
codebases=['bar']),
ForceScheduler(name="testched", builderNames=[],
codebases=['foo']))
def test_listofints_properties(self):
with self.assertRaisesConfigError(
"ForceScheduler 'testsched': properties must be a list of BaseParameters:"):
ForceScheduler(name='testsched', builderNames=[],
codebases=['bar'], username="******",
properties=[1234, 2345])
def test_listofunicode_builderNames(self):
ForceScheduler(name='testsched', builderNames=['a', 'b'])
def test_notstring_name(self):
with self.assertRaisesConfigError(
"ForceScheduler name must be a unicode string:"):
ForceScheduler(name=1234, builderNames=[], codebases=['bar'],
username="******")
def test_bad_username(self):
with self.assertRaisesConfigError(
"ForceScheduler 'testsched': username must be a StringParameter"):
ForceScheduler(name='testsched', builderNames=[],
codebases=['bar'], username="******")
def addSimpleProject(self, name, category, repourl, builderconfigs):
"""Private.
Add a project which builds when the source changes or when Force is clicked.
"""
# FACTORIES
# FIXME: get list of steps from buildshim here
#factory = BuildFactory()
# check out the source
# This fails with git-1.8 and up unless you specify the branch, so do this down lower where we now the branch
#factory.addStep(Git(repourl=repourl, mode='full', method='copy'))
# for step in ["patch", "install_deps", "configure", "compile", "check", "package", "upload", "uninstall_deps"]:
# factory.addStep(ShellCommand(command=["../../srclink/" + name + "/buildshim", step], description=step))
# BUILDERS AND SCHEDULERS
# For each builder in config file, see what OS they want to
# run on, and assign them to suitable slaves.
# Also create a force scheduler that knows about all the builders.
branchnames = []
buildernames = []
for builderconfig in builderconfigs:
bparams = ''
if "params" in builderconfig:
bparams = builderconfig["params"].encode('ascii', 'ignore')
bsuffix = ''
if "suffix" in builderconfig:
bsuffix = builderconfig["suffix"].encode('ascii', 'ignore')
sbranch = builderconfig["branch"].encode('ascii', 'ignore')
if sbranch not in branchnames:
branchnames.append(sbranch)
sosses = builderconfig["os"].encode('ascii', 'ignore').split('>')
sosses.reverse()
# The first OS in the list triggers when there's a source change
sos = sosses.pop()
buildername = name + '-' + sos + '-' + sbranch + bsuffix
factory = self.addSimpleBuilder(name, buildername, category,
repourl, builderconfig, sos,
sbranch, bparams)
self['schedulers'].append(
SingleBranchScheduler(
name=buildername,
change_filter=filter.ChangeFilter(branch=sbranch,
repository=repourl),
treeStableTimer=1 *
60, # Set this just high enough so you don't swamp the slaves, or to None if you don't want changes batched
builderNames=[buildername]))
buildernames.append(buildername)
# The rest of the OSes in the list, if any, are triggered when the previous OS in the list finishes
while len(sosses) > 0:
prev_factory = factory
sos = sosses.pop()
buildername = name + '-' + sos + '-' + sbranch + bsuffix
factory = self.addSimpleBuilder(name, buildername, category,
repourl, builderconfig, sos,
sbranch, bparams)
self['schedulers'].append(
triggerable.Triggerable(name=buildername,
builderNames=[buildername]))
prev_factory.addStep(
trigger.Trigger(schedulerNames=[buildername],
waitForFinish=False))
self['schedulers'].append(
ForceScheduler(
name=name + "-force",
builderNames=buildernames,
branch=FixedParameter(name="branch", default=""),
revision=FixedParameter(name="revision", default=""),
repository=FixedParameter(name="repository", default=""),
project=FixedParameter(name="project", default=""),
properties=[],
))
# CHANGESOURCES
# It's a git git git git git world
already = False
for cs in self['change_source']:
if cs.repourl == repourl:
log.msg(
"There's already a changesource for %s. Hope it has the branch you wanted."
% cs.repourl)
already = True
if not already:
self['change_source'].append(
# Fuzz the interval to avoid slamming the git server and hitting the MaxStartups or MaxSessions limits
# If you hit them, twistd.log will have lots of "ssh_exchange_identification: Connection closed by remote host" errors
# See http://trac.buildbot.net/ticket/2480
GitPoller(repourl,
branches=branchnames,
workdir='gitpoller-workdir-' + name,
pollinterval=60 + random.uniform(-10, 10)))
def test_emptystring_name(self):
with self.assertRaisesConfigError(
"ForceScheduler name must not be empty:"):
ForceScheduler(name='', builderNames=[], codebases=['bar'],
username="******")
factory=factoryRsyslogPkgRealRpmBuild,
tags=["rsyslogrpm", "real", "rpmbuild"],
properties={
"github_repo_owner": "rsyslog",
"github_repo_name": "rsyslog-pkg-rhel-centos",
},
))
lc['schedulers'].append(
ForceScheduler(name="pull_rsyslog_rsyslogrpm",
label="1. Pull Requests-rsyslog-rsyslogrpm",
builderNames=["rsyslogrpm rpmbuild"],
codebases=[
util.CodebaseParameter(
"",
branch=util.StringParameter(
name="branch",
label="Pull Request Number:",
required=True,
default="refs/pull/<NUMBER>/head",
size=80),
),
]))
lc['schedulers'].append(
ForceScheduler(name="forceall_rsyslog_rsyslogrpm",
label="2. Force All-rsyslog-rsyslogrpm",
builderNames=["rsyslogrpm rpmbuild"]))
lc['schedulers'].append(
SingleBranchScheduler(name="github_rsyslog-rsyslogrpm",
change_filter=filter.ChangeFilter(
def test_listofmixed_builderNames(self):
with self.assertRaisesConfigError(
"ForceScheduler 'testsched': builderNames must be a list of strings:"):
ForceScheduler(name='testsched', builderNames=['test', 1234],
codebases=['bar'], username="******")
c['change_source'] = []
####### SCHEDULERS
# Configure the Schedulers, which decide how to react to incoming changes. In this
# case, just kick off a 'runtests' build
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.changes import filter
c['schedulers'] = []
c['schedulers'].append(ForceScheduler(
name="force-master",
builderNames=["master"]))
c['schedulers'].append(SingleBranchScheduler(
name="git-master",
change_filter=filter.ChangeFilter(branch='master'),
treeStableTimer=30,
builderNames=["master"]))
# c['schedulers'].append(SingleBranchScheduler(
# name="git-develop",
# change_filter=filter.ChangeFilter(branch='develop'),
# treeStableTimer=30,
# builderNames=["develop"]))
####### BUILDERS
def test_compare_reason(self):
self.assertNotEqual(
ForceScheduler(name="testched", builderNames=[],
reason=FixedParameter("reason", "no fish for you!")),
ForceScheduler(name="testched", builderNames=[],
reason=FixedParameter("reason", "thanks for the fish!")))
def insertData(self):
EndpointBase.insertData(self)
self.master.allSchedulers = lambda: [
ForceScheduler(name="sched1", builderNames=["builder"])
]
