def scheduler_nightly_wintools(name, builders, codebases, hour, minute):
return schedulers.Nightly(name=name,
codebases=codebases,
builderNames=builders,
hour=hour,
minute=minute,
onlyIfChanged=True)
def _getBasicSchedulers(props):
pretty_branch_name = props['branch_pretty']
branch_cf = util.ChangeFilter(category=None,
branch_re=props['git_branch_name'])
schedDict = {}
schedDict['prs'] = getPullRequestScheduler(props, pretty_branch_name)
for build_type in ["Build", "Reports"]:
for jdk in common.getJDKBuilds(props):
sched = _getAnyBranchScheduler(
name=common.getBuildWithJDK(pretty_branch_name, build_type,
jdk),
change_filter=branch_cf,
properties=props,
builderNames=[
common.getBuildWithJDK(pretty_branch_name, build_type,
jdk),
])
schedDict[build_type + str(jdk)] = sched
sched = _getAnyBranchScheduler(
name=pretty_branch_name + " Quick Build",
change_filter=branch_cf,
properties=props,
builderNames=[
pretty_branch_name + " Markdown",
# pretty_branch_name + " Database Tests"
])
schedDict["markdowndb"] = sched
if str(props['package_all']).lower() != 'true':
sched = schedulers.Nightly(
name=pretty_branch_name + ' Package Generation',
change_filter=branch_cf,
hour={{nightly_build_hour}},
onlyIfChanged=True,
properties=props,
builderNames=[
pretty_branch_name + " Debian Packaging",
pretty_branch_name + " el7 RPM Packaging",
pretty_branch_name + " el8 RPM Packaging"
])
else:
defaultJDK = common.getJDKBuilds(props)[0]
sched = schedulers.Dependent(
name=pretty_branch_name + " Packaging Generation",
upstream=schedDict["Build" + str(defaultJDK)],
properties=props,
builderNames=[
pretty_branch_name + " Debian Packaging",
pretty_branch_name + " el7 RPM Packaging",
pretty_branch_name + " el8 RPM Packaging"
])
schedDict['package'] = sched
return schedDict
def scheduler_nightly(name, builders, template_dfl, codebases, hour, minute):
return schedulers.Nightly(name=name,
codebases=codebases,
properties={'template': template_dfl},
builderNames=builders,
hour=hour,
minute=minute,
onlyIfChanged=True)
def get(builders):
scheds = []
# pull request scheduler
scheds.append(
schedulers.AnyBranchScheduler(
name="gr_pull_request_handler",
change_filter=util.ChangeFilter(category='pull',
project="gnuradio/gnuradio"),
treeStableTimer=None,
builderNames=[
b.name for b in builders if "control" in b.tags
and "gnuradio" in b.tags and "pull" in b.tags
]))
scheds.append(
schedulers.AnyBranchScheduler(
name="volk_pull_request_handler",
change_filter=util.ChangeFilter(category='pull',
project="gnuradio/volk"),
treeStableTimer=None,
builderNames=[
b.name for b in builders if "control" in b.tags
and "volk" in b.tags and "pull" in b.tags
]))
# push event scheduler
def filter_for_push(change):
event = change.properties.getProperty("event")
project = change.properties.getProperty("project")
if event == "push":
return True
return False
scheds.append(
schedulers.AnyBranchScheduler(
name="commit_push_handler",
change_filter=util.ChangeFilter(filter_fn=filter_for_push,
project="gnuradio/gnuradio"),
treeStableTimer=60,
builderNames=[
b.name for b in builders
if "control" in b.tags and "push" in b.tags
]))
scheds.append(
schedulers.ForceScheduler(
name="force_pullrequest",
builderNames=["pull_request_runner"],
properties=[
util.StringParameter(name="github.number",
label="GitHub pull request number",
default="",
size=80),
util.StringParameter(name="github.base.ref",
label="pull request base branch",
default="master",
size=80)
],
codebases=[
util.CodebaseParameter(
"",
project=util.FixedParameter(name="project",
default="gnuradio/gnuradio"),
repository=util.FixedParameter(
name="repository",
default="https://github.com/gnuradio/gnuradio.git"),
branch=util.StringParameter(
name="branch",
label="pull request branch",
default="refs/pull/<PR#>/merge",
size=80),
revision=util.FixedParameter(name="revision", default=""))
]))
scheds.append(
schedulers.ForceScheduler(
name="force_build",
builderNames=["repo_push_runner"],
codebases=[
util.CodebaseParameter(
"",
project=util.FixedParameter(name="project",
default="gnuradio/gnuradio"),
repository=util.FixedParameter(
name="repository",
default="https://github.com/gnuradio/gnuradio.git"),
)
]))
scheds.append(
schedulers.ForceScheduler(
name="force_weekly",
builderNames=["weekly_runner"],
codebases=[
util.CodebaseParameter(
"",
project=util.FixedParameter(name="project",
default="gnuradio/gnuradio"),
repository=util.FixedParameter(
name="repository",
default="https://github.com/gnuradio/gnuradio.git"),
branch=util.StringParameter(name="branch",
label="test branch",
default="master",
size=80),
revision=util.FixedParameter(name="revision", default=""))
]))
scheds.append(
schedulers.Nightly(name="timed_weekly",
builderNames=["weekly_runner"],
codebases={
"": {
"repository":
"https://github.com/gnuradio/gnuradio.git",
"branch": "master",
"revision": "None"
}
},
dayOfWeek=[0, 4],
hour=4,
minute=0))
scheds.extend([
schedulers.Triggerable(name="trigger_" + b.name.lstrip("build_"),
builderNames=[b.name]) for b in builders
if "build" in b.tags
])
scheds.extend([
schedulers.Triggerable(name="trigger_" + b.name.lstrip("test_"),
builderNames=[b.name]) for b in builders
if "test" in b.tags
])
return scheds
launchTime = 20
for branch_item in NIGHTLY_BRANCHES_LIST:
nightlyProperties = DEFAULT_NIGHTLY_PROPERTIES.copy()
nightlyProperties["name"] = "nightly_test_{}".format(branch_item["branch"])
nightlyProperties['owners'] = constants.NIGHTLY_MAIL_LIST
nightlyProperties['buildHosts'] = ["bb-host"]
nightlyProperties['test_set'] = branch_item["test_set"]
nightlyProperties['cmake_flags'] = constants.DEFAULT_DAILY_TEST_CMAKE_FLAGS
nightlyProperties["targetInitMode"] = TargetInitOptions.GENERATE
nightlyScheduler = schedulers.Nightly(
name="build_and_test_{}_nightly".format(branch_item["branch"]),
builderNames=["build_and_test_parall"],
hour=launchTime % 24,
minute=0,
codebases={
"": {
"branch": branch_item["branch"],
"repository": constants.MAXSCALE_REPOSITORY
}
},
properties=nightlyProperties)
SCHEDULERS.append(nightlyScheduler)
launchTime += BUILD_INTERVAL
# Add scheduler for test with Valgrind
BUILD_INTERVAL = 8
launchTime = 8
for branch_item in VALGRIND_BRANCHES_LIST:
nightlyProperties = DEFAULT_NIGHTLY_PROPERTIES.copy()
nightlyProperties["name"] = "valgrind_test_{}".format(
branch_item["branch"])
#
# SCHEDULERS
#
# Configure the Schedulers, which decide how to react to incoming changes.
#
repo_sync_nightly_sched = schedulers.Nightly(
name='repo-sync-nightly-update',
builderNames=[
repo_sync_builder.name,
],
dayOfWeek='1,2,3,4,5', # only work days: from Monday (1) to Friday (5)
hour=0,
minute=0, # at 00:00
codebases={
"": {
"repository": setup.clipos_manifest_git_url,
"branch": "master",
}
},
properties={
"cleanup_workspace": True,
},
)
# CLIP OS builds schedulers:
clipos_incremental_build_intraday_sched = schedulers.Nightly(
name='clipos-master-intraday-incremental-build',
builderNames=[
reference_clipos_builder.name,
],
def ros_testbuild(c, job_name, url, branch, distro, arch, rosdistro, machines,
othermirror, keys, source=True, locks=[]):
# Create a Job for Source
if source:
project_name = '_'.join([job_name, rosdistro, 'source_build'])
c['change_source'].append(
GitPoller(
repourl=url,
name=url,
branch=branch,
category=project_name,
pollAtLaunch=True,
)
)
c['schedulers'].append(
schedulers.SingleBranchScheduler(
name=project_name,
builderNames=[project_name,],
change_filter=util.ChangeFilter(category=project_name)
)
)
c['schedulers'].append(
schedulers.Nightly(
name = project_name+'-nightly-master',
codebases = {url:{'repository':url,'branch':'master'}},
builderNames = [project_name,],
hour=3,
minute=0,
)
)
c['schedulers'].append(
schedulers.Nightly(
name = project_name+'-nightly-develop',
codebases = {url:{'repository':url,'branch':'develop'}},
builderNames = [project_name,],
hour=5,
minute=0,
)
)
c['schedulers'].append(
schedulers.ForceScheduler(
name=project_name+'-force',
codebases = [util.CodebaseParameter("",
branch=util.ChoiceStringParameter(
name="branch",
choices=["master", "devel"],
default="master"),
repository=util.FixedParameter(name="repository", default=url),
)],
builderNames=[project_name,],
)
)
else:
r_owner, r_name = (url.split(':')[1])[:-4].split('/')
project_name = '_'.join([job_name, rosdistro, 'pr_build'])
c['change_source'].append(
GitPRPoller(
owner=r_owner,
repo=r_name,
category=project_name,
branches=[branch],
pollInterval=10*60,
pollAtLaunch=True,
token=util.Secret("OathToken"),
repository_type='ssh'
)
)
c['schedulers'].append(
schedulers.SingleBranchScheduler(
name=project_name,
builderNames=[project_name,],
change_filter=util.ChangeFilter(category=project_name)
)
)
# Directory which will be bind-mounted
binddir = '/tmp/'+project_name
dockerworkdir = '/tmp/test/'
f = BuildFactory()
# Remove any old crud in build/src folder
f.addStep(
ShellCommand(
name='rm src',
command=['rm', '-rf', 'build/src'],
hideStepIf=success,
workdir=Interpolate('%(prop:builddir)s')
)
)
# Check out repository (to /build/src)
f.addStep(
Git(
repourl=util.Property('repository', default=url),
branch=util.Property('branch', default=branch),
alwaysUseLatest=True,
mode='full',
workdir=Interpolate('%(prop:builddir)s/build/src')
)
)
# Download testbuild_docker.py script from master
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='scripts/testbuild_docker.py',
workerdest=('testbuild_docker.py'),
hideStepIf=success
)
)
# Download Dockerfile_test script from master
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='docker_components/Dockerfile_test',
workerdest=('Dockerfile_test'),
hideStepIf=success
)
)
# Download docker-compose.py script from master
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='docker_components/docker-compose-test.yaml',
workerdest=('docker-compose-test.yaml'),
hideStepIf=success
)
)
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='docker_components/rosdep_private.yaml',
workerdest=('rosdep_private.yaml'),
hideStepIf=success
)
)
f.addStep(
FileDownload(
name=job_name+'-grab-script',
mastersrc='scripts/docker-container.py',
workerdest=('docker-container.py'),
hideStepIf=success
)
)
# create docker work environment
f.addStep(
ShellCommand(
command=['python','docker-container.py', job_name],
hideStepIf=success,
workdir=Interpolate('%(prop:builddir)s/build/')
)
)
# Make and run tests in a docker container
f.addStep(
ShellCommand(
name=job_name+'-build',
command=['docker', 'run',
'-v', 'ros-buildbot-docker_deb_repository:/home/package',
'--name='+project_name,
'scalable-env:'+job_name,
'python', '/tmp/build/testbuild_docker.py', binddir,
rosdistro],
descriptionDone=['make and test', job_name]
)
)
f.addStep(
ShellCommand(
name=job_name+'-copytestresults',
command=['docker', 'cp', project_name + ':' +binddir + '/testresults',
'testresults'],
logfiles={'tests': 'testresults'},
descriptionDone=['testresults', job_name]
)
)
f.addStep(
ShellCommand(
name=job_name+'-rm_container',
command=['docker', 'rm', project_name],
descriptionDone=['remove docker container', job_name]
)
)
f.addStep(
ShellCommand(
name=job_name+'-rm_image',
command=['docker', 'image', 'rm', 'scalable-env:'+job_name],
descriptionDone=['remove docker image', job_name]
)
)
c['builders'].append(
BuilderConfig(
name=project_name,
workernames=machines,
factory=f,
locks=locks
)
)
# return the name of the job created
return project_name
def ros_sysbuild(c, job_name, rosdistro, machines, source=True, locks=[]):
# Create a Job for system test job_name = sys_name
project_name = '_'.join([job_name, rosdistro, 'system_build'])
c['schedulers'].append(
schedulers.SingleBranchScheduler(
name=project_name,
builderNames=[
project_name,
],
change_filter=util.ChangeFilter(category=project_name)))
c['schedulers'].append(
schedulers.Nightly(
name=project_name + '-nightly-master',
builderNames=[
project_name,
],
hour=4,
minute=0,
))
# Directory which will be bind-mounted
binddir = '/tmp'
rosinstall_url = "https://raw.githubusercontent.com/ipa-rwu/scalable_system_setup/master/config/" + job_name + ".rosinstall"
f = BuildFactory()
# Remove any old crud in /tmp folder
f.addStep(
ShellCommand(name='rm src',
command=['rm', '-rf', 'scalable_ws'],
hideStepIf=success,
workdir=Interpolate('%(prop:builddir)s/build/')))
# wstool init src .rosinstall
f.addStep(
ShellCommand(
haltOnFailure=True,
name='wstool_rosintall',
command=['wstool', 'init', 'src', rosinstall_url],
hideStepIf=success,
workdir=Interpolate('%(prop:builddir)s/build/scalable_ws')))
# Download Dockerfile_sys.py script from master
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='docker_components/Dockerfile_sys',
workerdest=('Dockerfile_sys'),
hideStepIf=success))
# Download docker-compose-sys.py script from master
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='docker_components/docker-compose-sys.yaml',
workerdest=('docker-compose-sys.yaml'),
hideStepIf=success))
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='docker_components/rosdep_private.yaml',
workerdest=('rosdep_private.yaml'),
hideStepIf=success))
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='scripts/docker-container.py',
workerdest=('docker-container.py'),
hideStepIf=success))
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='shell/uplode_docker_image.sh',
workerdest=('upload_docker_image.sh'),
hideStepIf=success))
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='scripts/unique_docker_sys.py',
workerdest=('unique_docker_sys.py'),
mode=0o755,
hideStepIf=success))
f.addStep(
FileDownload(name=job_name + '-grab-script',
mastersrc='shell/test_sys.sh',
workerdest=('test_sys.sh'),
mode=0o755,
hideStepIf=success))
# reedit docker-compose-deb.yaml
f.addStep(
ShellCommand(haltOnFailure=True,
name=job_name + '-reedit-docker-compose',
command=[
'python', 'unique_docker_sys.py',
'docker-compose-sys.yaml',
Interpolate(job_name)
],
workdir=Interpolate('%(prop:builddir)s/build/'),
descriptionDone=['reedit docker-compose', job_name]))
# Build docker image for creating debian
f.addStep(
ShellCommand(haltOnFailure=True,
name=job_name + '-create_docker_image',
command=[
'docker-compose', '-f', 'docker-compose-sys.yaml',
'build'
],
workdir=Interpolate('%(prop:builddir)s/build/'),
descriptionDone=['sourcedeb', job_name]))
# Make and run tests in a docker container
f.addStep(
ShellCommand(name=job_name + '-test_system',
command=[
'docker', 'run', '--name=' + project_name,
'scalable-sys:' + job_name, 'bash',
'/usr/local/sbin/test_sys.sh'
],
descriptionDone=['make and test', job_name]))
f.addStep(
ShellCommand(name=job_name + '-upload_docker_image',
command=[
'bash', 'upload_docker_image.sh', project_name,
binddir, job_name
],
descriptionDone=['upload_docker_image', job_name],
workdir=Interpolate('%(prop:builddir)s/build/')))
f.addStep(
ShellCommand(name=job_name + '-rm_container',
command=['docker', 'rm', project_name],
descriptionDone=['remove docker container', job_name]))
f.addStep(
ShellCommand(
name=job_name + '-rm_image',
command=['docker', 'image', 'rm', 'scalable-sys:' + job_name],
descriptionDone=['remove docker image', job_name]))
c['builders'].append(
BuilderConfig(name=project_name,
workernames=machines,
factory=f,
locks=locks))
# return the name of the job created
return project_name
properties=properties.extractDefaultValues(
[properties.buildBoxCheckboxContainer()]))
SCHEDULERS = [MANUAL_SCHEDULER, TRIGGERABLE_SCHEDULER]
# Add schedulers for every active branch to be built every night
# The list of branches is defined by constants.NIGHTLY_SCHEDS
# (see maxscale/config/constants.py)
i = 0
for branch in constants.NIGHTLY_SCHEDS:
nightlyProperties = properties.extractDefaultValues(BUILD_ALL_PROPERTIES)
nightlyProperties['target'] = branch
nightlyProperties['host'] = "bb-host"
nightlyProperties['owners'] = constants.NIGHTLY_MAIL_LIST
nightlyScheduler = schedulers.Nightly(name=branch,
builderNames=['build_all'],
hour=4 + i,
minute=0,
codebases={
"": {
"branch":
branch,
"repository":
constants.MAXSCALE_REPOSITORY
}
},
properties=nightlyProperties)
SCHEDULERS.append(nightlyScheduler)
i = i + 1
from buildbot.plugins import schedulers
NIGHTLY_SCHEDULER = schedulers.Nightly(
name="generate_and_sync_repod",
builderNames=["generate_and_sync_repod"],
hour=9,
minute=00,
)
MANUAL_SCHEDULER = schedulers.ForceScheduler(
name="generate_and_sync_repod_manually",
buttonName="Start generation",
builderNames=["generate_and_sync_repod"])
SCHEDULERS = [NIGHTLY_SCHEDULER, MANUAL_SCHEDULER]
properties=ON_PUSH_PROPERTIES)
SCHEDULERS = [MANUAL_SCHEDULER, CHANGE_SOURCE_SCHEDULER]
# Add schedulers for every active branch to be built every night
# The list of branches is defined by constants.NIGHTLY_SCHEDS
# (see maxscale/config/constants.py)
BUILD_INTERVAL = 1
launchTime = 18
for branch in constants.NIGHTLY_SCHEDS:
nightlyProperties = properties.extractDefaultValues(
BUILD_AND_PERFORMANCE_TEST_PROPERTIES)
nightlyProperties["name"] = "nightly_test_{}".format(branch)
nightlyProperties['owners'] = constants.NIGHTLY_MAIL_LIST
nightlyProperties["targetInitMode"] = TargetInitOptions.GENERATE
nightlyScheduler = schedulers.Nightly(
name="build_and_performance_test_{}_nightly".format(branch),
builderNames=["build_and_performance_test"],
hour=launchTime % 24,
minute=0,
codebases={
"": {
"branch": branch,
"repository": constants.MAXSCALE_REPOSITORY
}
},
properties=nightlyProperties)
SCHEDULERS.append(nightlyScheduler)
launchTime += BUILD_INTERVAL
