def scheduler_nightly_wintools(name, builders, codebases, hour, minute): return schedulers.Nightly(name=name, codebases=codebases, builderNames=builders, hour=hour, minute=minute, onlyIfChanged=True)
def _getBasicSchedulers(props): pretty_branch_name = props['branch_pretty'] branch_cf = util.ChangeFilter(category=None, branch_re=props['git_branch_name']) schedDict = {} schedDict['prs'] = getPullRequestScheduler(props, pretty_branch_name) for build_type in ["Build", "Reports"]: for jdk in common.getJDKBuilds(props): sched = _getAnyBranchScheduler( name=common.getBuildWithJDK(pretty_branch_name, build_type, jdk), change_filter=branch_cf, properties=props, builderNames=[ common.getBuildWithJDK(pretty_branch_name, build_type, jdk), ]) schedDict[build_type + str(jdk)] = sched sched = _getAnyBranchScheduler( name=pretty_branch_name + " Quick Build", change_filter=branch_cf, properties=props, builderNames=[ pretty_branch_name + " Markdown", # pretty_branch_name + " Database Tests" ]) schedDict["markdowndb"] = sched if str(props['package_all']).lower() != 'true': sched = schedulers.Nightly( name=pretty_branch_name + ' Package Generation', change_filter=branch_cf, hour={{nightly_build_hour}}, onlyIfChanged=True, properties=props, builderNames=[ pretty_branch_name + " Debian Packaging", pretty_branch_name + " el7 RPM Packaging", pretty_branch_name + " el8 RPM Packaging" ]) else: defaultJDK = common.getJDKBuilds(props)[0] sched = schedulers.Dependent( name=pretty_branch_name + " Packaging Generation", upstream=schedDict["Build" + str(defaultJDK)], properties=props, builderNames=[ pretty_branch_name + " Debian Packaging", pretty_branch_name + " el7 RPM Packaging", pretty_branch_name + " el8 RPM Packaging" ]) schedDict['package'] = sched return schedDict
def scheduler_nightly(name, builders, template_dfl, codebases, hour, minute): return schedulers.Nightly(name=name, codebases=codebases, properties={'template': template_dfl}, builderNames=builders, hour=hour, minute=minute, onlyIfChanged=True)
def get(builders): scheds = [] # pull request scheduler scheds.append( schedulers.AnyBranchScheduler( name="gr_pull_request_handler", change_filter=util.ChangeFilter(category='pull', project="gnuradio/gnuradio"), treeStableTimer=None, builderNames=[ b.name for b in builders if "control" in b.tags and "gnuradio" in b.tags and "pull" in b.tags ])) scheds.append( schedulers.AnyBranchScheduler( name="volk_pull_request_handler", change_filter=util.ChangeFilter(category='pull', project="gnuradio/volk"), treeStableTimer=None, builderNames=[ b.name for b in builders if "control" in b.tags and "volk" in b.tags and "pull" in b.tags ])) # push event scheduler def filter_for_push(change): event = change.properties.getProperty("event") project = change.properties.getProperty("project") if event == "push": return True return False scheds.append( schedulers.AnyBranchScheduler( name="commit_push_handler", change_filter=util.ChangeFilter(filter_fn=filter_for_push, project="gnuradio/gnuradio"), treeStableTimer=60, builderNames=[ b.name for b in builders if "control" in b.tags and "push" in b.tags ])) scheds.append( schedulers.ForceScheduler( name="force_pullrequest", builderNames=["pull_request_runner"], properties=[ util.StringParameter(name="github.number", label="GitHub pull request number", default="", size=80), util.StringParameter(name="github.base.ref", label="pull request base branch", default="master", size=80) ], codebases=[ util.CodebaseParameter( "", project=util.FixedParameter(name="project", default="gnuradio/gnuradio"), repository=util.FixedParameter( name="repository", default="https://github.com/gnuradio/gnuradio.git"), branch=util.StringParameter( name="branch", label="pull request branch", default="refs/pull/<PR#>/merge", size=80), revision=util.FixedParameter(name="revision", default="")) ])) scheds.append( schedulers.ForceScheduler( name="force_build", builderNames=["repo_push_runner"], codebases=[ util.CodebaseParameter( "", project=util.FixedParameter(name="project", default="gnuradio/gnuradio"), repository=util.FixedParameter( name="repository", default="https://github.com/gnuradio/gnuradio.git"), ) ])) scheds.append( schedulers.ForceScheduler( name="force_weekly", builderNames=["weekly_runner"], codebases=[ util.CodebaseParameter( "", project=util.FixedParameter(name="project", default="gnuradio/gnuradio"), repository=util.FixedParameter( name="repository", default="https://github.com/gnuradio/gnuradio.git"), branch=util.StringParameter(name="branch", label="test branch", default="master", size=80), revision=util.FixedParameter(name="revision", default="")) ])) scheds.append( schedulers.Nightly(name="timed_weekly", builderNames=["weekly_runner"], codebases={ "": { "repository": "https://github.com/gnuradio/gnuradio.git", "branch": "master", "revision": "None" } }, dayOfWeek=[0, 4], hour=4, minute=0)) scheds.extend([ schedulers.Triggerable(name="trigger_" + b.name.lstrip("build_"), builderNames=[b.name]) for b in builders if "build" in b.tags ]) scheds.extend([ schedulers.Triggerable(name="trigger_" + b.name.lstrip("test_"), builderNames=[b.name]) for b in builders if "test" in b.tags ]) return scheds
launchTime = 20 for branch_item in NIGHTLY_BRANCHES_LIST: nightlyProperties = DEFAULT_NIGHTLY_PROPERTIES.copy() nightlyProperties["name"] = "nightly_test_{}".format(branch_item["branch"]) nightlyProperties['owners'] = constants.NIGHTLY_MAIL_LIST nightlyProperties['buildHosts'] = ["bb-host"] nightlyProperties['test_set'] = branch_item["test_set"] nightlyProperties['cmake_flags'] = constants.DEFAULT_DAILY_TEST_CMAKE_FLAGS nightlyProperties["targetInitMode"] = TargetInitOptions.GENERATE nightlyScheduler = schedulers.Nightly( name="build_and_test_{}_nightly".format(branch_item["branch"]), builderNames=["build_and_test_parall"], hour=launchTime % 24, minute=0, codebases={ "": { "branch": branch_item["branch"], "repository": constants.MAXSCALE_REPOSITORY } }, properties=nightlyProperties) SCHEDULERS.append(nightlyScheduler) launchTime += BUILD_INTERVAL # Add scheduler for test with Valgrind BUILD_INTERVAL = 8 launchTime = 8 for branch_item in VALGRIND_BRANCHES_LIST: nightlyProperties = DEFAULT_NIGHTLY_PROPERTIES.copy() nightlyProperties["name"] = "valgrind_test_{}".format( branch_item["branch"])
# # SCHEDULERS # # Configure the Schedulers, which decide how to react to incoming changes. # repo_sync_nightly_sched = schedulers.Nightly( name='repo-sync-nightly-update', builderNames=[ repo_sync_builder.name, ], dayOfWeek='1,2,3,4,5', # only work days: from Monday (1) to Friday (5) hour=0, minute=0, # at 00:00 codebases={ "": { "repository": setup.clipos_manifest_git_url, "branch": "master", } }, properties={ "cleanup_workspace": True, }, ) # CLIP OS builds schedulers: clipos_incremental_build_intraday_sched = schedulers.Nightly( name='clipos-master-intraday-incremental-build', builderNames=[ reference_clipos_builder.name, ],
def ros_testbuild(c, job_name, url, branch, distro, arch, rosdistro, machines, othermirror, keys, source=True, locks=[]): # Create a Job for Source if source: project_name = '_'.join([job_name, rosdistro, 'source_build']) c['change_source'].append( GitPoller( repourl=url, name=url, branch=branch, category=project_name, pollAtLaunch=True, ) ) c['schedulers'].append( schedulers.SingleBranchScheduler( name=project_name, builderNames=[project_name,], change_filter=util.ChangeFilter(category=project_name) ) ) c['schedulers'].append( schedulers.Nightly( name = project_name+'-nightly-master', codebases = {url:{'repository':url,'branch':'master'}}, builderNames = [project_name,], hour=3, minute=0, ) ) c['schedulers'].append( schedulers.Nightly( name = project_name+'-nightly-develop', codebases = {url:{'repository':url,'branch':'develop'}}, builderNames = [project_name,], hour=5, minute=0, ) ) c['schedulers'].append( schedulers.ForceScheduler( name=project_name+'-force', codebases = [util.CodebaseParameter("", branch=util.ChoiceStringParameter( name="branch", choices=["master", "devel"], default="master"), repository=util.FixedParameter(name="repository", default=url), )], builderNames=[project_name,], ) ) else: r_owner, r_name = (url.split(':')[1])[:-4].split('/') project_name = '_'.join([job_name, rosdistro, 'pr_build']) c['change_source'].append( GitPRPoller( owner=r_owner, repo=r_name, category=project_name, branches=[branch], pollInterval=10*60, pollAtLaunch=True, token=util.Secret("OathToken"), repository_type='ssh' ) ) c['schedulers'].append( schedulers.SingleBranchScheduler( name=project_name, builderNames=[project_name,], change_filter=util.ChangeFilter(category=project_name) ) ) # Directory which will be bind-mounted binddir = '/tmp/'+project_name dockerworkdir = '/tmp/test/' f = BuildFactory() # Remove any old crud in build/src folder f.addStep( ShellCommand( name='rm src', command=['rm', '-rf', 'build/src'], hideStepIf=success, workdir=Interpolate('%(prop:builddir)s') ) ) # Check out repository (to /build/src) f.addStep( Git( repourl=util.Property('repository', default=url), branch=util.Property('branch', default=branch), alwaysUseLatest=True, mode='full', workdir=Interpolate('%(prop:builddir)s/build/src') ) ) # Download testbuild_docker.py script from master f.addStep( FileDownload( name=job_name+'-grab-script', mastersrc='scripts/testbuild_docker.py', workerdest=('testbuild_docker.py'), hideStepIf=success ) ) # Download Dockerfile_test script from master f.addStep( FileDownload( name=job_name+'-grab-script', mastersrc='docker_components/Dockerfile_test', workerdest=('Dockerfile_test'), hideStepIf=success ) ) # Download docker-compose.py script from master f.addStep( FileDownload( name=job_name+'-grab-script', mastersrc='docker_components/docker-compose-test.yaml', workerdest=('docker-compose-test.yaml'), hideStepIf=success ) ) f.addStep( FileDownload( name=job_name+'-grab-script', mastersrc='docker_components/rosdep_private.yaml', workerdest=('rosdep_private.yaml'), hideStepIf=success ) ) f.addStep( FileDownload( name=job_name+'-grab-script', mastersrc='scripts/docker-container.py', workerdest=('docker-container.py'), hideStepIf=success ) ) # create docker work environment f.addStep( ShellCommand( command=['python','docker-container.py', job_name], hideStepIf=success, workdir=Interpolate('%(prop:builddir)s/build/') ) ) # Make and run tests in a docker container f.addStep( ShellCommand( name=job_name+'-build', command=['docker', 'run', '-v', 'ros-buildbot-docker_deb_repository:/home/package', '--name='+project_name, 'scalable-env:'+job_name, 'python', '/tmp/build/testbuild_docker.py', binddir, rosdistro], descriptionDone=['make and test', job_name] ) ) f.addStep( ShellCommand( name=job_name+'-copytestresults', command=['docker', 'cp', project_name + ':' +binddir + '/testresults', 'testresults'], logfiles={'tests': 'testresults'}, descriptionDone=['testresults', job_name] ) ) f.addStep( ShellCommand( name=job_name+'-rm_container', command=['docker', 'rm', project_name], descriptionDone=['remove docker container', job_name] ) ) f.addStep( ShellCommand( name=job_name+'-rm_image', command=['docker', 'image', 'rm', 'scalable-env:'+job_name], descriptionDone=['remove docker image', job_name] ) ) c['builders'].append( BuilderConfig( name=project_name, workernames=machines, factory=f, locks=locks ) ) # return the name of the job created return project_name
def ros_sysbuild(c, job_name, rosdistro, machines, source=True, locks=[]): # Create a Job for system test job_name = sys_name project_name = '_'.join([job_name, rosdistro, 'system_build']) c['schedulers'].append( schedulers.SingleBranchScheduler( name=project_name, builderNames=[ project_name, ], change_filter=util.ChangeFilter(category=project_name))) c['schedulers'].append( schedulers.Nightly( name=project_name + '-nightly-master', builderNames=[ project_name, ], hour=4, minute=0, )) # Directory which will be bind-mounted binddir = '/tmp' rosinstall_url = "https://raw.githubusercontent.com/ipa-rwu/scalable_system_setup/master/config/" + job_name + ".rosinstall" f = BuildFactory() # Remove any old crud in /tmp folder f.addStep( ShellCommand(name='rm src', command=['rm', '-rf', 'scalable_ws'], hideStepIf=success, workdir=Interpolate('%(prop:builddir)s/build/'))) # wstool init src .rosinstall f.addStep( ShellCommand( haltOnFailure=True, name='wstool_rosintall', command=['wstool', 'init', 'src', rosinstall_url], hideStepIf=success, workdir=Interpolate('%(prop:builddir)s/build/scalable_ws'))) # Download Dockerfile_sys.py script from master f.addStep( FileDownload(name=job_name + '-grab-script', mastersrc='docker_components/Dockerfile_sys', workerdest=('Dockerfile_sys'), hideStepIf=success)) # Download docker-compose-sys.py script from master f.addStep( FileDownload(name=job_name + '-grab-script', mastersrc='docker_components/docker-compose-sys.yaml', workerdest=('docker-compose-sys.yaml'), hideStepIf=success)) f.addStep( FileDownload(name=job_name + '-grab-script', mastersrc='docker_components/rosdep_private.yaml', workerdest=('rosdep_private.yaml'), hideStepIf=success)) f.addStep( FileDownload(name=job_name + '-grab-script', mastersrc='scripts/docker-container.py', workerdest=('docker-container.py'), hideStepIf=success)) f.addStep( FileDownload(name=job_name + '-grab-script', mastersrc='shell/uplode_docker_image.sh', workerdest=('upload_docker_image.sh'), hideStepIf=success)) f.addStep( FileDownload(name=job_name + '-grab-script', mastersrc='scripts/unique_docker_sys.py', workerdest=('unique_docker_sys.py'), mode=0o755, hideStepIf=success)) f.addStep( FileDownload(name=job_name + '-grab-script', mastersrc='shell/test_sys.sh', workerdest=('test_sys.sh'), mode=0o755, hideStepIf=success)) # reedit docker-compose-deb.yaml f.addStep( ShellCommand(haltOnFailure=True, name=job_name + '-reedit-docker-compose', command=[ 'python', 'unique_docker_sys.py', 'docker-compose-sys.yaml', Interpolate(job_name) ], workdir=Interpolate('%(prop:builddir)s/build/'), descriptionDone=['reedit docker-compose', job_name])) # Build docker image for creating debian f.addStep( ShellCommand(haltOnFailure=True, name=job_name + '-create_docker_image', command=[ 'docker-compose', '-f', 'docker-compose-sys.yaml', 'build' ], workdir=Interpolate('%(prop:builddir)s/build/'), descriptionDone=['sourcedeb', job_name])) # Make and run tests in a docker container f.addStep( ShellCommand(name=job_name + '-test_system', command=[ 'docker', 'run', '--name=' + project_name, 'scalable-sys:' + job_name, 'bash', '/usr/local/sbin/test_sys.sh' ], descriptionDone=['make and test', job_name])) f.addStep( ShellCommand(name=job_name + '-upload_docker_image', command=[ 'bash', 'upload_docker_image.sh', project_name, binddir, job_name ], descriptionDone=['upload_docker_image', job_name], workdir=Interpolate('%(prop:builddir)s/build/'))) f.addStep( ShellCommand(name=job_name + '-rm_container', command=['docker', 'rm', project_name], descriptionDone=['remove docker container', job_name])) f.addStep( ShellCommand( name=job_name + '-rm_image', command=['docker', 'image', 'rm', 'scalable-sys:' + job_name], descriptionDone=['remove docker image', job_name])) c['builders'].append( BuilderConfig(name=project_name, workernames=machines, factory=f, locks=locks)) # return the name of the job created return project_name
properties=properties.extractDefaultValues( [properties.buildBoxCheckboxContainer()])) SCHEDULERS = [MANUAL_SCHEDULER, TRIGGERABLE_SCHEDULER] # Add schedulers for every active branch to be built every night # The list of branches is defined by constants.NIGHTLY_SCHEDS # (see maxscale/config/constants.py) i = 0 for branch in constants.NIGHTLY_SCHEDS: nightlyProperties = properties.extractDefaultValues(BUILD_ALL_PROPERTIES) nightlyProperties['target'] = branch nightlyProperties['host'] = "bb-host" nightlyProperties['owners'] = constants.NIGHTLY_MAIL_LIST nightlyScheduler = schedulers.Nightly(name=branch, builderNames=['build_all'], hour=4 + i, minute=0, codebases={ "": { "branch": branch, "repository": constants.MAXSCALE_REPOSITORY } }, properties=nightlyProperties) SCHEDULERS.append(nightlyScheduler) i = i + 1
from buildbot.plugins import schedulers NIGHTLY_SCHEDULER = schedulers.Nightly( name="generate_and_sync_repod", builderNames=["generate_and_sync_repod"], hour=9, minute=00, ) MANUAL_SCHEDULER = schedulers.ForceScheduler( name="generate_and_sync_repod_manually", buttonName="Start generation", builderNames=["generate_and_sync_repod"]) SCHEDULERS = [NIGHTLY_SCHEDULER, MANUAL_SCHEDULER]
properties=ON_PUSH_PROPERTIES) SCHEDULERS = [MANUAL_SCHEDULER, CHANGE_SOURCE_SCHEDULER] # Add schedulers for every active branch to be built every night # The list of branches is defined by constants.NIGHTLY_SCHEDS # (see maxscale/config/constants.py) BUILD_INTERVAL = 1 launchTime = 18 for branch in constants.NIGHTLY_SCHEDS: nightlyProperties = properties.extractDefaultValues( BUILD_AND_PERFORMANCE_TEST_PROPERTIES) nightlyProperties["name"] = "nightly_test_{}".format(branch) nightlyProperties['owners'] = constants.NIGHTLY_MAIL_LIST nightlyProperties["targetInitMode"] = TargetInitOptions.GENERATE nightlyScheduler = schedulers.Nightly( name="build_and_performance_test_{}_nightly".format(branch), builderNames=["build_and_performance_test"], hour=launchTime % 24, minute=0, codebases={ "": { "branch": branch, "repository": constants.MAXSCALE_REPOSITORY } }, properties=nightlyProperties) SCHEDULERS.append(nightlyScheduler) launchTime += BUILD_INTERVAL