def __init__(self): ZcashBaseFactory.__init__(self) self.addSteps([ steps.MakeDirectory( dir='build/ibd-datadir', name='Create datadir', ), sh('touch', 'ibd-datadir/zcash.conf', name='Create zcash.conf'), InitialBlockDownloadTimeRunner( 'ibd-datadir', name='time-InitialBlockDownload', ), steps.RemoveDirectory( dir='build/ibd-datadir', name='Remove datadir', alwaysRun=True, ), steps.POST(urllib.parse.urljoin(CODESPEED_URL, '/result/add/json/'), data={'json': getPerfJson}, auth=('buildbot', CODESPEED_PASS), verify=CODESPEED_CERT, doStepIf=lambda s: s.getProperty('publish', False), hideStepIf=lambda results, s: results==util.SKIPPED, ), ])
def build_coverage(): remove_build = steps.RemoveDirectory("build") create_build = steps.MakeDirectory("build") cmake_step = steps.CMake(path=util.Property("src_dir"), definitions=util.Property("cmake_defs", {}), options=util.Property("cmake_opts", []), workdir="build", env=env) @util.renderer def join_make_opts(props): make_opts = props.getProperty("make_opts", []) return ["make"] + make_opts make_step = steps.Compile(command=join_make_opts, workdir="build", env=env) test_coverage = steps.ShellCommand(command=["make", "coverage"], workdir="build") upload_coverage_data = steps.ShellCommand(command=[ "bash", "-c", util.Interpolate("bash <(curl -s https://codecov.io/bash) -t " + tokens.codecovToken + " -C %(prop:revision)s -f coverage.info.cleaned") ], workdir="build") factory = util.BuildFactory() factory.addStep(remove_build) factory.addStep(create_build) factory.addStep(cmake_step) factory.addStep(make_step) factory.addStep(test_coverage) factory.addStep(upload_coverage_data) return factory
def build_volk_PR(): create_src = steps.MakeDirectory(name="create src directory", dir="volk") clone_step = steps.GitHub(name="fetch PR source", repourl=util.Property("repository"), mode="full", method="fresh", submodules=True, retryFetch=True, clobberOnFailure=True, workdir="volk") rm_src_dir = steps.RemoveDirectory( dir=util.Interpolate( os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s", "%(prop:github.base.ref)s")), hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS, ) copy_src = steps.CopyDirectory( name="copy src to srcdir", src="volk", dest=util.Interpolate( os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s", "%(prop:github.base.ref)s"), ), hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS, ) # load builders.json with definitions on how to build things parent_path = os.path.dirname(__file__) with open(os.path.join(parent_path, "volk_builders.json"), "r") as builders_file: build_config = json.loads(builders_file.read()) trigger_builds = custom_steps.BuildTrigger( name="trigger the right builders", build_config=build_config, schedulerNames=["trigger"], runner="pull", set_properties={ "pr_base": util.Property("github.base.ref"), "src_dir": util.Interpolate( os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s")) }, test_merge=False, updateSourceStamp=False, waitForFinish=True) factory = util.BuildFactory() factory.addStep(create_src) factory.addStep(clone_step) factory.addStep(rm_src_dir) factory.addStep(copy_src) factory.addStep(trigger_builds) return factory
def build_coverity(): remove_build = steps.RemoveDirectory("build") remove_src = steps.RemoveDirectory("src") create_build = steps.MakeDirectory("build") download_src_archive = steps.FileDownload( mastersrc=util.Property("src_archive"), workerdest="src.tar.xz", workdir="src") extract_src_archive = steps.ShellCommand( name="Extract source archive", command=["tar", "xJf", "src.tar.xz"], workdir="src") cmake_step = steps.CMake(path="../src/", definitions=util.Property("cmake_defs", {}), options=util.Property("cmake_opts", []), workdir="build", env=env) make_step = steps.Compile(command=[ "cov-build", "--dir", "cov-int", "make", "-j", "16", "-l", "32" ], workdir="build", env=env) compress = steps.ShellCommand( command=["tar", "czvf", "gnuradio.tgz", "cov-int"], workdir="build") upload = steps.ShellCommand(command=[ "curl", "--form", "token=" + tokens.coverityToken, "--form", "[email protected]", "--form", "[email protected]", "--form", util.Interpolate("version=%(prop:revision)s"), "--form", util.Interpolate( "description=\"Weekly Buildbot submission for %(prop:branch)s branch \"" ), "https://scan.coverity.com/builds?project=GNURadio" ], workdir="build") factory = util.BuildFactory() factory.addStep(remove_build) factory.addStep(remove_src) factory.addStep(create_build) factory.addStep(download_src_archive) factory.addStep(extract_src_archive) factory.addStep(cmake_step) factory.addStep(make_step) factory.addStep(compress) factory.addStep(upload) return factory
def build_and_test(): remove_build = steps.RemoveDirectory("build") remove_src = steps.RemoveDirectory("src") create_build = steps.MakeDirectory("build") download_src_archive = steps.FileDownload( mastersrc=util.Property("src_archive"), workerdest="src.tar.xz", workdir="src") extract_src_archive = steps.ShellCommand( name="Extract source archive", command=["tar", "xJf", "src.tar.xz"], workdir="src") cmake_step = steps.CMake(path="../src/", definitions=util.Property("cmake_defs", {}), options=util.Property("cmake_opts", []), workdir="build", env=env) @util.renderer def join_make_opts(props): make_opts = props.getProperty("make_opts", []) return ["make"] + make_opts make_step = steps.Compile(command=join_make_opts, workdir="build", env=env) @util.renderer def parse_test_excludes(props): command = ["ctest", "--output-on-failure", "--timeout", "120"] excludes = props.getProperty('test_excludes', []) excludes.append("qtgui") if excludes is not None: command += ["-E", "|".join(excludes)] return command test_step = steps.Test(command=parse_test_excludes, workdir="build") factory = util.BuildFactory() factory.addStep(remove_build) factory.addStep(remove_src) factory.addStep(create_build) factory.addStep(download_src_archive) factory.addStep(extract_src_archive) factory.addStep(cmake_step) factory.addStep(make_step) factory.addStep(test_step) return factory
def build_and_test(): remove_build = steps.RemoveDirectory("build") create_build = steps.MakeDirectory("build") cmake_step = steps.CMake(path=util.Property("src_dir"), definitions=util.Property("cmake_defs", {}), options=util.Property("cmake_opts", []), workdir="build", env=env) @util.renderer def join_make_opts(props): make_opts = props.getProperty("make_opts", []) return ["make"] + make_opts make_step = steps.Compile(command=join_make_opts, workdir="build", env=env) def parse_exclude_file(rc, stdout, stderr): exclude_tests = json.loads(stdout) return {"test_excludes": exclude_tests} load_exclude_file = steps.SetPropertyFromCommand( command=["cat", os.path.join("/config", "test_excludes.json")], extract_fn=parse_exclude_file, doStepIf=lambda steps: steps.getProperty("exclude_file", False)) @util.renderer def parse_test_excludes(props): command = ["ctest", "--output-on-failure", "--timeout", "10"] excludes = props.getProperty("test_excludes", None) if excludes is not None: command += ["-E", "|".join(excludes)] return command test_step = steps.Test(command=parse_test_excludes, workdir="build") factory = util.BuildFactory() factory.addStep(remove_build) factory.addStep(create_build) factory.addStep(load_exclude_file) factory.addStep(cmake_step) factory.addStep(make_step) factory.addStep(test_step) return factory
def build_PR(): create_src = steps.MakeDirectory(name="create src directory", dir="src") clone_step = steps.GitHub( name="fetch PR source", repourl=util.Property("repository"), mode="full", method="fresh", submodules=True, retryFetch=True, clobberOnFailure=True, workdir="src") rm_src_archive = steps.ShellCommand( name="remove old source archive", command=[ "rm", "-rf", util.Interpolate( os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s.tar.xz")) ], workdir="src", hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS) create_src_archive = steps.ShellCommand( name="create source archive", command=[ "tar", "cJf", util.Interpolate( os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s.tar.xz")), "." ], workdir="src", hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS, ) # load builders.json with definitions on how to build things parent_path = os.path.dirname(__file__) with open(os.path.join(parent_path, "builders.json"), "r") as builders_file: build_config = json.loads(builders_file.read()) trigger_builds = custom_steps.BuildTrigger( name="trigger the right builders", build_config=build_config, schedulerNames=["trigger"], runner="pull", set_properties={ "pr_base": util.Property("github.base.ref"), "src_archive": util.Interpolate( os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s.tar.xz")) }, updateSourceStamp=False, waitForFinish=True) factory = util.BuildFactory() factory.addStep(create_src) factory.addStep(clone_step) factory.addStep(rm_src_archive) factory.addStep(create_src_archive) factory.addStep(trigger_builds) return factory
def build_weekly(): create_src = steps.MakeDirectory(name="create src directory", dir="src") clone_step = steps.GitHub(name="fetch PR source", repourl=util.Property("repository"), mode="full", method="fresh", submodules=True, clobberOnFailure=True, getDescription=True, workdir="src") rm_src_dir = steps.RemoveDirectory(dir=util.Interpolate( os.path.join(_WEEKLY_SRC_BASE, "%(prop:branch)s", "%(prop:commit-description)s"))) copy_src = steps.CopyDirectory(name="copy src to srcdir", src="src", dest=util.Interpolate( os.path.join( _WEEKLY_SRC_BASE, "%(prop:branch)s", "%(prop:commit-description)s"), )) set_merge_property = steps.SetProperty( property=util.Interpolate("merge_%(prop:branch)s"), value=True, hideStepIf=True, ) # load builders.json with definitions on how to build things parent_path = os.path.dirname(__file__) with open(os.path.join(parent_path, "builders.json"), "r") as builders_file: build_config = json.loads(builders_file.read()) # now we have all necessary merge properties together, # we can actually kickoff builds for them trigger_builds = custom_steps.BuildTrigger( name="trigger all builders", build_config=build_config, schedulerNames=["trigger"], runner="time", set_properties={ "src_dir": util.Interpolate( os.path.join(_WEEKLY_SRC_BASE, "%(prop:branch)s", "%(prop:commit-description)s")), "got_revision": util.Property("got_revision"), }, updateSourceStamp=True, waitForFinish=True) factory = util.BuildFactory() factory.addStep(create_src) factory.addStep(clone_step) factory.addStep(rm_src_dir) factory.addStep(copy_src) factory.addStep(set_merge_property) factory.addStep(trigger_builds) return factory
def build_weekly(): create_src = steps.MakeDirectory( name="create src directory", dir="src") clone_step = steps.GitHub( name="fetch PR source", repourl=util.Property("repository"), mode="full", method="fresh", submodules=True, clobberOnFailure=True, getDescription=True, workdir="src") rm_src_archive = steps.ShellCommand( name="remove old source archive", command=[ "rm", "-rf", util.Interpolate( os.path.join(_WEEKLY_SRC_BASE, "%(prop:branch)s", "%(prop:commit-description)s.tar.xz")) ], workdir="src", hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS) create_src_archive = steps.ShellCommand( name="create source archive", command=[ "tar", "cJf", util.Interpolate( os.path.join(_WEEKLY_SRC_BASE, "%(prop:branch)s", "%(prop:commit-description)s.tar.xz")), "." ], workdir="src", hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS, ) # load builders.json with definitions on how to build things parent_path = os.path.dirname(__file__) with open(os.path.join(parent_path, "builders.json"), "r") as builders_file: build_config = json.loads(builders_file.read()) # now we have all necessary merge properties together, # we can actually kickoff builds for them trigger_builds = custom_steps.BuildTrigger( name="trigger all builders", build_config=build_config, schedulerNames=["trigger"], runner="time", set_properties={ "src_archive": util.Interpolate( os.path.join(_WEEKLY_SRC_BASE, "%(prop:branch)s","%(prop:commit-description)s.tar.xz")), "got_revision": util.Property("got_revision"), }, updateSourceStamp=True, waitForFinish=True ) factory = util.BuildFactory() factory.addStep(create_src) factory.addStep(clone_step) factory.addStep(rm_src_archive) factory.addStep(create_src_archive) factory.addStep(trigger_builds) return factory
def build_PR(): # @util.renderer # def check_mergeable(props): # mergeable = props.getProperty("github.mergeable", False) # return mergeable # check_mergeables = steps.Assert( # check_mergeable, # name="check if PR was mergeable", # haltOnFailure=True # ) create_src = steps.MakeDirectory(name="create src directory", dir="src") clone_step = steps.GitHub(name="fetch PR source", repourl=util.Property("repository"), mode="full", method="fresh", submodules=True, retryFetch=True, clobberOnFailure=True, workdir="src") set_merge_property = steps.SetProperty( name="set merge property", property=util.Interpolate("merge_%(prop:github.base.ref)s"), value=True, hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS, ) create_merge_branch = steps.ShellCommand( name="create test_merge branch for further steps", command=[ "git", "branch", "-f", util.Interpolate("test_merge_%(prop:github.base.ref)s") ], workdir="src") rm_src_dir = steps.RemoveDirectory( dir=util.Interpolate( os.path.join(_BASEPATH, "pull", "%(prop:github.number)s", "%(prop:github.base.ref)s")), hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS, ) copy_src = steps.CopyDirectory( name="copy src to srcdir", src="src", dest=util.Interpolate( os.path.join(_BASEPATH, "pull", "%(prop:github.number)s", "%(prop:github.base.ref)s"), ), hideStepIf=lambda results, s: results == SKIPPED or results == SUCCESS, ) master_steps = sequences.mergeability_sequence("master", "maint", _PULL_SRC_BASE) next_steps = sequences.mergeability_sequence("next", "master", _PULL_SRC_BASE) python3_steps = sequences.mergeability_sequence("python3", "next", _PULL_SRC_BASE) # load builders.json with definitions on how to build things parent_path = os.path.dirname(__file__) with open(os.path.join(parent_path, "builders.json"), "r") as builders_file: build_config = json.loads(builders_file.read()) trigger_builds = custom_steps.BuildTrigger( name="trigger the right builders", build_config=build_config, schedulerNames=["trigger"], runner="pull", set_properties={ "pr_base": util.Property("github.base.ref"), "src_dir": util.Interpolate( os.path.join(_PULL_SRC_BASE, "%(prop:github.number)s")) }, updateSourceStamp=False, waitForFinish=True) factory = util.BuildFactory() factory.addStep(create_src) factory.addStep(clone_step) factory.addStep(set_merge_property) factory.addStep(create_merge_branch) factory.addStep(rm_src_dir) factory.addStep(copy_src) factory.addSteps(master_steps) factory.addSteps(next_steps) factory.addSteps(python3_steps) factory.addStep(trigger_builds) return factory