def __init__(self, branch, python, projects, *a, **kw): BuildFactory.__init__(self, *a, **kw) self.addStep( Mercurial, repourl="http://hg.python.org/cpython", defaultBranch=branch, branchType='inrepo', mode="copy") self.addStep( ShellCommand, name="configure-python", description=["configuring", "python"], descriptionDone=["configure", "python"], command="./configure --prefix=$PWD/install") self.addStep( ShellCommand, name="install-python", description=["installing", "python"], descriptionDone=["install", "python"], command=["make", "install"]) pythonc = "install/bin/" + python self.addStep( ShellCommand, name="link-binary", description=["linking", "binary"], descriptionDone=["link", "binary"], command=["ln", "-nsf", "build/" + pythonc, "python"], workdir=".") self.buildModules(pythonc, projects)
def __init__(self, git_url, cmd, buildout_content): BuildFactory.__init__(self) self.addStep(PgSafeStopSb()) self.addStep(CleanUp()) self.addStep(CreateSandbox()) self.addStep(Clone(git_url, 'pkg')) self.addStep(BootstrapSb()) # Change the working directory of the next build steps wd = os.path.join(self.workdir, "pkg") self.addStep(CreateFile(self.buildout_file, buildout_content, workdir=wd)) self.addStep(Buildout(cmd_args=['-U', '-c', self.buildout_file])) self.addStep(PgStartSb()) self.addStep(ShellCommand( name="custom_cmd", haltOnFailure=False, flunkOnFailure=True, workdir=wd, command=cmd )) self.addStep(PgStopSb())
def create_single_worker_two_builder_config(self, controller_kwargs=None): if not controller_kwargs: controller_kwargs = {} controller = LatentController(self, 'local', **controller_kwargs) config_dict = { 'builders': [ BuilderConfig( name="testy-1", workernames=["local"], factory=BuildFactory(), ), BuilderConfig( name="testy-2", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, # Disable checks about missing scheduler. 'multiMaster': True, } master = yield self.getMaster(config_dict) builder_ids = [ (yield master.data.updates.findBuilderId('testy-1')), (yield master.data.updates.findBuilderId('testy-2')), ] return controller, master, builder_ids
def masterConfig(extra_config): c = {} from buildbot.config import BuilderConfig from buildbot.process.buildstep import BuildStep from buildbot.process.factory import BuildFactory from buildbot.process import results class MyBuildStep(BuildStep): def start(self): self.finished(results.SUCCESS) c['change_source'] = [] c['schedulers'] = [] # filled in above f1 = BuildFactory() f1.addStep(MyBuildStep(name='one')) f1.addStep(MyBuildStep(name='two')) c['builders'] = [ BuilderConfig(name="a", workernames=["local1"], factory=f1), ] c['status'] = [] c['title'] = "test" c['titleURL'] = "test" c['buildbotURL'] = "http://localhost:8010/" c['mq'] = {'debug': True} # test wants to influence the config, but we still return a new config # each time c.update(extra_config) return c
def triggeredBuildIsNotCreated(): f2 = BuildFactory() f2.addStep(steps.ShellCommand(command="echo 'hello'")) def nextBuild(*args, **kwargs): return defer.succeed(None) return setupTriggerConfiguration(f2, nextBuild=nextBuild)
def setupTriggerConfiguration(triggeredFactory, nextBuild=None): c = {} c['schedulers'] = [ schedulers.Triggerable( name="trigsched", builderNames=["triggered"]), schedulers.AnyBranchScheduler( name="sched", builderNames=["main"])] f = BuildFactory() f.addStep(steps.Trigger(schedulerNames=['trigsched'], waitForFinish=True, updateSourceStamp=True)) f.addStep(steps.ShellCommand(command='echo world')) mainBuilder = BuilderConfig(name="main", workernames=["local1"], factory=f) triggeredBuilderKwargs = {'name': "triggered", 'workernames': ["local1"], 'factory': triggeredFactory} if nextBuild is not None: triggeredBuilderKwargs['nextBuild'] = nextBuild triggeredBuilder = BuilderConfig(**triggeredBuilderKwargs) c['builders'] = [mainBuilder, triggeredBuilder] return c
def masterConfig(secret_specifier): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] # note that as of December 2018, the vault docker image default to kv # version 2 to be enabled by default c['secretsProviders'] = [ HashiCorpVaultSecretProvider(vaultToken='my_vaulttoken', vaultServer="http://localhost:8200", apiVersion=2) ] f = BuildFactory() f.addStep( ShellCommand(command=Interpolate('echo -n {} | base64'.format( secret_specifier)))) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def test_addSteps(self): factory = BuildFactory() factory.addSteps([BuildStep(), BuildStep()]) self.assertEqual( factory.steps, [_BuildStepFactory(BuildStep), _BuildStepFactory(BuildStep)])
def masterConfig(): c = {} from buildbot.worker import Worker from buildbot.config import BuilderConfig from buildbot.process.buildstep import BuildStep from buildbot.process.factory import BuildFactory from buildbot.process import results class MyBuildStep(BuildStep): def start(self): self.finished(results.SUCCESS) c['workers'] = [Worker("local1", "localpw")] c['protocols'] = {'pb': {'port': 'tcp:0'}} c['change_source'] = [] c['schedulers'] = [] # filled in above f1 = BuildFactory() f1.addStep(MyBuildStep(name='one')) f1.addStep(MyBuildStep(name='two')) c['builders'] = [ BuilderConfig(name="a", workernames=["local1"], factory=f1), ] c['status'] = [] c['title'] = "test" c['titleURL'] = "test" c['buildbotURL'] = "http://localhost:8010/" c['db'] = {'db_url': "sqlite:///state.sqlite"} c['mq'] = {'debug': True} # test wants to influence the config, but we still return a new config each time c.update(BuildmasterConfig) return c
def test_latent_max_builds(self): """ If max_builds is set, only one build is started on a latent worker at a time. """ controller = LatentController(self, 'local', max_builds=1) step_controller = StepController() config_dict = { 'builders': [ BuilderConfig( name="testy-1", workernames=["local"], factory=BuildFactory([step_controller]), ), BuilderConfig( name="testy-2", workernames=["local"], factory=BuildFactory([step_controller]), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, 'multiMaster': True, } yield self.setup_master(config_dict) builder_ids = [ (yield self.master.data.updates.findBuilderId('testy-1')), (yield self.master.data.updates.findBuilderId('testy-2')), ] started_builds = [] yield self.master.mq.startConsuming( lambda key, build: started_builds.append(build), ('builds', None, 'new')) # Trigger a buildrequest yield self.master.data.updates.addBuildset( waited_for=False, builderids=builder_ids, sourcestamps=[ { 'codebase': '', 'repository': '', 'branch': None, 'revision': None, 'project': '' }, ], ) # The worker fails to substantiate. controller.start_instance(True) yield controller.connect_worker() self.assertEqual(len(started_builds), 1) yield controller.auto_stop(True)
def create_single_worker_two_builder_step_lock_config( self, lock_cls, mode): lock = lock_cls("lock1", maxCount=1) stepcontrollers = [ BuildStepController(locks=[lock.access(mode)]), BuildStepController(locks=[lock.access(mode)]) ] config_dict = { 'builders': [ BuilderConfig(name='builder1', workernames=['worker1'], factory=BuildFactory([stepcontrollers[0].step])), BuilderConfig(name='builder2', workernames=['worker1'], factory=BuildFactory([stepcontrollers[1].step])), ], 'workers': [ self.createLocalWorker('worker1'), ], 'protocols': { 'null': {} }, 'multiMaster': True, } master = yield self.getMaster(config_dict) builder_ids = [ (yield master.data.updates.findBuilderId('builder1')), (yield master.data.updates.findBuilderId('builder2')), ] return stepcontrollers, master, builder_ids
def masterConfig(use_interpolation): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers, steps, util c['services'] = [FakeSecretReporter('http://example.com/hook', auth=('user', Interpolate('%(secret:httppasswd)s')))] c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] c['secretsProviders'] = [FakeSecretStorage( secretdict={"foo": "bar", "something": "more", 'httppasswd': 'myhttppasswd'})] f = BuildFactory() if use_interpolation: if os.name == "posix": # on posix we can also check whether the password was passed to the command command = Interpolate('echo %(secret:foo)s | sed "s/bar/The password was there/"') else: command = Interpolate('echo %(secret:foo)s') else: command = ['echo', util.Secret('foo')] f.addStep(steps.ShellCommand(command=command)) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f)] return c
def __init__( self, python, source, uncleanWarnings, trialTests=None, trialMode=None, virtualenv=False, ): if not isinstance(source, list): source = [source] else: source = list(source) # If permissions get messed up on a slave, this can fix it. # But it breaks on old slaves so it's not enabled all the time # (and it can't fix old slaves, obviously). # self._fixPermissions(source) BuildFactory.__init__(self, source) if type(python) is str: python = [python] self.python = python self.uncleanWarnings = uncleanWarnings self.trialMode = trialMode if trialTests is None: trialTests = [WithProperties("%(test-case-name:~twisted)s")] self.trialTests = trialTests if virtualenv: # Each time we create a new virtualenv as latest pip can build # wheels on the fly and install them from user's cache. self.addStep( shell.ShellCommand, command=[ 'virtualenv', '--clear', '-p', self.python[0], self._virtualEnvPath, ], ) self.addStep( ReportPythonModuleVersions, python=self.python, moduleInfo=[ ("Python", "sys", "sys.version"), ("OpenSSL", "OpenSSL", "OpenSSL.__version__"), ("PyCrypto", "Crypto", "Crypto.__version__"), ("gmpy", "gmpy", "gmpy.version()"), ("SOAPpy", "SOAPpy", "SOAPpy.__version__"), ("ctypes", "ctypes", "ctypes.__version__"), ("gtk", "gtk", "gtk.gtk_version"), ("pygtk", "gtk", "gtk.pygtk_version"), ("pywin32", "win32api", "win32api.GetFileVersionInfo(win32api.__file__, chr(92))['FileVersionLS'] >> 16"), ("pyasn1", "pyasn1", "pyasn1.__version__"), ("cffi", "cffi", "cffi.__version__"), ], pkg_resources=[ ("subunit", "subunit"), ("zope.interface", "zope.interface"), ])
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers from buildbot.plugins import util lock = util.MasterLock("lock") c['schedulers'] = [ schedulers.AnyBranchScheduler( name="sched", builderNames=["testy", "testy2", "testy3"]), ] f = BuildFactory() lockstep = LockedStep(locks=[lock.access('exclusive')]) f.addStep(lockstep) # assert lockstep._factory.buildStep() == lockstep._factory.buildStep() c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f), BuilderConfig(name="testy2", workernames=["local1"], factory=f), BuilderConfig(name="testy3", workernames=["local1"], factory=f)] return c
def masterConfig(): global num_reconfig num_reconfig += 1 c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.schedulers.forcesched import ForceScheduler from buildbot.steps.shell import ShellCommandNewStyle from buildbot.util.service import BuildbotService class MyShellCommand(ShellCommandNewStyle): def getResultSummary(self): service = self.master.service_manager.namedServices['myService'] return dict(step="num reconfig: %d" % (service.num_reconfig, )) class MyService(BuildbotService): name = "myService" def reconfigService(self, num_reconfig): self.num_reconfig = num_reconfig return defer.succeed(None) c['schedulers'] = [ForceScheduler(name="force", builderNames=["testy"])] f = BuildFactory() f.addStep(MyShellCommand(command='echo hei')) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] c['services'] = [MyService(num_reconfig=num_reconfig)] if num_reconfig == 3: c['services'].append( MyService(name="myService2", num_reconfig=num_reconfig)) return c
def __init__(self, steps=None, depends_on_projects=None, **kwargs): # Cannot use "super" here as BuildFactory is an old style class. BuildFactory.__init__(self, steps) if depends_on_projects is None: self.depends_on_projects = frozenset(['llvm']) else: self.depends_on_projects = frozenset(depends_on_projects) # Directories. self.llvm_srcdir = kwargs.pop('llvm_srcdir', None) self.obj_dir = kwargs.pop('obj_dir', None) self.install_dir = kwargs.pop('install_dir', None) # Preserve the rest of the given extra attributes if any, so we could # expand the factory later. for k,v in kwargs.items(): setattr(self, k, v) self.monorepo_dir = self.llvm_srcdir or "llvm-project" self.llvm_srcdir = \ "%(monorepo_dir)s/llvm" % {'monorepo_dir' : self.monorepo_dir} self.obj_dir = \ self.obj_dir or "build" # Repourl_prefix could be specified per builder. Otherwise we use github. self.repourl_prefix = kwargs.pop('repourl_prefix', 'https://github.com/llvm/')
def test_worker_reconfigure_with_new_builder(self): """ Checks if we can successfully reconfigure if we add new builders to worker. """ config_dict = { 'builders': [ BuilderConfig(name="builder1", workernames=['local1'], factory=BuildFactory()), ], 'workers': [self.createLocalWorker('local1', max_builds=1)], 'protocols': { 'null': {} }, # Disable checks about missing scheduler. 'multiMaster': True, } yield self.setup_master(config_dict) config_dict['builders'] += [ BuilderConfig(name="builder2", workernames=['local1'], factory=BuildFactory()), ] config_dict['workers'] = [ self.createLocalWorker('local1', max_builds=2) ] # reconfig should succeed yield self.reconfig_master(config_dict)
def masterConfig(extra_config): c = {} from buildbot.config import BuilderConfig from buildbot.process.buildstep import BuildStep from buildbot.process.factory import BuildFactory from buildbot.process import results class MyBuildStep(BuildStep): def start(self): self.finished(results.SUCCESS) c['change_source'] = [] c['schedulers'] = [] # filled in above f1 = BuildFactory() f1.addStep(MyBuildStep(name='one')) f1.addStep(MyBuildStep(name='two')) c['builders'] = [ BuilderConfig(name="a", workernames=["local1"], factory=f1), ] c['title'] = "test" c['titleURL'] = "test" c['buildbotURL'] = "http://localhost:8010/" c['mq'] = {'debug': True} # test wants to influence the config, but we still return a new config # each time c.update(extra_config) return c
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers c['schedulers'] = [ schedulers.Triggerable( name="trigsched", builderNames=["build"]), schedulers.AnyBranchScheduler( name="sched", builderNames=["testy"])] f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) f.addStep(steps.Trigger(schedulerNames=['trigsched'], waitForFinish=True, updateSourceStamp=True)) f.addStep(steps.ShellCommand(command='echo world')) f2 = BuildFactory() f2.addStep(steps.ShellCommand(command='echo ola')) c['builders'] = [ BuilderConfig(name="testy", slavenames=["local1"], factory=f), BuilderConfig(name="build", slavenames=["local1"], factory=f2)] return c
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers, reporters c['schedulers'] = [ schedulers.AnyBranchScheduler( name="sched", builderNames=["testy"]) ] f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] notifier = reporters.PushoverNotifier('1234', 'abcd', mode="all", watchedWorkers=['local1'], messageFormatter=MessageFormatter(template='This is a message.'), messageFormatterMissingWorker=MessageFormatterMissingWorker( template='No worker.')) c['services'] = [ reporters.MailNotifier("*****@*****.**", mode="all"), notifier ] return c
def __init__( self, python, source, uncleanWarnings, trialTests=None, trialMode=None, virtualenv=False, virtualenv_module="virtualenv", platform="unix", forceGarbageCollection=False, ): if not isinstance(source, list): source = [source] else: source = list(source) # If permissions get messed up on a slave, this can fix it. # But it breaks on old slaves so it's not enabled all the time # (and it can't fix old slaves, obviously). # self._fixPermissions(source) BuildFactory.__init__(self, source) if type(python) is str: python = [python, "-Wall"] assert platform in ["unix", "windows"] self._platform = platform if platform == "unix": self._path = posixpath elif platform == "windows": self._path = ntpath self.python = python self.virtualenv = virtualenv self.uncleanWarnings = uncleanWarnings self.forceGarbageCollection = forceGarbageCollection self.trialMode = trialMode if trialTests is None: trialTests = [WithProperties("%(test-case-name:~twisted)s")] self.trialTests = trialTests if virtualenv: # Hopefully temporary workaround for --clear not working: # https://github.com/pypa/virtualenv/issues/929 self.addStep( shell.ShellCommand, command=self.python + ["-c", "import shutil, sys;" "shutil.rmtree(sys.argv[1], ignore_errors=True)", self._virtualEnvPath], ) self.addStep(shell.ShellCommand, command=self.python + ["-m", virtualenv_module, self._virtualEnvPath]) else: # Report the versions, since we're using the system ones. If it's a # virtualenv, it's up to the venv factory to report the versions # itself. self._reportVersions(python=self.python)
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers, steps c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] c['secretsProviders'] = [FakeSecretStorage( secretdict={"foo": "bar", "something": "more"})] f = BuildFactory() if os.name == "posix": f.addStep(steps.ShellCommand(command=Interpolate( 'echo %(secret:foo)s | sed "s/bar/The password was there/"'))) else: f.addStep(steps.ShellCommand(command=Interpolate( 'echo %(secret:foo)s'))) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f)] return c
def makeCleanOldBuildsFactory(): """ Remove build results older than 14 days. """ # FIXME we shouldn't hard code this (DRY) basedir = r'/srv/buildmaster/data' path = os.path.join(basedir, "private_html") factory = BuildFactory() factory.addStep(MasterShellCommand( ['find', path, '-type', 'f', '-mtime', '+14', '-exec', 'unlink', '{}', ';', '-print'], description=['Removing', 'old', 'results'], descriptionDone=['Remove', 'old', 'results'], name='remove-old-results')) # Vagrant tutorial boxes are created on the Vagrant slave, and # uploaded to S3. However, the boxes must be kept on the slave # for running subsequent tests # (flocker/acceptance/vagrant/centos-7/zfs and # flocker/installed-package/vagrant/centos-7). This means there is # not an obvious place to remove the boxes. So, we periodically # cleanup old boxes here. "Old" is the number of days passed as # parameter to script. factory.addStep(ShellCommand( command=['python', '/home/buildslave/remove-old-boxes.py', '14'], description=['Removing', 'old', 'boxes'], descriptionDone=['Remove', 'old', 'boxes'], name='remove-old-boxes')) return factory
def __init__(self, test, timeout, branch=None, talos_config_file='sample.config', results_server=None, reboot=True, base_dir='/builds', reboot_cmd=['sudo', 'reboot-user'], nochrome=False, cvsroot=":pserver:[email protected]:/cvsroot", hg_host='http://hg.mozilla.org', tools_repo_path='build/tools', talos_tarball=None, pageloader_tarball=None, cleanup_glob='tools talos fennec* *.tar.bz2 *.zip', tp4_source='/tools/tp4', browser_wait=7, **kwargs): BuildFactory.__init__(self, **kwargs) self.test = test self.timeout = timeout self.branch = branch self.talos_config_file = talos_config_file self.results_server = results_server self.reboot = reboot self.base_dir = base_dir self.reboot_cmd = reboot_cmd self.nochrome = '' if nochrome is None else '--noChrome' self.cvsroot = cvsroot #We are using a static ip because of dns failures self.hg_host = hg_host self.tools_repo_path = tools_repo_path self.talos_tarball = talos_tarball self.pageloader_tarball = pageloader_tarball self.cleanup_glob = cleanup_glob self.tp4_source = tp4_source self.browser_wait = browser_wait self.addStartupSteps() self.addCleanupSteps() self.addSetupSteps() self.addObtainBuildSteps() self.addRunSteps() self.addFinalSteps()
def masterConfig(build_set_summary): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers, reporters c['schedulers'] = [ schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"]) ] f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] notifier = reporters.PushoverNotifier( '1234', 'abcd', mode="all", watchedWorkers=['local1'], buildSetSummary=build_set_summary, messageFormatter=MessageFormatter(template='This is a message.'), messageFormatterMissingWorker=MessageFormatterMissingWorker( template='No worker.')) c['services'] = [ reporters.MailNotifier("*****@*****.**", mode="all", buildSetSummary=build_set_summary), notifier ] return c
def __init__(self, sources, arch): BuildFactory.__init__(self, sources) # Download the helpers for helper in ("pkgdepends", "pkgprovides", "pkgversion", "ccm-setup", "changelog", "gitrev"): self.addStep(steps.FileDownload(name="helper " + helper, mastersrc="helpers/pkgbuild/" + helper, slavedest="../helpers/" + helper, mode=0755)) # Create a directory to hold the packages that have been built self.addStep(steps.RemoveDirectory(name="rm-repository", dir="repository")) # Create a directory to hold the packages that have been built self.addStep(steps.MakeDirectory(name="mkdir-repository", dir="repository")) # Create or update the chroot self.addStep(PrepareCcm(arch=arch)) # Copy the channel configuration from slave to master self.addStep(steps.FileUpload("channels.yml", "tmp/channels.yml", name="config-upload")) self.addStep(steps.FileUpload("buildinfo.yml", "tmp/buildinfo.yml", name="buildinfo-upload")) # Scan repository and find packages to build self.addStep(RepositoryScan(arch=arch)) # Create a changelog for the repo self.addStep(Changelog(arch=arch)) self.addStep(steps.FileDownload(mastersrc="tmp/buildinfo.yml", slavedest="buildinfo.yml", name="buildinfo-download")) # Publish the repository self.addStep(steps.MasterShellCommand(command="rm -rf /srv/http/repos/papyros/" + arch)) self.addStep(steps.DirectoryUpload('../repository', '/srv/http/repos/papyros/' + arch)) self.addStep(steps.MasterShellCommand(command="chmod a+rX -R /srv/http/repos/papyros")) # Push back changes to version control (only push for x86_64 so we don't have duplicates) if arch == "x86_64": self.addStep(PushSourceChanges())
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] # note that as of December 2018, the vault docker image default to kv # version 2 to be enabled by default c['secretsProviders'] = [HashiCorpVaultSecretProvider( vaultToken='my_vaulttoken', vaultServer="http://localhost:8200", apiVersion=2 )] f = BuildFactory() f.addStep(ShellCommand(command=[Interpolate('echo %(secret:key)s')])) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f)] return c
def __init__(self, repo, steps=None): self.basicSteps = [ # Checkout s( SocialtextSVN , mode='update' , baseURL=repo , defaultBranch='/trunk' ), # Fix tarball (harmless on nonMacs) nlwShellStep( 'fix help tarball' , 'dev-bin/fix-help-tarball-for-mac' ), # Stop test servers; clear cached fixtures. nlwShellStep( 'cleanup' , 'if [ -d t/tmp ]; then dev-bin/nlwctl -t stop; rm -r t/tmp*; fi' ), # Configure. nlwShellStep( 'configure' , './configure --dev=1 --apache-proxy=1 [email protected]' )] BuildFactory.__init__(self, self.basicSteps + steps)
def clean_chroot(cls, slaves): base_dir = "../../../" factory = BuildFactory() step_clean_chroot = ShellCommand( command = ["scripts/chroot-cleaner", WithProperties('%(branch)s') ], workdir=base_dir, haltOnFailure = True, flunkOnFailure = True, description = "cleanup chroots", descriptionDone = "cleaned chroots", name = "cleanup chroots", interruptSignal="TERM") factory.addStep(step_clean_chroot) builder = BuilderConfig( name = "clean_chroot", category = "clean", slavenames = [ i.slavename for i in slaves ], properties = { "pkgname": "clean_chroot" }, factory = factory, builddir = "builders/clean_chroot", slavebuilddir = "builders/clean_chroot", ) return builder
def __init__(self, repourl, submodules=False, branch='master', codebase='', imageset=None): BuildFactory.__init__(self) self.addStep(steps.SetProperty(property='datestamp', value=datestamp)) self.addStep(steps.Git(repourl=repourl, submodules=submodules, branch=branch, codebase=codebase, name='git-checkout-{}'.format(branch), mode=('full' if submodules else 'incremental'), method='clobber', doStepIf=lambda step: not is_pull_request(step.build.getProperties()), hideStepIf=lambda results, step: results == bbres.SKIPPED)) self.addStep(steps.GitHub(repourl=repourl, submodules=submodules, branch=branch, codebase=codebase, name='git-checkout-pullrequest-ref', mode=('full' if submodules else 'incremental'), method='clobber', doStepIf=lambda step: is_pull_request(step.build.getProperties()), hideStepIf=lambda results, step: results == bbres.SKIPPED)) env_vars = ENV_VARS.copy() # First, remove duplicates from PATH, # then strip out the virtualenv bin directory if we're in a virtualenv. setup_cmd = 'PATH=`echo -n "$PATH" | awk -v RS=: -v ORS=: \'!arr[$0]++\'`;' + \ 'if [ -n "$VIRTUAL_ENV" ]; then ' + \ 'PATH=`echo "$PATH" | sed -re "s,(^|:)$VIRTUAL_ENV/bin(:|$),\\2,g;s,^:,,"`; ' + \ 'fi; . %(prop:setup_script)s; printenv' # Setup steps self.addStep(steps.RemoveDirectory('build/build', name='cleanup', description=['Removing', 'old', 'build', 'directory'], descriptionDone=['Removed', 'old', 'build', 'directory'])) self.addStep(steps.SetPropertyFromCommand(command=['bash', '-c', util.Interpolate(setup_cmd)], extract_fn=extract_env_vars, name='EnvironmentSetup', description=['Running', 'setup', 'script'], descriptionDone=['Ran', 'setup', 'script'])) self.addStep(steps.StringDownload(s=make_autoconf, workerdest='auto.conf', workdir=util.Interpolate("%(prop:BUILDDIR)s/conf"), name='make-auto.conf', description=['Creating', 'auto.conf'], descriptionDone=['Created', 'auto.conf'])) for i, img in enumerate(imageset.imagespecs, start=1): tgtenv = env_vars.copy() tgtenv.update(img.env) bbcmd = "bitbake" if img.keep_going: bbcmd += " -k" cmd = util.Interpolate(bbcmd + "%(kw:bitbake_options)s " + ' '.join(img.args), bitbake_options=bitbake_options) self.addStep(steps.ShellCommand(command=['bash', '-c', cmd], timeout=None, env=tgtenv, workdir=util.Property('BUILDDIR'), name='build_%s_%s' % (imageset.name, img.name), description=['Building', imageset.name, img.name], descriptionDone=['Built', imageset.name, img.name])) self.addStep(steps.ShellCommand(command=store_artifacts_cmd, workdir=util.Property('BUILDDIR'), name='StoreArtifacts', timeout=None, description=['Storing', 'artifacts'], descriptionDone=['Stored', 'artifacts']))
def getFactory(self, branch, combo): factory = BuildFactory() factory.addStep( Git(repourl=self._project_git_repo, mode='copy', branch=branch)) factory.addStep( EmacsCompile(command=["make", "clean", "all"])) return factory
def triggerRunsForever(): f2 = BuildFactory() f2.addStep( steps.ShellCommand(command="\n".join( ['while :', 'do', ' echo "sleeping";', ' sleep 1;' 'done']))) return setupTriggerConfiguration(f2)
def makeCleanOldResources(): """ Remove cloud instances (from acceptance and client tests) and cloud volumes more than two hours old. """ factory = BuildFactory() factory.addStep(CleanAcceptanceInstances(lag=timedelta(hours=2))) factory.addStep(CleanVolumes(lag=timedelta(minutes=30))) return factory
def getFactory(self): factory = BuildFactory() for step in self.factorySteps: if type(step) is list: raise Exception('Error: build step is list') if step is None: raise Exception('Error: build step is None') factory.addStep(step) return factory
def triggerRunsForever(): f2 = BuildFactory() f2.addStep(steps.ShellCommand(command="\n".join(['while :', 'do', ' echo "sleeping";', ' sleep 1;' 'done']))) return setupTriggerConfiguration(f2)
def make_bundler_builder(self): builder_name = "builder_bundler" factory = BuildFactory() repo_name = "bitmask_bundler" repo_url = "https://github.com/leapcode/" + repo_name + ".git" branch = "develop" workdir = "build" repo_dir = workdir + "/" + repo_name bundler_output_dir = "bundler_output" sumo_tarball = "leap.bitmask-latest-SUMO.tar.gz" publish_bundle = self._publish_bundle_command( '`ls -t *.tar.gz | head -1`') factory.addSteps([ steps.Git(repourl=repo_url, branch=branch, workdir=repo_dir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url), steps.ShellCommand(command="rm -rf " + bundler_output_dir, workdir=workdir, name="Remove previous bundler dir"), steps.ShellCommand(command="mkdir " + bundler_output_dir, workdir=workdir, name="Create bundler dir"), steps.ShellCommand(command="cp bundle_pyinstaller.sh ../" + bundler_output_dir, workdir=repo_dir, haltOnFailure=True, name="Copy bundle_pyinstaller"), steps.ShellCommand(command="mkdir files", workdir=workdir + '/' + bundler_output_dir, name="Create auxiliary folder"), steps.ShellCommand( command="wget http://lizard.leap.se/sumo-tarball/%s" % sumo_tarball, workdir=workdir + '/' + bundler_output_dir, haltOnFailure=True, name="Download sumo"), steps.ShellCommand(command="./bundle_pyinstaller.sh " + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, name="Create bundle"), steps.ShellCommand(command=publish_bundle, workdir=workdir + '/' + bundler_output_dir, name="Publish bundle") ]) return BuilderConfig(name=builder_name, slavenames=self.slaves.leap_names(), factory=factory)
def test_local_worker_max_builds(self): """ If max_builds is set, only one build is started on a worker at a time. """ step_controller = StepController() config_dict = { 'builders': [ BuilderConfig( name="testy-1", workernames=["local"], factory=BuildFactory([step_controller]), ), BuilderConfig( name="testy-2", workernames=["local"], factory=BuildFactory([step_controller]), ), ], 'workers': [LocalWorker('local', max_builds=1)], 'protocols': { 'null': {} }, 'multiMaster': True, } self.master = master = self.successResultOf( getMaster(self, self.reactor, config_dict)) builder_ids = [ self.successResultOf(master.data.updates.findBuilderId('testy-1')), self.successResultOf(master.data.updates.findBuilderId('testy-2')), ] started_builds = [] self.successResultOf( master.mq.startConsuming( lambda key, build: started_builds.append(build), ('builds', None, 'new'))) # Trigger a buildrequest bsid, brids = self.successResultOf( master.data.updates.addBuildset( waited_for=False, builderids=builder_ids, sourcestamps=[ { 'codebase': '', 'repository': '', 'branch': None, 'revision': None, 'project': '' }, ], )) self.assertEqual(len(started_builds), 1)
def MakeObsBuilder(): f = BuildFactory() f.addSteps(svn_co) f.addStep(ShellCommand( name = 'upload', command = ['bash', 'os/obs/obs-upload.sh', Property('project'), WithProperties('%(version)s.%(got_revision)s')], haltOnFailure = True, logEnviron = False )) return f
def getTestingFactory(self, branch): factory = BuildFactory() _emacs_prop = 'EMACS=%(slave/binaries/emacs:-emacs)s' _eflags = '-L ../mocker' for p, r, t in self._deps: factory.addStep( DetachedGit(repourl=r, branch=t, workdir=p, logEnviron=False, description=['updating %s' % (p)], descriptionDone=['update %s' % (p)])) factory.addStep( Git(repourl=self._project_git_repo, mode='copy', branch=branch, logEnviron=False)) factory.addStep( EmacsCompile(command=["make", WithProperties(_emacs_prop), "clean", "all"], logEnviron=False)) factory.addStep( EmacsTest(command=["make", WithProperties(_emacs_prop), "EFLAGS=%s" % (_eflags), "test"], logEnviron=False)) return factory
def test_addStep_deprecated(self): """ Passing keyword arguments to L{BuildFactory.addStep} is deprecated, but pass the arguments to the first argument, to construct a step. """ factory = BuildFactory() factory.addStep(BuildStep) self.assertEqual(factory.steps, [_BuildStepFactory(BuildStep)]) warnings = self.flushWarnings([self.test_addStep_deprecated]) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0]['category'], DeprecationWarning)
def build_factory(): factory = BuildFactory() # Step 1: check out the source factory.addStep(Git(repourl='git://github.com/buildbot/pyflakes.git', mode='copy')) # Step 2: run the tests # (note that this will require that 'trial' is installed) factory.addStep(ShellCommand(command=['trial', 'pyflakes'])) return factory
def getBasicFactory(self, branch): factory = BuildFactory() _emacs_prop = 'EMACS=%(slave/binaries/emacs:-emacs)s' factory.addStep( Git(repourl=self._project_git_repo, mode='copy', branch=branch, logEnviron=False)) factory.addStep( EmacsCompile(command=["make", WithProperties(_emacs_prop), "clean", "all"], logEnviron=False)) return factory
def ros_debbuild(c, job_name, packages, url, distro, arch, rosdistro, version, machines, othermirror, keys, trigger_pkgs = None): gbp_args = ['-uc', '-us', '--git-ignore-branch', '--git-ignore-new', '--git-verbose', '--git-dist='+distro, '--git-arch='+arch] f = BuildFactory() # Remove the build directory. f.addStep( RemoveDirectory( name = job_name+'-clean', dir = Interpolate('%(prop:workdir)s'), hideStepIf = success, ) ) # Check out the repository master branch, since releases are tagged and not branched f.addStep( Git( repourl = url, branch = 'master', alwaysUseLatest = True, # this avoids broken builds when schedulers send wrong tag/rev mode = 'full' # clean out old versions ) ) # Update the cowbuilder f.addStep( ShellCommand( command = ['cowbuilder-update.py', distro, arch] + keys, hideStepIf = success ) ) # Need to build each package in order for package in packages: debian_pkg = 'ros-'+rosdistro+'-'+package.replace('_','-') # debian package name (ros-groovy-foo) branch_name = 'debian/'+debian_pkg+'_%(prop:release_version)s_'+distro # release branch from bloom deb_name = debian_pkg+'_%(prop:release_version)s'+distro final_name = debian_pkg+'_%(prop:release_version)s-%(prop:datestamp)s'+distro+'_'+arch+'.deb' # Check out the proper tag. Use --force to delete changes from previous deb stamping f.addStep( ShellCommand( haltOnFailure = True, name = package+'-checkout', command = ['git', 'checkout', Interpolate(branch_name), '--force'], hideStepIf = success ) ) # Download script for building the source deb f.addStep( FileDownload( name = job_name+'-grab-build-source-deb-script', mastersrc = 'scripts/build_source_deb.py', slavedest = Interpolate('%(prop:workdir)s/build_source_deb.py'), mode = 0755, hideStepIf = success ) )
def __init__(self, python, source, uncleanWarnings, trialTests=None, trialMode=None, virtualenv=False, virtualenv_module='virtualenv', platform='unix', forceGarbageCollection=False): if not isinstance(source, list): source = [source] else: source = list(source) # If permissions get messed up on a slave, this can fix it. # But it breaks on old slaves so it's not enabled all the time # (and it can't fix old slaves, obviously). # self._fixPermissions(source) BuildFactory.__init__(self, source) if type(python) is str: python = [python, "-Wall"] assert platform in ["unix", "windows"] self._platform = platform if platform == "unix": self._path = posixpath elif platform == "windows": self._path = ntpath self.python = python self.virtualenv = virtualenv self.uncleanWarnings = uncleanWarnings self.forceGarbageCollection = forceGarbageCollection self.trialMode = trialMode if trialTests is None: trialTests = [WithProperties("%(test-case-name:~twisted)s")] self.trialTests = trialTests if virtualenv: # Each time we create a new virtualenv as latest pip can build # wheels on the fly and install them from user's cache. self.addStep( shell.ShellCommand, command = self.python + [ "-m", virtualenv_module, '--clear', self._virtualEnvPath, ], ) else: # Report the versions, since we're using the system ones. If it's a # virtualenv, it's up to the venv factory to report the versions # itself. self._reportVersions()
class Job(object): """The base class for all job types. It doesn't do much by itself, except for creating a new Builder and a new Force Scheduler so that users can forcibly trigger a new build. This class can be used as a context manager although, by default, it does nothing. """ def __init__(self, name, workers): self.name = utils.name(name) self.workers = workers self.build = BuildFactory() BuildmasterConfig['builders'].append(BuilderConfig( name=self.name, factory=self.build, slavenames=self.workers)) BuildmasterConfig['schedulers'].append(ForceScheduler( name=scheduler_name(self, 'force'), builderNames=[self.name], branch=FixedParameter(name="reason", default=""), reason=FixedParameter(name="reason", default="Forced build"), revision=FixedParameter(name="revision", default=""), repository=FixedParameter(name="repository", default=""), project=FixedParameter(name="project", default=""), properties=[], username=FixedParameter(name="username", default="WebUI"))) def __enter__(self): return self def __exit__(self, type, value, traceback): pass def add_step(self, step): """Adds a build step on this Job.""" self.build.addStep(step) def trigger(self, job_or_jobs): """Adds a build step which triggers execution of another job.""" if type(job_or_jobs) is list: self.add_step(Trigger( schedulerNames=[scheduler_name(j, 'trigger') for j in job_or_jobs], waitForFinish=True)) else: self.add_step(Trigger( schedulerNames=[scheduler_name(job_or_jobs, 'trigger')], waitForFinish=True))
def test_worker_multiple_substantiations_succeed(self): """ If multiple builders trigger try to substantiate a worker at the same time, if the substantiation succeeds then all of the builds proceed. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig( name="testy-1", workernames=["local"], factory=BuildFactory(), ), BuilderConfig( name="testy-2", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, 'multiMaster': True, } master = self.getMaster(config_dict) builder_ids = [ self.successResultOf(master.data.updates.findBuilderId('testy-1')), self.successResultOf(master.data.updates.findBuilderId('testy-2')), ] finished_builds = [] self.successResultOf( master.mq.startConsuming( lambda key, build: finished_builds.append(build), ('builds', None, 'finished'))) # Trigger a buildrequest bsid, brids = self.createBuildrequest(master, builder_ids) # The worker succeeds to substantiate. controller.start_instance(True) controller.connect_worker(self) # We check that there were two builds that finished, and # that they both finished with success self.assertEqual([build['results'] for build in finished_builds], [SUCCESS] * 2) controller.auto_stop(True)
def config_for_master_command(self, **kwargs): c = {} c['schedulers'] = [ schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"]) ] f = BuildFactory() f.addStep(steps.MasterShellCommand(**kwargs)) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def __init__(self, python, source, uncleanWarnings, trialTests=None, trialMode=None): if not isinstance(source, list): source = [source] else: source = list(source) # If permissions get messed up on a slave, this can fix it. # But it breaks on old slaves so it's not enabled all the time # (and it can't fix old slaves, obviously). # self._fixPermissions(source) BuildFactory.__init__(self, source) if type(python) is str: python = [python] self.python = python self.uncleanWarnings = uncleanWarnings self.trialMode = trialMode if trialTests is None: trialTests = [WithProperties("%(test-case-name:~twisted)s")] self.trialTests = trialTests self.addStep( ReportPythonModuleVersions, python=self.python, moduleInfo=[ ("Python", "sys", "sys.version"), ("OpenSSL", "OpenSSL", "OpenSSL.__version__"), ("PyCrypto", "Crypto", "Crypto.__version__"), ("gmpy", "gmpy", "gmpy.version()"), ("SOAPpy", "SOAPpy", "SOAPpy.__version__"), ("ctypes", "ctypes", "ctypes.__version__"), ("gtk", "gtk", "gtk.gtk_version"), ("pygtk", "gtk", "gtk.pygtk_version"), ("pywin32", "win32api", "win32api.GetFileVersionInfo(win32api.__file__, chr(92))['FileVersionLS'] >> 16" ), ("pyasn1", "pyasn1", "pyasn1.majorVersionId"), ("cffi", "cffi", "cffi.__version"), ], pkg_resources=[ ("subunit", "subunit"), ("zope.interface", "zope.interface"), ])
def addSimpleBuilder(self, name, buildername, category, repourl, builderconfig, sos, sbranch, bparams): """Private. Add a single builder on the given OS type for the given repo and branch. """ factory = BuildFactory() factory.addStep(Git(repourl=repourl, mode='full', submodules=True, method='copy', branch=sbranch, getDescription={'tags': True})) if "tag" in builderconfig and not(builderconfig["tag"] is None): stag = builderconfig["tag"].encode('ascii', 'ignore') factory.addStep(ShellCommand( command=['git', 'checkout', stag], workdir="build", description="checkout tag")) # Delegate all knowlege of how to build to a script called buildshim in the project # directory. Call this script in nine standardize steps. It should ignore steps that it doesn't need. # Pass the step name as the first arg, and if params was given in the json for this builder, pass that as the # second arg. for step in ["patch", "install_deps", "configure", "compile", "check", "package", "upload", "compile_extra", "uninstall_deps"]: factory.addStep(ShellCommand(command=["./buildshim", step, bparams], description=step, haltOnFailure=True)) self['builders'].append( BuilderConfig(name=buildername, slavenames=self._os2slaves[sos], factory=factory, category=category)) return factory
def testGoodStep(self): f = BuildFactory() f.addStep(SetProperty(command=["echo", "value"], property="propname")) ss = SourceStamp() req = FakeBuildRequest("Testing", ss, None) b = f.newBuild([req]) b.build_status = FakeBuildStatus() b.slavebuilder = FakeSlaveBuilder() # This shouldn't raise an exception b.setupBuild(None)
def createLinuxCIFactory(): f = BuildFactory() f.addStep( Git( description="fetching sources", descriptionDone="sources", haltOnFailure=True, repourl=repositoryUri, mode='full', method='clobber', )) f.addStep( ShellCommand(description="fetching packages", descriptionDone="packages", haltOnFailure=True, command=["mono", "paket.exe", "restore"], workdir=workingDirectory)) f.addStep( ShellCommand(description="building", descriptionDone="build", haltOnFailure=True, command=["xbuild", "CorvusAlba.ToyFactory.Linux.sln"], workdir=workingDirectory)) return f
def catkin_ws_steps(job_name, repo, arch, oscode, distro): f = BuildFactory() # Remove the build directory. f.addStep( RemoveDirectory( name=job_name + '-clean', dir=Interpolate('%(prop:builddir)s'), )) # Check out the repository master branch, since releases are tagged and not branched f.addStep( Git( repourl=repo["url"], branch=repo["branch"], alwaysUseLatest= True, # this avoids broken builds when schedulers send wrong tag/rev mode='full' # clean out old versions )) # Build all packages in catkin_ws # If one package built failed, then failed f.addStep( ShellCommand( haltOnFailure=True, name=job_name + '-buildbinary', # Current workdir is the git source dir command=['bloom-local-deb'], descriptionDone=['binarydeb', job_name], )) return f
def MakeMacBuilder(): f = BuildFactory() f.addSteps(svn_co) f.addStep( ShellCommand( name='revision', command=['bash', 'revision.sh'], workdir='build/os/macosx', haltOnFailure=True, env={'SCHAT_REVISION': Property('got_revision')}, )) f.addStep( ShellCommand( name='dmg', command=['bash', 'deploy.sh'], workdir='build/os/macosx', haltOnFailure=True, )) f.addStep( FileUpload( mode=0644, slavesrc=WithProperties( 'os/macosx/dmg/SimpleChat2-%(version)s.dmg'), masterdest=UploadFileName('SimpleChat2-%(version)s%(suffix)s.dmg'), url=WithProperties( 'https://download.schat.me/schat2/snapshots/%(version)s/r%(got_revision)s/SimpleChat2-%(version)s%(suffix)s.dmg' ), ))
def MakeReleaseBuilder(): f = BuildFactory() f.addStep( MasterShellCommand( name='Upload', command=[ 'bash', 'upload.sh', SCHAT_VERSION, Property('revision'), ], )) f.addStep( MasterShellCommand(name='Create Update Channel', command=[ 'php', 'update.php', '--channel', Property('channel', default='stable'), '--version', SCHAT_VERSION, '--revision', Property('revision'), '--os', Property('os', default='win32,osx,ubuntu'), ])) f.addStep( MasterShellCommand(name='Update Site', command=[ 'php', 'site.php', '--version', SCHAT_VERSION, ])) return f
def createFactory(): factory = BuildFactory() factory.addSteps(common.initTargetProperty()) factory.addStep(steps.Trigger( name="Call the 'build' scheduler. Build CentOS", schedulerNames=['build'], waitForFinish=True, haltOnFailure=True, copy_properties=COMMON_BUILD_AND_TEST_SNAPSHOT_PROPERTIES, set_properties={ "box": util.Property("box"), 'try_already_running': 'yes', "target": util.Property("target"), 'virtual_builder_name': util.Interpolate('Build for %(prop:box)s'), } )) factory.addStep(steps.Trigger( name="Call the 'run_test_snapshot' scheduler. Run functional tests", schedulerNames=['run_test_snapshot'], waitForFinish=True, copy_properties=COMMON_BUILD_AND_TEST_SNAPSHOT_PROPERTIES, set_properties={ "box": util.Property("box"), "target": util.Property("target"), "test_branch": util.Property("branch"), "test_set": common.renderTestSet, "backend_ssl": util.Property("backend_ssl"), "use_valgrind": util.Property("use_valgrind"), } )) return factory
def create_factory(): factory = BuildFactory() factory.addSteps(common.initTargetProperty()) factory.addStep( steps.Trigger(name="Call the 'build' scheduler", schedulerNames=['build'], waitForFinish=True, haltOnFailure=True, copy_properties=COMMON_PROPERTIES, set_properties={ 'virtual_builder_name': util.Interpolate('Build for %(prop:box)s'), 'box': 'ubuntu_bionic_libvirt', 'try_already_running': 'no', })) factory.addStep( steps.Trigger(name="Call the 'run_performance_test' scheduler", schedulerNames=['run_performance_test_trigger'], waitForFinish=True, copy_properties=COMMON_PROPERTIES, set_properties={ 'test_branch': util.Property('branch'), 'host': 'max-tst-01', })) return factory