def __init__(self, branch, python, projects, *a, **kw):
     BuildFactory.__init__(self, *a, **kw)
     self.addStep(
         Mercurial,
         repourl="http://hg.python.org/cpython",
         defaultBranch=branch,
         branchType='inrepo',
         mode="copy")
     self.addStep(
         ShellCommand,
         name="configure-python",
         description=["configuring", "python"],
         descriptionDone=["configure", "python"],
         command="./configure --prefix=$PWD/install")
     self.addStep(
         ShellCommand,
         name="install-python",
         description=["installing", "python"],
         descriptionDone=["install", "python"],
         command=["make", "install"])
     pythonc = "install/bin/" + python
     self.addStep(
         ShellCommand,
         name="link-binary",
         description=["linking", "binary"],
         descriptionDone=["link", "binary"],
         command=["ln", "-nsf", "build/" + pythonc, "python"],
         workdir=".")
     self.buildModules(pythonc, projects)
    def __init__(self, git_url, cmd, buildout_content):
        BuildFactory.__init__(self)

        self.addStep(PgSafeStopSb())
        self.addStep(CleanUp())
        self.addStep(CreateSandbox())
        self.addStep(Clone(git_url, 'pkg'))
        self.addStep(BootstrapSb())

        # Change the working directory of the next build steps
        wd = os.path.join(self.workdir, "pkg")

        self.addStep(CreateFile(self.buildout_file,
                                buildout_content,
                                workdir=wd))

        self.addStep(Buildout(cmd_args=['-U', '-c', self.buildout_file]))
        self.addStep(PgStartSb())
        self.addStep(ShellCommand(
            name="custom_cmd",
            haltOnFailure=False,
            flunkOnFailure=True,
            workdir=wd,
            command=cmd
        ))
        self.addStep(PgStopSb())
Beispiel #3
0
    def create_single_worker_two_builder_config(self, controller_kwargs=None):
        if not controller_kwargs:
            controller_kwargs = {}

        controller = LatentController(self, 'local', **controller_kwargs)
        config_dict = {
            'builders': [
                BuilderConfig(
                    name="testy-1",
                    workernames=["local"],
                    factory=BuildFactory(),
                ),
                BuilderConfig(
                    name="testy-2",
                    workernames=["local"],
                    factory=BuildFactory(),
                ),
            ],
            'workers': [controller.worker],
            'protocols': {
                'null': {}
            },
            # Disable checks about missing scheduler.
            'multiMaster':
            True,
        }
        master = yield self.getMaster(config_dict)
        builder_ids = [
            (yield master.data.updates.findBuilderId('testy-1')),
            (yield master.data.updates.findBuilderId('testy-2')),
        ]

        return controller, master, builder_ids
Beispiel #4
0
def masterConfig(extra_config):
    c = {}
    from buildbot.config import BuilderConfig
    from buildbot.process.buildstep import BuildStep
    from buildbot.process.factory import BuildFactory
    from buildbot.process import results

    class MyBuildStep(BuildStep):
        def start(self):
            self.finished(results.SUCCESS)

    c['change_source'] = []
    c['schedulers'] = []  # filled in above
    f1 = BuildFactory()
    f1.addStep(MyBuildStep(name='one'))
    f1.addStep(MyBuildStep(name='two'))
    c['builders'] = [
        BuilderConfig(name="a", workernames=["local1"], factory=f1),
    ]
    c['status'] = []
    c['title'] = "test"
    c['titleURL'] = "test"
    c['buildbotURL'] = "http://localhost:8010/"
    c['mq'] = {'debug': True}
    # test wants to influence the config, but we still return a new config
    # each time
    c.update(extra_config)
    return c
Beispiel #5
0
def triggeredBuildIsNotCreated():
    f2 = BuildFactory()
    f2.addStep(steps.ShellCommand(command="echo 'hello'"))

    def nextBuild(*args, **kwargs):
        return defer.succeed(None)
    return setupTriggerConfiguration(f2, nextBuild=nextBuild)
Beispiel #6
0
def setupTriggerConfiguration(triggeredFactory, nextBuild=None):
    c = {}

    c['schedulers'] = [
        schedulers.Triggerable(
            name="trigsched",
            builderNames=["triggered"]),
        schedulers.AnyBranchScheduler(
            name="sched",
            builderNames=["main"])]

    f = BuildFactory()
    f.addStep(steps.Trigger(schedulerNames=['trigsched'],
                            waitForFinish=True,
                            updateSourceStamp=True))
    f.addStep(steps.ShellCommand(command='echo world'))

    mainBuilder = BuilderConfig(name="main",
                                workernames=["local1"],
                                factory=f)

    triggeredBuilderKwargs = {'name': "triggered",
                              'workernames': ["local1"],
                              'factory': triggeredFactory}

    if nextBuild is not None:
        triggeredBuilderKwargs['nextBuild'] = nextBuild

    triggeredBuilder = BuilderConfig(**triggeredBuilderKwargs)

    c['builders'] = [mainBuilder, triggeredBuilder]
    return c
Beispiel #7
0
def masterConfig(secret_specifier):
    c = {}
    from buildbot.config import BuilderConfig
    from buildbot.process.factory import BuildFactory
    from buildbot.plugins import schedulers

    c['schedulers'] = [
        schedulers.ForceScheduler(name="force", builderNames=["testy"])
    ]

    # note that as of December 2018, the vault docker image default to kv
    # version 2 to be enabled by default
    c['secretsProviders'] = [
        HashiCorpVaultSecretProvider(vaultToken='my_vaulttoken',
                                     vaultServer="http://localhost:8200",
                                     apiVersion=2)
    ]

    f = BuildFactory()
    f.addStep(
        ShellCommand(command=Interpolate('echo -n {} | base64'.format(
            secret_specifier))))

    c['builders'] = [
        BuilderConfig(name="testy", workernames=["local1"], factory=f)
    ]
    return c
Beispiel #8
0
 def test_addSteps(self):
     factory = BuildFactory()
     factory.addSteps([BuildStep(), BuildStep()])
     self.assertEqual(
         factory.steps,
         [_BuildStepFactory(BuildStep),
          _BuildStepFactory(BuildStep)])
Beispiel #9
0
def masterConfig():
    c = {}
    from buildbot.worker import Worker
    from buildbot.config import BuilderConfig
    from buildbot.process.buildstep import BuildStep
    from buildbot.process.factory import BuildFactory
    from buildbot.process import results

    class MyBuildStep(BuildStep):

        def start(self):
            self.finished(results.SUCCESS)

    c['workers'] = [Worker("local1", "localpw")]
    c['protocols'] = {'pb': {'port': 'tcp:0'}}
    c['change_source'] = []
    c['schedulers'] = []  # filled in above
    f1 = BuildFactory()
    f1.addStep(MyBuildStep(name='one'))
    f1.addStep(MyBuildStep(name='two'))
    c['builders'] = [
        BuilderConfig(name="a", workernames=["local1"], factory=f1),
    ]
    c['status'] = []
    c['title'] = "test"
    c['titleURL'] = "test"
    c['buildbotURL'] = "http://localhost:8010/"
    c['db'] = {'db_url': "sqlite:///state.sqlite"}
    c['mq'] = {'debug': True}
    # test wants to influence the config, but we still return a new config each time
    c.update(BuildmasterConfig)
    return c
Beispiel #10
0
    def test_latent_max_builds(self):
        """
        If max_builds is set, only one build is started on a latent
        worker at a time.
        """
        controller = LatentController(self, 'local', max_builds=1)
        step_controller = StepController()
        config_dict = {
            'builders': [
                BuilderConfig(
                    name="testy-1",
                    workernames=["local"],
                    factory=BuildFactory([step_controller]),
                ),
                BuilderConfig(
                    name="testy-2",
                    workernames=["local"],
                    factory=BuildFactory([step_controller]),
                ),
            ],
            'workers': [controller.worker],
            'protocols': {
                'null': {}
            },
            'multiMaster':
            True,
        }
        yield self.setup_master(config_dict)
        builder_ids = [
            (yield self.master.data.updates.findBuilderId('testy-1')),
            (yield self.master.data.updates.findBuilderId('testy-2')),
        ]

        started_builds = []
        yield self.master.mq.startConsuming(
            lambda key, build: started_builds.append(build),
            ('builds', None, 'new'))

        # Trigger a buildrequest
        yield self.master.data.updates.addBuildset(
            waited_for=False,
            builderids=builder_ids,
            sourcestamps=[
                {
                    'codebase': '',
                    'repository': '',
                    'branch': None,
                    'revision': None,
                    'project': ''
                },
            ],
        )

        # The worker fails to substantiate.
        controller.start_instance(True)

        yield controller.connect_worker()

        self.assertEqual(len(started_builds), 1)
        yield controller.auto_stop(True)
Beispiel #11
0
    def create_single_worker_two_builder_step_lock_config(
            self, lock_cls, mode):
        lock = lock_cls("lock1", maxCount=1)

        stepcontrollers = [
            BuildStepController(locks=[lock.access(mode)]),
            BuildStepController(locks=[lock.access(mode)])
        ]

        config_dict = {
            'builders': [
                BuilderConfig(name='builder1',
                              workernames=['worker1'],
                              factory=BuildFactory([stepcontrollers[0].step])),
                BuilderConfig(name='builder2',
                              workernames=['worker1'],
                              factory=BuildFactory([stepcontrollers[1].step])),
            ],
            'workers': [
                self.createLocalWorker('worker1'),
            ],
            'protocols': {
                'null': {}
            },
            'multiMaster':
            True,
        }
        master = yield self.getMaster(config_dict)
        builder_ids = [
            (yield master.data.updates.findBuilderId('builder1')),
            (yield master.data.updates.findBuilderId('builder2')),
        ]

        return stepcontrollers, master, builder_ids
def masterConfig(use_interpolation):
    c = {}
    from buildbot.config import BuilderConfig
    from buildbot.process.factory import BuildFactory
    from buildbot.plugins import schedulers, steps, util

    c['services'] = [FakeSecretReporter('http://example.com/hook',
                                        auth=('user', Interpolate('%(secret:httppasswd)s')))]
    c['schedulers'] = [
        schedulers.ForceScheduler(
            name="force",
            builderNames=["testy"])]

    c['secretsProviders'] = [FakeSecretStorage(
        secretdict={"foo": "bar", "something": "more", 'httppasswd': 'myhttppasswd'})]
    f = BuildFactory()

    if use_interpolation:
        if os.name == "posix":
            # on posix we can also check whether the password was passed to the command
            command = Interpolate('echo %(secret:foo)s | sed "s/bar/The password was there/"')
        else:
            command = Interpolate('echo %(secret:foo)s')
    else:
        command = ['echo', util.Secret('foo')]

    f.addStep(steps.ShellCommand(command=command))

    c['builders'] = [
        BuilderConfig(name="testy",
                      workernames=["local1"],
                      factory=f)]
    return c
    def __init__(
        self, python, source, uncleanWarnings, trialTests=None,
        trialMode=None, virtualenv=False,
            ):
        if not isinstance(source, list):
            source = [source]
        else:
            source = list(source)

        # If permissions get messed up on a slave, this can fix it.
        # But it breaks on old slaves so it's not enabled all the time
        # (and it can't fix old slaves, obviously).

        # self._fixPermissions(source)

        BuildFactory.__init__(self, source)

        if type(python) is str:
            python = [python]

        self.python = python
        self.uncleanWarnings = uncleanWarnings
        self.trialMode = trialMode
        if trialTests is None:
            trialTests = [WithProperties("%(test-case-name:~twisted)s")]
        self.trialTests = trialTests

        if virtualenv:
            # Each time we create a new virtualenv as latest pip can build
            # wheels on the fly and install them from user's cache.
            self.addStep(
                shell.ShellCommand,
                command=[
                    'virtualenv', '--clear',
                    '-p', self.python[0],
                    self._virtualEnvPath,
                    ],
                )

        self.addStep(
            ReportPythonModuleVersions,
            python=self.python,
            moduleInfo=[
                ("Python", "sys", "sys.version"),
                ("OpenSSL", "OpenSSL", "OpenSSL.__version__"),
                ("PyCrypto", "Crypto", "Crypto.__version__"),
                ("gmpy", "gmpy", "gmpy.version()"),
                ("SOAPpy", "SOAPpy", "SOAPpy.__version__"),
                ("ctypes", "ctypes", "ctypes.__version__"),
                ("gtk", "gtk", "gtk.gtk_version"),
                ("pygtk", "gtk", "gtk.pygtk_version"),
                ("pywin32", "win32api",
                 "win32api.GetFileVersionInfo(win32api.__file__, chr(92))['FileVersionLS'] >> 16"),
                ("pyasn1", "pyasn1", "pyasn1.__version__"),
                ("cffi", "cffi", "cffi.__version__"),
                ],
            pkg_resources=[
                ("subunit", "subunit"),
                ("zope.interface", "zope.interface"),
                ])
Beispiel #14
0
def masterConfig():
    c = {}
    from buildbot.worker import Worker
    from buildbot.config import BuilderConfig
    from buildbot.process.buildstep import BuildStep
    from buildbot.process.factory import BuildFactory
    from buildbot.process import results

    class MyBuildStep(BuildStep):
        def start(self):
            self.finished(results.SUCCESS)

    c['workers'] = [Worker("local1", "localpw")]
    c['protocols'] = {'pb': {'port': 'tcp:0'}}
    c['change_source'] = []
    c['schedulers'] = []  # filled in above
    f1 = BuildFactory()
    f1.addStep(MyBuildStep(name='one'))
    f1.addStep(MyBuildStep(name='two'))
    c['builders'] = [
        BuilderConfig(name="a", workernames=["local1"], factory=f1),
    ]
    c['status'] = []
    c['title'] = "test"
    c['titleURL'] = "test"
    c['buildbotURL'] = "http://localhost:8010/"
    c['db'] = {'db_url': "sqlite:///state.sqlite"}
    c['mq'] = {'debug': True}
    # test wants to influence the config, but we still return a new config each time
    c.update(BuildmasterConfig)
    return c
Beispiel #15
0
def masterConfig():
    c = {}
    from buildbot.config import BuilderConfig
    from buildbot.process.factory import BuildFactory
    from buildbot.plugins import schedulers
    from buildbot.plugins import util
    lock = util.MasterLock("lock")

    c['schedulers'] = [
        schedulers.AnyBranchScheduler(
            name="sched",
            builderNames=["testy", "testy2", "testy3"]),
        ]
    f = BuildFactory()
    lockstep = LockedStep(locks=[lock.access('exclusive')])
    f.addStep(lockstep)
    # assert lockstep._factory.buildStep() == lockstep._factory.buildStep()
    c['builders'] = [
        BuilderConfig(name="testy",
                      workernames=["local1"],
                      factory=f),
        BuilderConfig(name="testy2",
                      workernames=["local1"],
                      factory=f),
        BuilderConfig(name="testy3",
              workernames=["local1"],
              factory=f)]

    return c
Beispiel #16
0
def masterConfig():
    global num_reconfig
    num_reconfig += 1
    c = {}
    from buildbot.config import BuilderConfig
    from buildbot.process.factory import BuildFactory
    from buildbot.schedulers.forcesched import ForceScheduler
    from buildbot.steps.shell import ShellCommandNewStyle
    from buildbot.util.service import BuildbotService

    class MyShellCommand(ShellCommandNewStyle):
        def getResultSummary(self):
            service = self.master.service_manager.namedServices['myService']
            return dict(step="num reconfig: %d" % (service.num_reconfig, ))

    class MyService(BuildbotService):
        name = "myService"

        def reconfigService(self, num_reconfig):
            self.num_reconfig = num_reconfig
            return defer.succeed(None)

    c['schedulers'] = [ForceScheduler(name="force", builderNames=["testy"])]

    f = BuildFactory()
    f.addStep(MyShellCommand(command='echo hei'))
    c['builders'] = [
        BuilderConfig(name="testy", workernames=["local1"], factory=f)
    ]

    c['services'] = [MyService(num_reconfig=num_reconfig)]
    if num_reconfig == 3:
        c['services'].append(
            MyService(name="myService2", num_reconfig=num_reconfig))
    return c
Beispiel #17
0
    def __init__(self, steps=None, depends_on_projects=None, **kwargs):
        # Cannot use "super" here as BuildFactory is an old style class.
        BuildFactory.__init__(self, steps)

        if depends_on_projects is None:
            self.depends_on_projects = frozenset(['llvm'])
        else:
            self.depends_on_projects = frozenset(depends_on_projects)

        # Directories.
        self.llvm_srcdir = kwargs.pop('llvm_srcdir', None)
        self.obj_dir = kwargs.pop('obj_dir', None)
        self.install_dir = kwargs.pop('install_dir', None)

        # Preserve the rest of the given extra attributes if any, so we could
        # expand the factory later.
        for k,v in kwargs.items():
            setattr(self, k, v)

        self.monorepo_dir = self.llvm_srcdir or "llvm-project"
        self.llvm_srcdir = \
                "%(monorepo_dir)s/llvm" % {'monorepo_dir' : self.monorepo_dir}
        self.obj_dir = \
                self.obj_dir or "build"

        # Repourl_prefix could be specified per builder. Otherwise we use github.
        self.repourl_prefix = kwargs.pop('repourl_prefix', 'https://github.com/llvm/')
Beispiel #18
0
    def test_worker_reconfigure_with_new_builder(self):
        """
        Checks if we can successfully reconfigure if we add new builders to worker.
        """
        config_dict = {
            'builders': [
                BuilderConfig(name="builder1",
                              workernames=['local1'],
                              factory=BuildFactory()),
            ],
            'workers': [self.createLocalWorker('local1', max_builds=1)],
            'protocols': {
                'null': {}
            },
            # Disable checks about missing scheduler.
            'multiMaster':
            True,
        }
        yield self.setup_master(config_dict)

        config_dict['builders'] += [
            BuilderConfig(name="builder2",
                          workernames=['local1'],
                          factory=BuildFactory()),
        ]
        config_dict['workers'] = [
            self.createLocalWorker('local1', max_builds=2)
        ]

        # reconfig should succeed
        yield self.reconfig_master(config_dict)
Beispiel #19
0
def masterConfig(extra_config):
    c = {}
    from buildbot.config import BuilderConfig
    from buildbot.process.buildstep import BuildStep
    from buildbot.process.factory import BuildFactory
    from buildbot.process import results

    class MyBuildStep(BuildStep):

        def start(self):
            self.finished(results.SUCCESS)

    c['change_source'] = []
    c['schedulers'] = []  # filled in above
    f1 = BuildFactory()
    f1.addStep(MyBuildStep(name='one'))
    f1.addStep(MyBuildStep(name='two'))
    c['builders'] = [
        BuilderConfig(name="a", workernames=["local1"], factory=f1),
    ]
    c['title'] = "test"
    c['titleURL'] = "test"
    c['buildbotURL'] = "http://localhost:8010/"
    c['mq'] = {'debug': True}
    # test wants to influence the config, but we still return a new config
    # each time
    c.update(extra_config)
    return c
Beispiel #20
0
def masterConfig():
    c = {}
    from buildbot.config import BuilderConfig
    from buildbot.process.factory import BuildFactory
    from buildbot.plugins import steps, schedulers

    c['schedulers'] = [
        schedulers.Triggerable(
            name="trigsched",
            builderNames=["build"]),
        schedulers.AnyBranchScheduler(
            name="sched",
            builderNames=["testy"])]

    f = BuildFactory()
    f.addStep(steps.ShellCommand(command='echo hello'))
    f.addStep(steps.Trigger(schedulerNames=['trigsched'],
                            waitForFinish=True,
                            updateSourceStamp=True))
    f.addStep(steps.ShellCommand(command='echo world'))
    f2 = BuildFactory()
    f2.addStep(steps.ShellCommand(command='echo ola'))
    c['builders'] = [
        BuilderConfig(name="testy",
                      slavenames=["local1"],
                      factory=f),
        BuilderConfig(name="build",
                      slavenames=["local1"],
                      factory=f2)]
    return c
def triggeredBuildIsNotCreated():
    f2 = BuildFactory()
    f2.addStep(steps.ShellCommand(command="echo 'hello'"))

    def nextBuild(*args, **kwargs):
        return defer.succeed(None)
    return setupTriggerConfiguration(f2, nextBuild=nextBuild)
Beispiel #22
0
def masterConfig():
    c = {}
    from buildbot.config import BuilderConfig
    from buildbot.process.factory import BuildFactory
    from buildbot.plugins import steps, schedulers, reporters
    c['schedulers'] = [
        schedulers.AnyBranchScheduler(
            name="sched",
            builderNames=["testy"])
    ]
    f = BuildFactory()
    f.addStep(steps.ShellCommand(command='echo hello'))
    c['builders'] = [
        BuilderConfig(name="testy",
                      workernames=["local1"],
                      factory=f)
    ]
    notifier = reporters.PushoverNotifier('1234', 'abcd', mode="all", watchedWorkers=['local1'],
                                          messageFormatter=MessageFormatter(template='This is a message.'),
                                          messageFormatterMissingWorker=MessageFormatterMissingWorker(
                                              template='No worker.'))
    c['services'] = [
        reporters.MailNotifier("*****@*****.**", mode="all"),
        notifier
    ]
    return c
Beispiel #23
0
    def __init__(
        self,
        python,
        source,
        uncleanWarnings,
        trialTests=None,
        trialMode=None,
        virtualenv=False,
        virtualenv_module="virtualenv",
        platform="unix",
        forceGarbageCollection=False,
    ):
        if not isinstance(source, list):
            source = [source]
        else:
            source = list(source)

        # If permissions get messed up on a slave, this can fix it.
        # But it breaks on old slaves so it's not enabled all the time
        # (and it can't fix old slaves, obviously).

        # self._fixPermissions(source)

        BuildFactory.__init__(self, source)

        if type(python) is str:
            python = [python, "-Wall"]

        assert platform in ["unix", "windows"]

        self._platform = platform
        if platform == "unix":
            self._path = posixpath
        elif platform == "windows":
            self._path = ntpath

        self.python = python
        self.virtualenv = virtualenv
        self.uncleanWarnings = uncleanWarnings
        self.forceGarbageCollection = forceGarbageCollection
        self.trialMode = trialMode
        if trialTests is None:
            trialTests = [WithProperties("%(test-case-name:~twisted)s")]
        self.trialTests = trialTests

        if virtualenv:
            # Hopefully temporary workaround for --clear not working:
            # https://github.com/pypa/virtualenv/issues/929
            self.addStep(
                shell.ShellCommand,
                command=self.python
                + ["-c", "import shutil, sys;" "shutil.rmtree(sys.argv[1], ignore_errors=True)", self._virtualEnvPath],
            )
            self.addStep(shell.ShellCommand, command=self.python + ["-m", virtualenv_module, self._virtualEnvPath])

        else:
            # Report the versions, since we're using the system ones. If it's a
            # virtualenv, it's up to the venv factory to report the versions
            # itself.
            self._reportVersions(python=self.python)
def masterConfig():
    c = {}
    from buildbot.config import BuilderConfig
    from buildbot.process.factory import BuildFactory
    from buildbot.plugins import schedulers, steps

    c['schedulers'] = [
        schedulers.ForceScheduler(
            name="force",
            builderNames=["testy"])]

    c['secretsProviders'] = [FakeSecretStorage(
        secretdict={"foo": "bar", "something": "more"})]
    f = BuildFactory()
    if os.name == "posix":
        f.addStep(steps.ShellCommand(command=Interpolate(
            'echo %(secret:foo)s | sed "s/bar/The password was there/"')))
    else:
        f.addStep(steps.ShellCommand(command=Interpolate(
            'echo %(secret:foo)s')))
    c['builders'] = [
        BuilderConfig(name="testy",
                      workernames=["local1"],
                      factory=f)]
    return c
def makeCleanOldBuildsFactory():
    """
    Remove build results older than 14 days.
    """
    # FIXME we shouldn't hard code this (DRY)
    basedir = r'/srv/buildmaster/data'
    path = os.path.join(basedir, "private_html")

    factory = BuildFactory()
    factory.addStep(MasterShellCommand(
        ['find', path,
         '-type', 'f', '-mtime', '+14',
         '-exec', 'unlink', '{}', ';', '-print'],
        description=['Removing', 'old', 'results'],
        descriptionDone=['Remove', 'old', 'results'],
        name='remove-old-results'))

    # Vagrant tutorial boxes are created on the Vagrant slave, and
    # uploaded to S3.  However, the boxes must be kept on the slave
    # for running subsequent tests
    # (flocker/acceptance/vagrant/centos-7/zfs and
    # flocker/installed-package/vagrant/centos-7). This means there is
    # not an obvious place to remove the boxes.  So, we periodically
    # cleanup old boxes here. "Old" is the number of days passed as
    # parameter to script.
    factory.addStep(ShellCommand(
        command=['python', '/home/buildslave/remove-old-boxes.py', '14'],
        description=['Removing', 'old', 'boxes'],
        descriptionDone=['Remove', 'old', 'boxes'],
        name='remove-old-boxes'))

    return factory
    def __init__(self, test, timeout, branch=None, talos_config_file='sample.config',
                 results_server=None, reboot=True, base_dir='/builds',
                 reboot_cmd=['sudo', 'reboot-user'], nochrome=False,
                 cvsroot=":pserver:[email protected]:/cvsroot",
                 hg_host='http://hg.mozilla.org', tools_repo_path='build/tools',
                 talos_tarball=None, pageloader_tarball=None,
                 cleanup_glob='tools talos fennec* *.tar.bz2 *.zip',
                 tp4_source='/tools/tp4', browser_wait=7, **kwargs):
        BuildFactory.__init__(self, **kwargs)
        self.test = test
        self.timeout = timeout
        self.branch = branch
        self.talos_config_file = talos_config_file
        self.results_server = results_server
        self.reboot = reboot
        self.base_dir = base_dir
        self.reboot_cmd = reboot_cmd
        self.nochrome = '' if nochrome is None else '--noChrome'
        self.cvsroot = cvsroot #We are using a static ip because of dns failures
        self.hg_host = hg_host
        self.tools_repo_path = tools_repo_path
        self.talos_tarball = talos_tarball
        self.pageloader_tarball = pageloader_tarball
        self.cleanup_glob = cleanup_glob
        self.tp4_source = tp4_source
        self.browser_wait = browser_wait

        self.addStartupSteps()
        self.addCleanupSteps()
        self.addSetupSteps()
        self.addObtainBuildSteps()
        self.addRunSteps()
        self.addFinalSteps()
Beispiel #27
0
def masterConfig(build_set_summary):
    c = {}
    from buildbot.config import BuilderConfig
    from buildbot.process.factory import BuildFactory
    from buildbot.plugins import steps, schedulers, reporters
    c['schedulers'] = [
        schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"])
    ]
    f = BuildFactory()
    f.addStep(steps.ShellCommand(command='echo hello'))
    c['builders'] = [
        BuilderConfig(name="testy", workernames=["local1"], factory=f)
    ]
    notifier = reporters.PushoverNotifier(
        '1234',
        'abcd',
        mode="all",
        watchedWorkers=['local1'],
        buildSetSummary=build_set_summary,
        messageFormatter=MessageFormatter(template='This is a message.'),
        messageFormatterMissingWorker=MessageFormatterMissingWorker(
            template='No worker.'))
    c['services'] = [
        reporters.MailNotifier("*****@*****.**",
                               mode="all",
                               buildSetSummary=build_set_summary), notifier
    ]
    return c
Beispiel #28
0
    def __init__(self, sources, arch):
        BuildFactory.__init__(self, sources)

        # Download the helpers
        for helper in ("pkgdepends", "pkgprovides", "pkgversion", "ccm-setup", 
                "changelog", "gitrev"):
            self.addStep(steps.FileDownload(name="helper " + helper,
                                            mastersrc="helpers/pkgbuild/" + helper,
                                            slavedest="../helpers/" + helper,
                                            mode=0755))
        # Create a directory to hold the packages that have been built
        self.addStep(steps.RemoveDirectory(name="rm-repository", dir="repository"))
        # Create a directory to hold the packages that have been built
        self.addStep(steps.MakeDirectory(name="mkdir-repository", dir="repository"))
        # Create or update the chroot
        self.addStep(PrepareCcm(arch=arch))
        # Copy the channel configuration from slave to master
        self.addStep(steps.FileUpload("channels.yml", "tmp/channels.yml", name="config-upload"))
        self.addStep(steps.FileUpload("buildinfo.yml", "tmp/buildinfo.yml", name="buildinfo-upload"))
        # Scan repository and find packages to build
        self.addStep(RepositoryScan(arch=arch))
        # Create a changelog for the repo
        self.addStep(Changelog(arch=arch))
        self.addStep(steps.FileDownload(mastersrc="tmp/buildinfo.yml", slavedest="buildinfo.yml", 
                name="buildinfo-download"))
        # Publish the repository
        self.addStep(steps.MasterShellCommand(command="rm -rf /srv/http/repos/papyros/" + arch))
        self.addStep(steps.DirectoryUpload('../repository', '/srv/http/repos/papyros/' + arch))
        self.addStep(steps.MasterShellCommand(command="chmod a+rX -R /srv/http/repos/papyros"))
        # Push back changes to version control (only push for x86_64 so we don't have duplicates)
        if arch == "x86_64":
            self.addStep(PushSourceChanges())
Beispiel #29
0
def masterConfig():
    c = {}
    from buildbot.config import BuilderConfig
    from buildbot.process.factory import BuildFactory
    from buildbot.plugins import schedulers

    c['schedulers'] = [
        schedulers.ForceScheduler(
            name="force",
            builderNames=["testy"])]

    # note that as of December 2018, the vault docker image default to kv
    # version 2 to be enabled by default
    c['secretsProviders'] = [HashiCorpVaultSecretProvider(
        vaultToken='my_vaulttoken',
        vaultServer="http://localhost:8200",
        apiVersion=2
    )]

    f = BuildFactory()
    f.addStep(ShellCommand(command=[Interpolate('echo %(secret:key)s')]))

    c['builders'] = [
        BuilderConfig(name="testy",
                      workernames=["local1"],
                      factory=f)]
    return c
Beispiel #30
0
    def __init__(self, repo, steps=None):
        self.basicSteps = [

        # Checkout
        s( SocialtextSVN
         , mode='update'
         , baseURL=repo
         , defaultBranch='/trunk'
         ),

        # Fix tarball (harmless on nonMacs)
        nlwShellStep( 'fix help tarball'
                    , 'dev-bin/fix-help-tarball-for-mac'
                    ),

        # Stop test servers; clear cached fixtures.
        nlwShellStep( 'cleanup'
                    , 'if [ -d t/tmp ]; then dev-bin/nlwctl -t stop; rm -r t/tmp*; fi'
                    ),

        # Configure.
        nlwShellStep( 'configure'
                    , './configure --dev=1 --apache-proxy=1 [email protected]'
                    )]

        BuildFactory.__init__(self, self.basicSteps + steps)
Beispiel #31
0
	def clean_chroot(cls, slaves):
		base_dir = "../../../"

		factory = BuildFactory()

		step_clean_chroot = ShellCommand(
				command = ["scripts/chroot-cleaner",
					WithProperties('%(branch)s')
					],
				workdir=base_dir,
				haltOnFailure = True,
				flunkOnFailure = True,
				description = "cleanup chroots",
				descriptionDone = "cleaned chroots", name = "cleanup chroots",
				interruptSignal="TERM")

		factory.addStep(step_clean_chroot)

		builder = BuilderConfig(
				name = "clean_chroot",
				category = "clean",
				slavenames = [ i.slavename for i in slaves ],
				properties = { "pkgname": "clean_chroot" },
				factory = factory,
				builddir = "builders/clean_chroot",
				slavebuilddir = "builders/clean_chroot",
				)

		return builder
Beispiel #32
0
    def __init__(self, repourl, submodules=False, branch='master',
                 codebase='', imageset=None):
        BuildFactory.__init__(self)
        self.addStep(steps.SetProperty(property='datestamp', value=datestamp))
        self.addStep(steps.Git(repourl=repourl, submodules=submodules,
                               branch=branch, codebase=codebase,
                               name='git-checkout-{}'.format(branch),
                               mode=('full' if submodules else 'incremental'),
                               method='clobber',
                               doStepIf=lambda step: not is_pull_request(step.build.getProperties()),
                               hideStepIf=lambda results, step: results == bbres.SKIPPED))
        self.addStep(steps.GitHub(repourl=repourl, submodules=submodules,
                                  branch=branch, codebase=codebase,
                                  name='git-checkout-pullrequest-ref',
                                  mode=('full' if submodules else 'incremental'),
                                  method='clobber',
                                  doStepIf=lambda step: is_pull_request(step.build.getProperties()),
                                  hideStepIf=lambda results, step: results == bbres.SKIPPED))
        env_vars = ENV_VARS.copy()
        # First, remove duplicates from PATH,
        # then strip out the virtualenv bin directory if we're in a virtualenv.
        setup_cmd = 'PATH=`echo -n "$PATH" | awk -v RS=: -v ORS=: \'!arr[$0]++\'`;' + \
                    'if [ -n "$VIRTUAL_ENV" ]; then ' + \
                    'PATH=`echo "$PATH" | sed -re "s,(^|:)$VIRTUAL_ENV/bin(:|$),\\2,g;s,^:,,"`; ' + \
                    'fi; . %(prop:setup_script)s; printenv'
        # Setup steps

        self.addStep(steps.RemoveDirectory('build/build', name='cleanup',
                                           description=['Removing', 'old', 'build', 'directory'],
                                           descriptionDone=['Removed', 'old', 'build', 'directory']))
        self.addStep(steps.SetPropertyFromCommand(command=['bash', '-c',
                                                           util.Interpolate(setup_cmd)],
                                                  extract_fn=extract_env_vars,
                                                  name='EnvironmentSetup',
                                                  description=['Running', 'setup', 'script'],
                                                  descriptionDone=['Ran', 'setup', 'script']))
        self.addStep(steps.StringDownload(s=make_autoconf, workerdest='auto.conf',
                                          workdir=util.Interpolate("%(prop:BUILDDIR)s/conf"), name='make-auto.conf',
                                          description=['Creating', 'auto.conf'],
                                          descriptionDone=['Created', 'auto.conf']))

        for i, img in enumerate(imageset.imagespecs, start=1):
            tgtenv = env_vars.copy()
            tgtenv.update(img.env)
            bbcmd = "bitbake"
            if img.keep_going:
                bbcmd += " -k"
            cmd = util.Interpolate(bbcmd + "%(kw:bitbake_options)s " + ' '.join(img.args),
                                   bitbake_options=bitbake_options)
            self.addStep(steps.ShellCommand(command=['bash', '-c', cmd], timeout=None,
                                            env=tgtenv, workdir=util.Property('BUILDDIR'),
                                            name='build_%s_%s' % (imageset.name, img.name),
                                            description=['Building', imageset.name, img.name],
                                            descriptionDone=['Built', imageset.name, img.name]))

        self.addStep(steps.ShellCommand(command=store_artifacts_cmd, workdir=util.Property('BUILDDIR'),
                                        name='StoreArtifacts', timeout=None,
                                        description=['Storing', 'artifacts'],
                                        descriptionDone=['Stored', 'artifacts']))
Beispiel #33
0
 def getFactory(self, branch, combo):
     factory = BuildFactory()
     factory.addStep(
         Git(repourl=self._project_git_repo, mode='copy',
             branch=branch))
     factory.addStep(
         EmacsCompile(command=["make", "clean", "all"]))
     return factory
Beispiel #34
0
def triggerRunsForever():
    f2 = BuildFactory()
    f2.addStep(
        steps.ShellCommand(command="\n".join(
            ['while :', 'do', ' echo "sleeping";', ' sleep 1;'
             'done'])))

    return setupTriggerConfiguration(f2)
def makeCleanOldResources():
    """
    Remove cloud instances (from acceptance and client tests) and cloud volumes
    more than two hours old.
    """
    factory = BuildFactory()
    factory.addStep(CleanAcceptanceInstances(lag=timedelta(hours=2)))
    factory.addStep(CleanVolumes(lag=timedelta(minutes=30)))
    return factory
 def getFactory(self):
     factory = BuildFactory()
     for step in self.factorySteps:
         if type(step) is list:
             raise Exception('Error: build step is list')
         if step is None:
             raise Exception('Error: build step is None')
         factory.addStep(step)
     return factory
def triggerRunsForever():
    f2 = BuildFactory()
    f2.addStep(steps.ShellCommand(command="\n".join(['while :',
                                                     'do',
                                                     ' echo "sleeping";',
                                                     ' sleep 1;'
                                                     'done'])))

    return setupTriggerConfiguration(f2)
Beispiel #38
0
    def make_bundler_builder(self):
        builder_name = "builder_bundler"
        factory = BuildFactory()
        repo_name = "bitmask_bundler"
        repo_url = "https://github.com/leapcode/" + repo_name + ".git"
        branch = "develop"

        workdir = "build"
        repo_dir = workdir + "/" + repo_name
        bundler_output_dir = "bundler_output"
        sumo_tarball = "leap.bitmask-latest-SUMO.tar.gz"

        publish_bundle = self._publish_bundle_command(
            '`ls -t *.tar.gz | head -1`')

        factory.addSteps([
            steps.Git(repourl=repo_url,
                      branch=branch,
                      workdir=repo_dir,
                      mode='full',
                      method='clobber',
                      shallow=True,
                      haltOnFailure=True,
                      name="Pull " + repo_url),
            steps.ShellCommand(command="rm -rf " + bundler_output_dir,
                               workdir=workdir,
                               name="Remove previous bundler dir"),
            steps.ShellCommand(command="mkdir " + bundler_output_dir,
                               workdir=workdir,
                               name="Create bundler dir"),
            steps.ShellCommand(command="cp bundle_pyinstaller.sh ../" +
                               bundler_output_dir,
                               workdir=repo_dir,
                               haltOnFailure=True,
                               name="Copy bundle_pyinstaller"),
            steps.ShellCommand(command="mkdir files",
                               workdir=workdir + '/' + bundler_output_dir,
                               name="Create auxiliary folder"),
            steps.ShellCommand(
                command="wget http://lizard.leap.se/sumo-tarball/%s" %
                sumo_tarball,
                workdir=workdir + '/' + bundler_output_dir,
                haltOnFailure=True,
                name="Download sumo"),
            steps.ShellCommand(command="./bundle_pyinstaller.sh " +
                               sumo_tarball,
                               workdir=workdir + '/' + bundler_output_dir,
                               name="Create bundle"),
            steps.ShellCommand(command=publish_bundle,
                               workdir=workdir + '/' + bundler_output_dir,
                               name="Publish bundle")
        ])

        return BuilderConfig(name=builder_name,
                             slavenames=self.slaves.leap_names(),
                             factory=factory)
Beispiel #39
0
    def test_local_worker_max_builds(self):
        """
        If max_builds is set, only one build is started on a worker
        at a time.
        """
        step_controller = StepController()
        config_dict = {
            'builders': [
                BuilderConfig(
                    name="testy-1",
                    workernames=["local"],
                    factory=BuildFactory([step_controller]),
                ),
                BuilderConfig(
                    name="testy-2",
                    workernames=["local"],
                    factory=BuildFactory([step_controller]),
                ),
            ],
            'workers': [LocalWorker('local', max_builds=1)],
            'protocols': {
                'null': {}
            },
            'multiMaster':
            True,
        }
        self.master = master = self.successResultOf(
            getMaster(self, self.reactor, config_dict))
        builder_ids = [
            self.successResultOf(master.data.updates.findBuilderId('testy-1')),
            self.successResultOf(master.data.updates.findBuilderId('testy-2')),
        ]

        started_builds = []
        self.successResultOf(
            master.mq.startConsuming(
                lambda key, build: started_builds.append(build),
                ('builds', None, 'new')))

        # Trigger a buildrequest
        bsid, brids = self.successResultOf(
            master.data.updates.addBuildset(
                waited_for=False,
                builderids=builder_ids,
                sourcestamps=[
                    {
                        'codebase': '',
                        'repository': '',
                        'branch': None,
                        'revision': None,
                        'project': ''
                    },
                ],
            ))

        self.assertEqual(len(started_builds), 1)
Beispiel #40
0
def MakeObsBuilder():
  f = BuildFactory()
  f.addSteps(svn_co)
  f.addStep(ShellCommand(
    name          = 'upload',
    command       = ['bash', 'os/obs/obs-upload.sh', Property('project'), WithProperties('%(version)s.%(got_revision)s')],
    haltOnFailure = True,
    logEnviron    = False
  ))
  return f
Beispiel #41
0
    def getTestingFactory(self, branch):
        factory = BuildFactory()
        _emacs_prop = 'EMACS=%(slave/binaries/emacs:-emacs)s'
        _eflags = '-L ../mocker'

        for p, r, t in self._deps:
            factory.addStep(
                DetachedGit(repourl=r, branch=t,
                            workdir=p, logEnviron=False,
                            description=['updating %s' % (p)],
                            descriptionDone=['update %s' % (p)]))
        factory.addStep(
            Git(repourl=self._project_git_repo, mode='copy',
                branch=branch, logEnviron=False))
        factory.addStep(
            EmacsCompile(command=["make",
                                  WithProperties(_emacs_prop),
                                  "clean", "all"],
                         logEnviron=False))
        factory.addStep(
            EmacsTest(command=["make",
                               WithProperties(_emacs_prop),
                               "EFLAGS=%s" % (_eflags),
                               "test"],
                      logEnviron=False))
        return factory
 def test_addStep_deprecated(self):
     """
     Passing keyword arguments to L{BuildFactory.addStep} is deprecated,
     but pass the arguments to the first argument, to construct a step.
     """
     factory = BuildFactory()
     factory.addStep(BuildStep)
     self.assertEqual(factory.steps, [_BuildStepFactory(BuildStep)])
     warnings = self.flushWarnings([self.test_addStep_deprecated])
     self.assertEqual(len(warnings), 1)
     self.assertEqual(warnings[0]['category'], DeprecationWarning)
def build_factory():
    factory = BuildFactory()

    # Step 1: check out the source
    factory.addStep(Git(repourl='git://github.com/buildbot/pyflakes.git',
                        mode='copy'))
    # Step 2: run the tests
    # (note that this will require that 'trial' is installed) 
    factory.addStep(ShellCommand(command=['trial', 'pyflakes']))
    
    return factory
Beispiel #44
0
 def test_addStep_deprecated(self):
     """
     Passing keyword arguments to L{BuildFactory.addStep} is deprecated,
     but pass the arguments to the first argument, to construct a step.
     """
     factory = BuildFactory()
     factory.addStep(BuildStep)
     self.assertEqual(factory.steps, [_BuildStepFactory(BuildStep)])
     warnings = self.flushWarnings([self.test_addStep_deprecated])
     self.assertEqual(len(warnings), 1)
     self.assertEqual(warnings[0]['category'], DeprecationWarning)
Beispiel #45
0
 def getBasicFactory(self, branch):
     factory = BuildFactory()
     _emacs_prop = 'EMACS=%(slave/binaries/emacs:-emacs)s'
     factory.addStep(
         Git(repourl=self._project_git_repo, mode='copy',
             branch=branch, logEnviron=False))
     factory.addStep(
         EmacsCompile(command=["make",
                               WithProperties(_emacs_prop),
                               "clean", "all"],
                      logEnviron=False))
     return factory
Beispiel #46
0
def ros_debbuild(c, job_name, packages, url, distro, arch, rosdistro, version, machines, othermirror, keys, trigger_pkgs = None):
    gbp_args = ['-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
                '--git-verbose', '--git-dist='+distro, '--git-arch='+arch]
    f = BuildFactory()
    # Remove the build directory.
    f.addStep(
        RemoveDirectory(
            name = job_name+'-clean',
            dir = Interpolate('%(prop:workdir)s'),
            hideStepIf = success,
        )
    )
    # Check out the repository master branch, since releases are tagged and not branched
    f.addStep(
        Git(
            repourl = url,
            branch = 'master',
            alwaysUseLatest = True, # this avoids broken builds when schedulers send wrong tag/rev
            mode = 'full' # clean out old versions
        )
    )
    # Update the cowbuilder
    f.addStep(
        ShellCommand(
            command = ['cowbuilder-update.py', distro, arch] + keys,
            hideStepIf = success
        )
    )
    # Need to build each package in order
    for package in packages:
        debian_pkg = 'ros-'+rosdistro+'-'+package.replace('_','-')  # debian package name (ros-groovy-foo)
        branch_name = 'debian/'+debian_pkg+'_%(prop:release_version)s_'+distro  # release branch from bloom
        deb_name = debian_pkg+'_%(prop:release_version)s'+distro
        final_name = debian_pkg+'_%(prop:release_version)s-%(prop:datestamp)s'+distro+'_'+arch+'.deb'
        # Check out the proper tag. Use --force to delete changes from previous deb stamping
        f.addStep(
            ShellCommand(
                haltOnFailure = True,
                name = package+'-checkout',
                command = ['git', 'checkout', Interpolate(branch_name), '--force'],
                hideStepIf = success
            )
        )
        # Download script for building the source deb
        f.addStep(
            FileDownload(
                name = job_name+'-grab-build-source-deb-script',
                mastersrc = 'scripts/build_source_deb.py',
                slavedest = Interpolate('%(prop:workdir)s/build_source_deb.py'),
                mode = 0755,
                hideStepIf = success
            )
        )
Beispiel #47
0
    def __init__(self, python, source, uncleanWarnings, trialTests=None,
                 trialMode=None, virtualenv=False,
                 virtualenv_module='virtualenv',
                 platform='unix',
                 forceGarbageCollection=False):
        if not isinstance(source, list):
            source = [source]
        else:
            source = list(source)

        # If permissions get messed up on a slave, this can fix it.
        # But it breaks on old slaves so it's not enabled all the time
        # (and it can't fix old slaves, obviously).

        # self._fixPermissions(source)

        BuildFactory.__init__(self, source)

        if type(python) is str:
            python = [python, "-Wall"]

        assert platform in ["unix", "windows"]

        self._platform = platform
        if platform == "unix":
            self._path = posixpath
        elif platform == "windows":
            self._path = ntpath

        self.python = python
        self.virtualenv = virtualenv
        self.uncleanWarnings = uncleanWarnings
        self.forceGarbageCollection = forceGarbageCollection
        self.trialMode = trialMode
        if trialTests is None:
            trialTests = [WithProperties("%(test-case-name:~twisted)s")]
        self.trialTests = trialTests

        if virtualenv:
            # Each time we create a new virtualenv as latest pip can build
            # wheels on the fly and install them from user's cache.
            self.addStep(
                shell.ShellCommand,
                command = self.python + [
                    "-m", virtualenv_module, '--clear', self._virtualEnvPath,
                    ],
                )

        else:
            # Report the versions, since we're using the system ones. If it's a
            # virtualenv, it's up to the venv factory to report the versions
            # itself.
            self._reportVersions()
class Job(object):
    """The base class for all job types.

    It doesn't do much by itself, except for creating a new Builder and a new Force Scheduler so
    that users can forcibly trigger a new build.

    This class can be used as a context manager although, by default, it does nothing.

    """

    def __init__(self, name, workers):
        self.name = utils.name(name)
        self.workers = workers

        self.build = BuildFactory()

        BuildmasterConfig['builders'].append(BuilderConfig(
            name=self.name,
            factory=self.build,
            slavenames=self.workers))

        BuildmasterConfig['schedulers'].append(ForceScheduler(
            name=scheduler_name(self, 'force'),
            builderNames=[self.name],
            branch=FixedParameter(name="reason", default=""),
            reason=FixedParameter(name="reason", default="Forced build"),
            revision=FixedParameter(name="revision", default=""),
            repository=FixedParameter(name="repository", default=""),
            project=FixedParameter(name="project", default=""),
            properties=[],
            username=FixedParameter(name="username", default="WebUI")))

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        pass

    def add_step(self, step):
        """Adds a build step on this Job."""
        self.build.addStep(step)

    def trigger(self, job_or_jobs):
        """Adds a build step which triggers execution of another job."""
        if type(job_or_jobs) is list:
            self.add_step(Trigger(
                schedulerNames=[scheduler_name(j, 'trigger') for j in job_or_jobs],
                waitForFinish=True))
        else:
            self.add_step(Trigger(
                schedulerNames=[scheduler_name(job_or_jobs, 'trigger')],
                waitForFinish=True))
Beispiel #49
0
    def test_worker_multiple_substantiations_succeed(self):
        """
        If multiple builders trigger try to substantiate a worker at
        the same time, if the substantiation succeeds then all of
        the builds proceed.
        """
        controller = LatentController('local')
        config_dict = {
            'builders': [
                BuilderConfig(
                    name="testy-1",
                    workernames=["local"],
                    factory=BuildFactory(),
                ),
                BuilderConfig(
                    name="testy-2",
                    workernames=["local"],
                    factory=BuildFactory(),
                ),
            ],
            'workers': [controller.worker],
            'protocols': {
                'null': {}
            },
            'multiMaster':
            True,
        }
        master = self.getMaster(config_dict)
        builder_ids = [
            self.successResultOf(master.data.updates.findBuilderId('testy-1')),
            self.successResultOf(master.data.updates.findBuilderId('testy-2')),
        ]

        finished_builds = []
        self.successResultOf(
            master.mq.startConsuming(
                lambda key, build: finished_builds.append(build),
                ('builds', None, 'finished')))

        # Trigger a buildrequest
        bsid, brids = self.createBuildrequest(master, builder_ids)

        # The worker succeeds to substantiate.
        controller.start_instance(True)

        controller.connect_worker(self)

        # We check that there were two builds that finished, and
        # that they both finished with success
        self.assertEqual([build['results'] for build in finished_builds],
                         [SUCCESS] * 2)
        controller.auto_stop(True)
Beispiel #50
0
    def config_for_master_command(self, **kwargs):
        c = {}

        c['schedulers'] = [
            schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"])
        ]

        f = BuildFactory()
        f.addStep(steps.MasterShellCommand(**kwargs))
        c['builders'] = [
            BuilderConfig(name="testy", workernames=["local1"], factory=f)
        ]
        return c
Beispiel #51
0
    def __init__(self,
                 python,
                 source,
                 uncleanWarnings,
                 trialTests=None,
                 trialMode=None):
        if not isinstance(source, list):
            source = [source]
        else:
            source = list(source)

        # If permissions get messed up on a slave, this can fix it.
        # But it breaks on old slaves so it's not enabled all the time
        # (and it can't fix old slaves, obviously).

        # self._fixPermissions(source)

        BuildFactory.__init__(self, source)

        if type(python) is str:
            python = [python]

        self.python = python
        self.uncleanWarnings = uncleanWarnings
        self.trialMode = trialMode
        if trialTests is None:
            trialTests = [WithProperties("%(test-case-name:~twisted)s")]
        self.trialTests = trialTests

        self.addStep(
            ReportPythonModuleVersions,
            python=self.python,
            moduleInfo=[
                ("Python", "sys", "sys.version"),
                ("OpenSSL", "OpenSSL", "OpenSSL.__version__"),
                ("PyCrypto", "Crypto", "Crypto.__version__"),
                ("gmpy", "gmpy", "gmpy.version()"),
                ("SOAPpy", "SOAPpy", "SOAPpy.__version__"),
                ("ctypes", "ctypes", "ctypes.__version__"),
                ("gtk", "gtk", "gtk.gtk_version"),
                ("pygtk", "gtk", "gtk.pygtk_version"),
                ("pywin32", "win32api",
                 "win32api.GetFileVersionInfo(win32api.__file__, chr(92))['FileVersionLS'] >> 16"
                 ),
                ("pyasn1", "pyasn1", "pyasn1.majorVersionId"),
                ("cffi", "cffi", "cffi.__version"),
            ],
            pkg_resources=[
                ("subunit", "subunit"),
                ("zope.interface", "zope.interface"),
            ])
Beispiel #52
0
    def addSimpleBuilder(self, name, buildername, category, repourl, builderconfig, sos, sbranch, bparams):
        """Private.
        Add a single builder on the given OS type for the given repo and branch.

        """

        factory = BuildFactory()
        factory.addStep(Git(repourl=repourl, mode='full', submodules=True, method='copy', branch=sbranch, getDescription={'tags': True}))
        if "tag" in builderconfig and not(builderconfig["tag"] is None):
            stag = builderconfig["tag"].encode('ascii', 'ignore')
            factory.addStep(ShellCommand(
                command=['git', 'checkout', stag],
                workdir="build",
                description="checkout tag"))
        # Delegate all knowlege of how to build to a script called buildshim in the project
        # directory.  Call this script in nine standardize steps.  It should ignore steps that it doesn't need.
        # Pass the step name as the first arg, and if params was given in the json for this builder, pass that as the
        # second arg.
        for step in ["patch", "install_deps", "configure", "compile", "check", "package", "upload", "compile_extra", "uninstall_deps"]:
            factory.addStep(ShellCommand(command=["./buildshim", step, bparams], description=step, haltOnFailure=True))

        self['builders'].append(
            BuilderConfig(name=buildername,
                          slavenames=self._os2slaves[sos],
                          factory=factory,
                          category=category))

        return factory
    def testGoodStep(self):
        f = BuildFactory()
        f.addStep(SetProperty(command=["echo", "value"], property="propname"))

        ss = SourceStamp()

        req = FakeBuildRequest("Testing", ss, None)

        b = f.newBuild([req])
        b.build_status = FakeBuildStatus()
        b.slavebuilder = FakeSlaveBuilder()

        # This shouldn't raise an exception
        b.setupBuild(None)
Beispiel #54
0
def createLinuxCIFactory():
    f = BuildFactory()

    f.addStep(
        Git(
            description="fetching sources",
            descriptionDone="sources",
            haltOnFailure=True,
            repourl=repositoryUri,
            mode='full',
            method='clobber',
        ))

    f.addStep(
        ShellCommand(description="fetching packages",
                     descriptionDone="packages",
                     haltOnFailure=True,
                     command=["mono", "paket.exe", "restore"],
                     workdir=workingDirectory))

    f.addStep(
        ShellCommand(description="building",
                     descriptionDone="build",
                     haltOnFailure=True,
                     command=["xbuild", "CorvusAlba.ToyFactory.Linux.sln"],
                     workdir=workingDirectory))

    return f
def catkin_ws_steps(job_name, repo, arch, oscode, distro):
    f = BuildFactory()
    # Remove the build directory.
    f.addStep(
        RemoveDirectory(
            name=job_name + '-clean',
            dir=Interpolate('%(prop:builddir)s'),
        ))
    # Check out the repository master branch, since releases are tagged and not branched
    f.addStep(
        Git(
            repourl=repo["url"],
            branch=repo["branch"],
            alwaysUseLatest=
            True,  # this avoids broken builds when schedulers send wrong tag/rev
            mode='full'  # clean out old versions
        ))

    # Build all packages in catkin_ws
    # If one package built failed, then failed
    f.addStep(
        ShellCommand(
            haltOnFailure=True,
            name=job_name + '-buildbinary',
            # Current workdir is the git source dir
            command=['bloom-local-deb'],
            descriptionDone=['binarydeb', job_name],
        ))
    return f
Beispiel #56
0
def MakeMacBuilder():
    f = BuildFactory()
    f.addSteps(svn_co)
    f.addStep(
        ShellCommand(
            name='revision',
            command=['bash', 'revision.sh'],
            workdir='build/os/macosx',
            haltOnFailure=True,
            env={'SCHAT_REVISION': Property('got_revision')},
        ))
    f.addStep(
        ShellCommand(
            name='dmg',
            command=['bash', 'deploy.sh'],
            workdir='build/os/macosx',
            haltOnFailure=True,
        ))
    f.addStep(
        FileUpload(
            mode=0644,
            slavesrc=WithProperties(
                'os/macosx/dmg/SimpleChat2-%(version)s.dmg'),
            masterdest=UploadFileName('SimpleChat2-%(version)s%(suffix)s.dmg'),
            url=WithProperties(
                'https://download.schat.me/schat2/snapshots/%(version)s/r%(got_revision)s/SimpleChat2-%(version)s%(suffix)s.dmg'
            ),
        ))
Beispiel #57
0
def MakeReleaseBuilder():
    f = BuildFactory()
    f.addStep(
        MasterShellCommand(
            name='Upload',
            command=[
                'bash',
                'upload.sh',
                SCHAT_VERSION,
                Property('revision'),
            ],
        ))
    f.addStep(
        MasterShellCommand(name='Create Update Channel',
                           command=[
                               'php',
                               'update.php',
                               '--channel',
                               Property('channel', default='stable'),
                               '--version',
                               SCHAT_VERSION,
                               '--revision',
                               Property('revision'),
                               '--os',
                               Property('os', default='win32,osx,ubuntu'),
                           ]))
    f.addStep(
        MasterShellCommand(name='Update Site',
                           command=[
                               'php',
                               'site.php',
                               '--version',
                               SCHAT_VERSION,
                           ]))
    return f
def createFactory():
    factory = BuildFactory()
    factory.addSteps(common.initTargetProperty())
    factory.addStep(steps.Trigger(
        name="Call the 'build' scheduler. Build CentOS",
        schedulerNames=['build'],
        waitForFinish=True,
        haltOnFailure=True,
        copy_properties=COMMON_BUILD_AND_TEST_SNAPSHOT_PROPERTIES,
        set_properties={
            "box": util.Property("box"),
            'try_already_running': 'yes',
            "target": util.Property("target"),
            'virtual_builder_name': util.Interpolate('Build for %(prop:box)s'),
        }
    ))
    factory.addStep(steps.Trigger(
        name="Call the 'run_test_snapshot' scheduler. Run functional tests",
        schedulerNames=['run_test_snapshot'],
        waitForFinish=True,
        copy_properties=COMMON_BUILD_AND_TEST_SNAPSHOT_PROPERTIES,
        set_properties={
            "box": util.Property("box"),
            "target": util.Property("target"),
            "test_branch": util.Property("branch"),
            "test_set": common.renderTestSet,
            "backend_ssl": util.Property("backend_ssl"),
            "use_valgrind": util.Property("use_valgrind"),
        }
    ))
    return factory
def create_factory():
    factory = BuildFactory()
    factory.addSteps(common.initTargetProperty())
    factory.addStep(
        steps.Trigger(name="Call the 'build' scheduler",
                      schedulerNames=['build'],
                      waitForFinish=True,
                      haltOnFailure=True,
                      copy_properties=COMMON_PROPERTIES,
                      set_properties={
                          'virtual_builder_name':
                          util.Interpolate('Build for %(prop:box)s'),
                          'box':
                          'ubuntu_bionic_libvirt',
                          'try_already_running':
                          'no',
                      }))

    factory.addStep(
        steps.Trigger(name="Call the 'run_performance_test' scheduler",
                      schedulerNames=['run_performance_test_trigger'],
                      waitForFinish=True,
                      copy_properties=COMMON_PROPERTIES,
                      set_properties={
                          'test_branch': util.Property('branch'),
                          'host': 'max-tst-01',
                      }))

    return factory