class TestInterpolateSecrets(unittest.TestCase, ConfigErrorsMixin):

    def setUp(self):
        self.master = fakemaster.make_master()
        fakeStorageService = FakeSecretStorage()
        fakeStorageService.reconfigService(secretdict={"foo": "bar",
                                                       "other": "value"})
        self.secretsrv = SecretManager()
        self.secretsrv.services = [fakeStorageService]
        self.secretsrv.setServiceParent(self.master)
        self.build = FakeBuildWithMaster(self.master)

    @defer.inlineCallbacks
    def test_secret(self):
        command = Interpolate("echo %(secret:foo)s")
        rendered = yield self.build.render(command)
        self.assertEqual(rendered, "echo bar")

    @defer.inlineCallbacks
    def test_secret_not_found(self):
        command = Interpolate("echo %(secret:fuo)s")
        yield self.assertFailure(self.build.render(command), defer.FirstError)
        gc.collect()
        self.flushLoggedErrors(defer.FirstError)
        self.flushLoggedErrors(KeyError)
class TestRenderSecrets(unittest.TestCase):

    def setUp(self):
        self.master = fakemaster.make_master()
        fakeStorageService = FakeSecretStorage(secretdict={"foo": "bar",
                                                       "other": "value"})
        self.secretsrv = SecretManager()
        self.secretsrv.services = [fakeStorageService]
        self.secretsrv.setServiceParent(self.master)
        self.srvtest = FakeServiceUsingSecrets()
        self.srvtest.setServiceParent(self.master)
        self.successResultOf(self.master.startService())

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.master.stopService()

    @defer.inlineCallbacks
    def test_secret_rendered(self):
        yield self.srvtest.configureService()
        new = FakeServiceUsingSecrets(foo=Secret("foo"), other=Secret("other"))
        yield self.srvtest.reconfigServiceWithSibling(new)
        self.assertEqual("bar", self.srvtest.returnRenderedSecrets("foo"))

    @defer.inlineCallbacks
    def test_secret_rendered_not_found(self):
        new = FakeServiceUsingSecrets(foo=Secret("foo"))
        yield self.srvtest.reconfigServiceWithSibling(new)
        self.assertRaises(Exception, self.srvtest.returnRenderedSecrets, "more")
Esempio n. 3
0
    def setUp(self):
        if requests is None:
            raise unittest.SkipTest("Need to install requests to test oauth2")

        self.patch(requests, 'request', mock.Mock(spec=requests.request))
        self.patch(requests, 'post', mock.Mock(spec=requests.post))
        self.patch(requests, 'get', mock.Mock(spec=requests.get))

        self.googleAuth = oauth2.GoogleAuth("ggclientID", "clientSECRET")
        self.githubAuth = oauth2.GitHubAuth("ghclientID", "clientSECRET")
        self.githubAuth_v4 = oauth2.GitHubAuth(
            "ghclientID", "clientSECRET", apiVersion=4)
        self.githubAuth_v4_teams = oauth2.GitHubAuth(
            "ghclientID", "clientSECRET", apiVersion=4, getTeamsMembership=True)
        self.githubAuthEnt = oauth2.GitHubAuth(
            "ghclientID", "clientSECRET", serverURL="https://git.corp.fakecorp.com")
        self.gitlabAuth = oauth2.GitLabAuth(
            "https://gitlab.test/", "glclientID", "clientSECRET")
        self.bitbucketAuth = oauth2.BitbucketAuth("bbclientID", "clientSECRET")

        for auth in [self.googleAuth, self.githubAuth, self.githubAuth_v4, self.githubAuth_v4_teams,
                     self.githubAuthEnt, self.gitlabAuth, self.bitbucketAuth]:
            self._master = master = self.make_master(url='h:/a/b/', auth=auth)
            auth.reconfigAuth(master, master.config)

        self.githubAuth_secret = oauth2.GitHubAuth(
            Secret("client-id"), Secret("client-secret"), apiVersion=4)
        self._master = master = self.make_master(url='h:/a/b/', auth=auth)
        fake_storage_service = FakeSecretStorage()
        fake_storage_service.reconfigService(secretdict={"client-id": "secretClientId",
                                                         "client-secret": "secretClientSecret"})
        secret_service = SecretManager()
        secret_service.services = [fake_storage_service]
        secret_service.setServiceParent(self._master)
        self.githubAuth_secret.reconfigAuth(master, master.config)
 def testGetNoDataManagerService(self):
     secret_service_manager = SecretManager()
     fakeStorageService = FakeSecretStorage()
     fakeStorageService.reconfigService(secretdict={"foo": "bar",
                                                    "other": "value"})
     secret_service_manager.services = [fakeStorageService]
     secret_result = yield secret_service_manager.get("foo2")
     self.assertEqual(secret_result, None)
 def testGetNoDataManagerService(self):
     secret_service_manager = SecretManager()
     fakeStorageService = FakeSecretStorage()
     fakeStorageService.reconfigService(secretdict={
         "foo": "bar",
         "other": "value"
     })
     secret_service_manager.services = [fakeStorageService]
     secret_result = yield secret_service_manager.get("foo2")
     self.assertEqual(secret_result, None)
 def testGetDataMultipleManagerServiceNoDatas(self):
     secret_service_manager = SecretManager()
     self.master.config.secretsProviders = [FakeSecretStorage({"foo": "bar",
                                                              "other": "value"}),
                                            FakeSecretStorage({"foo2": "bar",
                                                              "other2": "value"})
                                            ]
     SecretManager.master = self.master
     secret_result = yield secret_service_manager.get("foo3")
     self.assertEqual(secret_result, None)
Esempio n. 7
0
    def setUp(self):
        self.setUpTestReactor()
        if requests is None:
            raise unittest.SkipTest("Need to install requests to test oauth2")

        self.patch(requests, 'request', mock.Mock(spec=requests.request))
        self.patch(requests, 'post', mock.Mock(spec=requests.post))
        self.patch(requests, 'get', mock.Mock(spec=requests.get))

        self.googleAuth = oauth2.GoogleAuth("ggclientID", "clientSECRET")
        self.githubAuth = oauth2.GitHubAuth("ghclientID", "clientSECRET")
        self.githubAuth_v4 = oauth2.GitHubAuth("ghclientID",
                                               "clientSECRET",
                                               apiVersion=4)
        self.githubAuth_v4_teams = oauth2.GitHubAuth("ghclientID",
                                                     "clientSECRET",
                                                     apiVersion=4,
                                                     getTeamsMembership=True)
        self.githubAuthEnt = oauth2.GitHubAuth(
            "ghclientID",
            "clientSECRET",
            serverURL="https://git.corp.fakecorp.com")
        self.githubAuthEnt_v4 = oauth2.GitHubAuth(
            "ghclientID",
            "clientSECRET",
            apiVersion=4,
            getTeamsMembership=True,
            serverURL="https://git.corp.fakecorp.com")
        self.gitlabAuth = oauth2.GitLabAuth("https://gitlab.test/",
                                            "glclientID", "clientSECRET")
        self.bitbucketAuth = oauth2.BitbucketAuth("bbclientID", "clientSECRET")

        for auth in [
                self.googleAuth, self.githubAuth, self.githubAuth_v4,
                self.githubAuth_v4_teams, self.githubAuthEnt, self.gitlabAuth,
                self.bitbucketAuth, self.githubAuthEnt_v4
        ]:
            self._master = master = self.make_master(url='h:/a/b/', auth=auth)
            auth.reconfigAuth(master, master.config)

        self.githubAuth_secret = oauth2.GitHubAuth(Secret("client-id"),
                                                   Secret("client-secret"),
                                                   apiVersion=4)
        self._master = master = self.make_master(url='h:/a/b/', auth=auth)
        fake_storage_service = FakeSecretStorage()
        fake_storage_service.reconfigService(
            secretdict={
                "client-id": "secretClientId",
                "client-secret": "secretClientSecret"
            })
        secret_service = SecretManager()
        secret_service.services = [fake_storage_service]
        yield secret_service.setServiceParent(self._master)
        self.githubAuth_secret.reconfigAuth(master, master.config)
Esempio n. 8
0
 def setUp(self):
     self.setUpTestReactor()
     self.master = fakemaster.make_master(self)
     fakeStorageService = FakeSecretStorage(secretdict={"foo": "bar",
                                                    "other": "value"})
     self.secretsrv = SecretManager()
     self.secretsrv.services = [fakeStorageService]
     self.secretsrv.setServiceParent(self.master)
     self.srvtest = FakeServiceUsingSecrets()
     self.srvtest.setServiceParent(self.master)
     self.successResultOf(self.master.startService())
 def setUp(self):
     self.setup_test_reactor()
     self.master = fakemaster.make_master(self)
     fakeStorageService = FakeSecretStorage()
     password = "******"
     fakeStorageService.reconfigService(
         secretdict={"foo": password, "other": password + "random", "empty": ""})
     self.secretsrv = SecretManager()
     self.secretsrv.services = [fakeStorageService]
     yield self.secretsrv.setServiceParent(self.master)
     self.build = FakeBuildWithMaster(self.master)
Esempio n. 10
0
 def setUp(self):
     self.master = fakemaster.make_master()
     fakeStorageService = FakeSecretStorage()
     fakeStorageService.reconfigService(secretdict={
         "foo": "bar",
         "other": "value"
     })
     self.secretsrv = SecretManager()
     self.secretsrv.services = [fakeStorageService]
     self.secretsrv.setServiceParent(self.master)
     self.build = FakeBuildWithMaster(self.master)
Esempio n. 11
0
 def setUp(self):
     self.setUpTestReactor()
     self.master = fakemaster.make_master(self)
     fakeStorageService = FakeSecretStorage()
     password = "******"
     secretdict = {"foo": password, "other": password + 'random'}
     fakeStorageService.reconfigService(secretdict=secretdict)
     self.secretsrv = SecretManager()
     self.secretsrv.services = [fakeStorageService]
     yield self.secretsrv.setServiceParent(self.master)
     self.build = FakeBuildWithMaster(self.master)
Esempio n. 12
0
    def setUp(self):
        self.setUpTestReactor()

        fakeStorageService = FakeSecretStorage()
        fakeStorageService.reconfigService(secretdict={"secret_key": self._SECRET})

        secretService = SecretManager()
        secretService.services = [fakeStorageService]

        self.changeHook = _prepare_github_change_hook(self, strict=True,
                                                      secret=util.Secret("secret_key"))
        self.changeHook.master.addService(secretService)
 def testGetDataMultipleManagerValues(self):
     secret_service_manager = SecretManager()
     self.master.config.secretsProviders = [FakeSecretStorage({"foo": "bar",
                                                              "other": ""}),
                                            OtherFakeSecretStorage({"foo2": "bar2", "other": ""})
                                            ]
     SecretManager.master = self.master
     expectedSecretDetail = SecretDetails(FakeSecretStorage.__name__,
                                          "other",
                                          "")
     secret_result = yield secret_service_manager.get("other")
     self.assertEqual(secret_result, expectedSecretDetail)
Esempio n. 14
0
    def test_constructor_secrets(self):
        fake_storage_service = FakeSecretStorage()

        secret_service = SecretManager()
        secret_service.services = [fake_storage_service]
        yield secret_service.setServiceParent(self.master)

        fake_storage_service.reconfigService(secretdict={"passkey": "1234"})

        bs = yield self.createWorker('bot', util.Secret('passkey'))
        yield bs.startService()
        self.assertEqual(bs.password, '1234')
 def setUp(self):
     self.setup_test_reactor()
     self.master = fakemaster.make_master(self)
     fakeStorageService = FakeSecretStorage(secretdict={
         "foo": "bar",
         "other": "value"
     })
     self.secretsrv = SecretManager()
     self.secretsrv.services = [fakeStorageService]
     yield self.secretsrv.setServiceParent(self.master)
     self.srvtest = FakeServiceUsingSecrets()
     yield self.srvtest.setServiceParent(self.master)
     yield self.master.startService()
Esempio n. 16
0
 def testGetManagerService(self):
     secret_service_manager = SecretManager()
     SecretManager.master = self.master
     expectedClassName = FakeSecretStorage.__name__
     expectedSecretDetail = SecretDetails(expectedClassName, "foo", "bar")
     secret_result = yield secret_service_manager.get("foo")
     strExpectedSecretDetail = str(secret_result)
     self.assertEqual(secret_result, expectedSecretDetail)
     self.assertEqual(secret_result.key, "foo")
     self.assertEqual(secret_result.value, "bar")
     self.assertEqual(secret_result.source, expectedClassName)
     self.assertEqual(strExpectedSecretDetail,
                      "FakeSecretStorage foo: 'bar'")
Esempio n. 17
0
    def setUp(self):
        self.setUpTestReactor()

        fakeStorageService = FakeSecretStorage()
        fakeStorageService.reconfigService(
            secretdict={"secret_key": self._SECRET})

        secretService = SecretManager()
        secretService.services = [fakeStorageService]

        self.changeHook = _prepare_github_change_hook(
            self, strict=True, secret=util.Secret("secret_key"))
        self.changeHook.master.addService(secretService)
Esempio n. 18
0
    def setUp(self):
        self.setUpTestReactor()
        yield self.setUpChangeSource()

        fake_storage_service = FakeSecretStorage()

        secret_service = SecretManager()
        secret_service.services = [fake_storage_service]
        yield secret_service.setServiceParent(self.master)

        yield self.master.startService()

        fake_storage_service.reconfigService(secretdict={"token": "1234"})
 def testGetManagerService(self):
     secret_service_manager = SecretManager()
     SecretManager.master = self.master
     expectedClassName = FakeSecretStorage.__name__
     expectedSecretDetail = SecretDetails(expectedClassName, "foo", "bar")
     secret_result = yield secret_service_manager.get("foo")
     strExpectedSecretDetail = str(secret_result)
     self.assertEqual(secret_result, expectedSecretDetail)
     self.assertEqual(secret_result.key, "foo")
     self.assertEqual(secret_result.value, "bar")
     self.assertEqual(secret_result.source, expectedClassName)
     self.assertEqual(strExpectedSecretDetail,
                      "FakeSecretStorage foo: 'bar'")
    def testGetDataMultipleManagerValues(self):
        secret_service_manager = SecretManager()
        fakeStorageService = FakeSecretStorage()
        fakeStorageService.reconfigService(secretdict={"foo": "bar",
                                                              "other": ""})
        otherFakeStorageService = FakeSecretStorage()
        otherFakeStorageService.reconfigService(secretdict={"foo2": "bar2",
                                                            "other": ""})

        secret_service_manager.services = [fakeStorageService, otherFakeStorageService]
        expectedSecretDetail = SecretDetails(FakeSecretStorage.__name__,
                                             "other",
                                             "")
        secret_result = yield secret_service_manager.get("other")
        self.assertEqual(secret_result, expectedSecretDetail)
Esempio n. 21
0
 def testGetDataMultipleManagerServiceNoDatas(self):
     secret_service_manager = SecretManager()
     self.master.config.secretsManagers = [
         FakeSecretStorage({
             "foo": "bar",
             "other": "value"
         }),
         FakeSecretStorage({
             "foo2": "bar",
             "other2": "value"
         })
     ]
     SecretManager.master = self.master
     secret_result = yield secret_service_manager.get("foo3")
     self.assertEqual(secret_result, None)
Esempio n. 22
0
    def testGetDataMultipleManagerValues(self):
        secret_service_manager = SecretManager()
        fakeStorageService = FakeSecretStorage()
        fakeStorageService.reconfigService(secretdict={"foo": "bar",
                                                              "other": ""})
        otherFakeStorageService = FakeSecretStorage()
        otherFakeStorageService.reconfigService(secretdict={"foo2": "bar2",
                                                            "other": ""})

        secret_service_manager.services = [fakeStorageService, otherFakeStorageService]
        expectedSecretDetail = SecretDetails(FakeSecretStorage.__name__,
                                             "other",
                                             "")
        secret_result = yield secret_service_manager.get("other")
        self.assertEqual(secret_result, expectedSecretDetail)
 def testGetManagerService(self):
     secret_service_manager = SecretManager()
     fakeStorageService = FakeSecretStorage()
     fakeStorageService.reconfigService(secretdict={"foo": "bar",
                                                    "other": "value"})
     secret_service_manager.services = [fakeStorageService]
     expectedClassName = FakeSecretStorage.__name__
     expectedSecretDetail = SecretDetails(expectedClassName, "foo", "bar")
     secret_result = yield secret_service_manager.get("foo")
     strExpectedSecretDetail = str(secret_result)
     self.assertEqual(secret_result, expectedSecretDetail)
     self.assertEqual(secret_result.key, "foo")
     self.assertEqual(secret_result.value, "bar")
     self.assertEqual(secret_result.source, expectedClassName)
     self.assertEqual(strExpectedSecretDetail,
                      "FakeSecretStorage foo: 'bar'")
Esempio n. 24
0
 def setUp(self):
     self.master = fakemaster.make_master()
     self.master.config.secretsProviders = [FakeSecretStorage({"foo": "bar",
                                                              "other": "value"})]
     self.secretsrv = SecretManager()
     self.secretsrv.setServiceParent(self.master)
     self.build = FakeBuildWithMaster(self.master)
Esempio n. 25
0
class TestInterpolateSecrets(TestReactorMixin, unittest.TestCase,
                             ConfigErrorsMixin):

    @defer.inlineCallbacks
    def setUp(self):
        self.setup_test_reactor()
        self.master = fakemaster.make_master(self)
        fakeStorageService = FakeSecretStorage()
        fakeStorageService.reconfigService(secretdict={"foo": "bar",
                                                       "other": "value"})
        self.secretsrv = SecretManager()
        self.secretsrv.services = [fakeStorageService]
        yield self.secretsrv.setServiceParent(self.master)
        self.build = FakeBuildWithMaster(self.master)

    @defer.inlineCallbacks
    def test_secret(self):
        command = Interpolate("echo %(secret:foo)s")
        rendered = yield self.build.render(command)
        self.assertEqual(rendered, "echo bar")

    @defer.inlineCallbacks
    def test_secret_not_found(self):
        command = Interpolate("echo %(secret:fuo)s")
        yield self.assertFailure(self.build.render(command), defer.FirstError)
        gc.collect()
        self.flushLoggedErrors(defer.FirstError)
        self.flushLoggedErrors(KeyError)
Esempio n. 26
0
class TestInterpolateSecretsHiddenSecrets(TestReactorMixin, unittest.TestCase):

    @defer.inlineCallbacks
    def setUp(self):
        self.setup_test_reactor()
        self.master = fakemaster.make_master(self)
        fakeStorageService = FakeSecretStorage()
        password = "******"
        fakeStorageService.reconfigService(
            secretdict={"foo": password, "other": password + "random", "empty": ""})
        self.secretsrv = SecretManager()
        self.secretsrv.services = [fakeStorageService]
        yield self.secretsrv.setServiceParent(self.master)
        self.build = FakeBuildWithMaster(self.master)

    @defer.inlineCallbacks
    def test_secret(self):
        command = Interpolate("echo %(secret:foo)s")
        rendered = yield self.build.render(command)
        cleantext = self.build.properties.cleanupTextFromSecrets(rendered)
        self.assertEqual(cleantext, "echo <foo>")

    @defer.inlineCallbacks
    def test_secret_replace(self):
        command = Interpolate("echo %(secret:foo)s %(secret:other)s")
        rendered = yield self.build.render(command)
        cleantext = self.build.properties.cleanupTextFromSecrets(rendered)
        self.assertEqual(cleantext, "echo <foo> <other>")

    @defer.inlineCallbacks
    def test_secret_replace_with_empty_secret(self):
        command = Interpolate("echo %(secret:empty)s %(secret:other)s")
        rendered = yield self.build.render(command)
        cleantext = self.build.properties.cleanupTextFromSecrets(rendered)
        self.assertEqual(cleantext, "echo  <other>")
Esempio n. 27
0
 def testGetManagerService(self):
     secret_service_manager = SecretManager()
     fakeStorageService = FakeSecretStorage()
     fakeStorageService.reconfigService(secretdict={"foo": "bar",
                                                    "other": "value"})
     secret_service_manager.services = [fakeStorageService]
     expectedClassName = FakeSecretStorage.__name__
     expectedSecretDetail = SecretDetails(expectedClassName, "foo", "bar")
     secret_result = yield secret_service_manager.get("foo")
     strExpectedSecretDetail = str(secret_result)
     self.assertEqual(secret_result, expectedSecretDetail)
     self.assertEqual(secret_result.key, "foo")
     self.assertEqual(secret_result.value, "bar")
     self.assertEqual(secret_result.source, expectedClassName)
     self.assertEqual(strExpectedSecretDetail,
                      "FakeSecretStorage foo: 'bar'")
Esempio n. 28
0
class TestRenderSecrets(TestReactorMixin, unittest.TestCase):
    @defer.inlineCallbacks
    def setUp(self):
        self.setUpTestReactor()
        self.master = fakemaster.make_master(self)
        fakeStorageService = FakeSecretStorage(secretdict={
            "foo": "bar",
            "other": "value"
        })
        self.secretsrv = SecretManager()
        self.secretsrv.services = [fakeStorageService]
        yield self.secretsrv.setServiceParent(self.master)
        self.srvtest = FakeServiceUsingSecrets()
        yield self.srvtest.setServiceParent(self.master)
        yield self.master.startService()

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.master.stopService()

    @defer.inlineCallbacks
    def test_secret_rendered(self):
        yield self.srvtest.configureService()
        new = FakeServiceUsingSecrets(foo=Secret("foo"), other=Secret("other"))
        yield self.srvtest.reconfigServiceWithSibling(new)
        self.assertEqual("bar", self.srvtest.returnRenderedSecrets("foo"))

    @defer.inlineCallbacks
    def test_secret_rendered_not_found(self):
        new = FakeServiceUsingSecrets(foo=Secret("foo"))
        yield self.srvtest.reconfigServiceWithSibling(new)
        with self.assertRaises(Exception):
            self.srvtest.returnRenderedSecrets("more")
Esempio n. 29
0
 def testGetDataMultipleManagerService(self):
     secret_service_manager = SecretManager()
     self.master.config.secretsManagers = [
         FakeSecretStorage({
             "foo": "bar",
             "other": "value"
         }),
         OtherFakeSecretStorage({
             "foo2": "bar",
             "other2": "value"
         },
                                props={"property": "value_prop"})
     ]
     SecretManager.master = self.master
     expectedSecretDetail = SecretDetails(OtherFakeSecretStorage.__name__,
                                          "foo2", "bar")
     secret_result = yield secret_service_manager.get("foo2")
     self.assertEqual(secret_result, expectedSecretDetail)
Esempio n. 30
0
 def setUp(self):
     self.master = fakemaster.make_master()
     fakeStorageService = FakeSecretStorage()
     fakeStorageService.reconfigService(secretdict={"foo": "bar",
                                                    "other": "value"})
     self.secretsrv = SecretManager()
     self.secretsrv.services = [fakeStorageService]
     self.secretsrv.setServiceParent(self.master)
     self.build = FakeBuildWithMaster(self.master)
Esempio n. 31
0
class TestInterpolateSecretsHiddenSecrets(unittest.TestCase):

    def setUp(self):
        self.master = fakemaster.make_master()
        fakeStorageService = FakeSecretStorage()
        fakeStorageService.reconfigService(secretdict={"foo": "bar",
                                                       "other": "value"})
        self.secretsrv = SecretManager()
        self.secretsrv.services = [fakeStorageService]
        self.secretsrv.setServiceParent(self.master)
        self.build = FakeBuildWithMaster(self.master)

    @defer.inlineCallbacks
    def test_secret(self):
        command = Interpolate("echo %(secret:foo)s")
        rendered = yield self.build.render(command)
        cleantext = self.build.build_status.properties.cleanupTextFromSecrets(rendered)
        self.assertEqual(cleantext, "echo <foo>")
Esempio n. 32
0
class TestInterpolateSecretsHiddenSecrets(unittest.TestCase):

    def setUp(self):
        self.master = fakemaster.make_master()
        fakeStorageService = FakeSecretStorage()
        fakeStorageService.reconfigService(secretdict={"foo": "bar",
                                                       "other": "value"})
        self.secretsrv = SecretManager()
        self.secretsrv.services = [fakeStorageService]
        self.secretsrv.setServiceParent(self.master)
        self.build = FakeBuildWithMaster(self.master)

    @defer.inlineCallbacks
    def test_secret(self):
        command = Interpolate("echo %(secret:foo)s")
        rendered = yield self.build.render(command)
        cleantext = self.build.build_status.properties.cleanupTextFromSecrets(rendered)
        self.assertEqual(cleantext, "echo <foo>")
 def setUp(self):
     self.master = fakemaster.make_master()
     fakeStorageService = FakeSecretStorage(secretdict={"foo": "bar",
                                                    "other": "value"})
     self.secretsrv = SecretManager()
     self.secretsrv.services = [fakeStorageService]
     self.secretsrv.setServiceParent(self.master)
     self.srvtest = FakeServiceUsingSecrets()
     self.srvtest.setServiceParent(self.master)
     self.successResultOf(self.master.startService())
class TestInterpolateSecrets(unittest.TestCase, ConfigErrorsMixin):

    def setUp(self):
        self.master = fakemaster.make_master()
        self.master.config.secretsProviders = [FakeSecretStorage({"foo": "bar",
                                                                 "other": "value"})]
        self.secretsrv = SecretManager()
        self.secretsrv.setServiceParent(self.master)
        self.build = FakeBuildWithMaster(self.master)

    @defer.inlineCallbacks
    def test_secret(self):
        command = Interpolate("echo %(secrets:foo)s")
        rendered = yield self.build.render(command)
        self.assertEqual(rendered, "echo bar")

    def test_secret_not_found(self):
        command = Interpolate("echo %(secrets:fuo)s")
        self.assertFailure(self.build.render(command), defer.FirstError)
        self.flushLoggedErrors(defer.FirstError)
        self.flushLoggedErrors(KeyError)
Esempio n. 35
0
    def setUp(self):
        self.setup_test_reactor()
        self.master = fakeMasterForHooks(self)

        fakeStorageService = FakeSecretStorage()
        fakeStorageService.reconfigService(secretdict={"secret_key": self._SECRET})

        self.secretService = SecretManager()
        self.secretService.services = [fakeStorageService]
        self.master.addService(self.secretService)

        self.changeHook = change_hook.ChangeHookResource(
            dialects={'gitlab': {'secret': util.Secret("secret_key")}},
            master=self.master)
Esempio n. 36
0
    def create_child_services(self):
        # note that these are order-dependent.  If you get the order wrong,
        # you'll know it, as the master will fail to start.

        self.metrics = metrics.MetricLogObserver()
        self.metrics.setServiceParent(self)

        self.caches = cache.CacheManager()
        self.caches.setServiceParent(self)

        self.pbmanager = buildbot.pbmanager.PBManager()
        self.pbmanager.setServiceParent(self)

        self.workers = workermanager.WorkerManager(self)
        self.workers.setServiceParent(self)

        self.change_svc = ChangeManager()
        self.change_svc.setServiceParent(self)

        self.botmaster = BotMaster()
        self.botmaster.setServiceParent(self)

        self.scheduler_manager = SchedulerManager()
        self.scheduler_manager.setServiceParent(self)

        self.user_manager = UserManagerManager(self)
        self.user_manager.setServiceParent(self)

        self.db = dbconnector.DBConnector(self.basedir)
        self.db.setServiceParent(self)

        self.wamp = wampconnector.WampConnector()
        self.wamp.setServiceParent(self)

        self.mq = mqconnector.MQConnector()
        self.mq.setServiceParent(self)

        self.data = dataconnector.DataConnector()
        self.data.setServiceParent(self)

        self.www = wwwservice.WWWService()
        self.www.setServiceParent(self)

        self.debug = debug.DebugServices()
        self.debug.setServiceParent(self)

        self.status = Status()
        self.status.setServiceParent(self)

        self.secrets_manager = SecretManager()
        self.secrets_manager.setServiceParent(self)
        self.secrets_manager.reconfig_priority = 2000

        self.service_manager = service.BuildbotServiceManager()
        self.service_manager.setServiceParent(self)
        self.service_manager.reconfig_priority = 1000

        self.masterHouskeepingTimer = 0

        @defer.inlineCallbacks
        def heartbeat():
            if self.masterid is not None:
                yield self.data.updates.masterActive(name=self.name,
                                                     masterid=self.masterid)
            yield self.data.updates.expireMasters()

        self.masterHeartbeatService = internet.TimerService(60, heartbeat)
        self.masterHeartbeatService.clock = self.reactor
 def testGetNoDataManagerService(self):
     secret_service_manager = SecretManager()
     SecretManager.master = self.master
     secret_result = yield secret_service_manager.get("foo2")
     self.assertEqual(secret_result, None)
Esempio n. 38
0
class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService,
                  WorkerAPICompatMixin):

    # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
    # unclaimed; this should be at least 2 to avoid false positives
    UNCLAIMED_BUILD_FACTOR = 6

    def __init__(self, basedir, configFileName=None, umask=None, reactor=None, config_loader=None):
        service.AsyncMultiService.__init__(self)

        if reactor is None:
            from twisted.internet import reactor
        self.reactor = reactor

        self.setName("buildmaster")

        self.umask = umask

        self.basedir = basedir
        if basedir is not None:  # None is used in tests
            assert os.path.isdir(self.basedir)

        if config_loader is not None and configFileName is not None:
            raise config.ConfigErrors([
                "Can't specify both `config_loader` and `configFilename`.",
            ])
        elif config_loader is None:
            if configFileName is None:
                configFileName = 'master.cfg'
            config_loader = config.FileLoader(self.basedir, configFileName)
        self.config_loader = config_loader
        self.configFileName = configFileName

        # flag so we don't try to do fancy things before the master is ready
        self._master_initialized = False
        self.initLock = defer.DeferredLock()

        # set up child services
        self.create_child_services()

        # db configured values
        self.configured_db_url = None

        # configuration / reconfiguration handling
        self.config = config.MasterConfig()
        self.reconfig_active = False
        self.reconfig_requested = False
        self.reconfig_notifier = None

        # this stores parameters used in the tac file, and is accessed by the
        # WebStatus to duplicate those values.
        self.log_rotation = LogRotation()

        # local cache for this master's object ID
        self._object_id = None

        # Check environment is sensible
        check_functional_environment(self.config)

        # figure out local hostname
        try:
            self.hostname = os.uname()[1]  # only on unix
        except AttributeError:
            self.hostname = socket.getfqdn()

        # public attributes
        self.name = ("%s:%s" % (self.hostname,
                                os.path.abspath(self.basedir or '.')))
        if isinstance(self.name, bytes):
            self.name = self.name.decode('ascii', 'replace')
        self.masterid = None

    def create_child_services(self):
        # note that these are order-dependent.  If you get the order wrong,
        # you'll know it, as the master will fail to start.

        self.metrics = metrics.MetricLogObserver()
        self.metrics.setServiceParent(self)

        self.caches = cache.CacheManager()
        self.caches.setServiceParent(self)

        self.pbmanager = buildbot.pbmanager.PBManager()
        self.pbmanager.setServiceParent(self)

        self.workers = workermanager.WorkerManager(self)
        self.workers.setServiceParent(self)

        self.change_svc = ChangeManager()
        self.change_svc.setServiceParent(self)

        self.botmaster = BotMaster()
        self.botmaster.setServiceParent(self)

        self.scheduler_manager = SchedulerManager()
        self.scheduler_manager.setServiceParent(self)

        self.user_manager = UserManagerManager(self)
        self.user_manager.setServiceParent(self)

        self.db = dbconnector.DBConnector(self.basedir)
        self.db.setServiceParent(self)

        self.wamp = wampconnector.WampConnector()
        self.wamp.setServiceParent(self)

        self.mq = mqconnector.MQConnector()
        self.mq.setServiceParent(self)

        self.data = dataconnector.DataConnector()
        self.data.setServiceParent(self)

        self.www = wwwservice.WWWService()
        self.www.setServiceParent(self)

        self.debug = debug.DebugServices()
        self.debug.setServiceParent(self)

        self.status = Status()
        self.status.setServiceParent(self)

        self.secrets_manager = SecretManager()
        self.secrets_manager.setServiceParent(self)
        self.secrets_manager.reconfig_priority = 2000

        self.service_manager = service.BuildbotServiceManager()
        self.service_manager.setServiceParent(self)
        self.service_manager.reconfig_priority = 1000

        self.masterHouskeepingTimer = 0

        @defer.inlineCallbacks
        def heartbeat():
            if self.masterid is not None:
                yield self.data.updates.masterActive(name=self.name,
                                                     masterid=self.masterid)
            yield self.data.updates.expireMasters()
        self.masterHeartbeatService = internet.TimerService(60, heartbeat)
        # we do setServiceParent only when the master is configured
        # master should advertise itself only at that time

    # setup and reconfig handling

    _already_started = False

    @defer.inlineCallbacks
    def startService(self):
        assert not self._already_started, "can only start the master once"
        self._already_started = True

        log.msg("Starting BuildMaster -- buildbot.version: %s" %
                buildbot.version)

        # Set umask
        if self.umask is not None:
            os.umask(self.umask)

        # first, apply all monkeypatches
        monkeypatches.patch_all()

        # we want to wait until the reactor is running, so we can call
        # reactor.stop() for fatal errors
        d = defer.Deferred()
        self.reactor.callWhenRunning(d.callback, None)
        yield d

        startup_succeed = False
        try:
            yield self.initLock.acquire()
            # load the configuration file, treating errors as fatal
            try:
                # run the master.cfg in thread, so that it can use blocking
                # code
                self.config = yield threads.deferToThreadPool(
                    self.reactor, self.reactor.getThreadPool(),
                    self.config_loader.loadConfig)

            except config.ConfigErrors as e:
                log.msg("Configuration Errors:")
                for msg in e.errors:
                    log.msg("  " + msg)
                log.msg("Halting master.")
                self.reactor.stop()
                return
            except Exception:
                log.err(failure.Failure(), 'while starting BuildMaster')
                self.reactor.stop()
                return

            # set up services that need access to the config before everything
            # else gets told to reconfig
            try:
                yield self.db.setup()
            except exceptions.DatabaseNotReadyError:
                # (message was already logged)
                self.reactor.stop()
                return

            self.mq.setup()

            if hasattr(signal, "SIGHUP"):
                def sighup(*args):
                    eventually(self.reconfig)
                signal.signal(signal.SIGHUP, sighup)

            if hasattr(signal, "SIGUSR1"):
                def sigusr1(*args):
                    eventually(self.botmaster.cleanShutdown)
                signal.signal(signal.SIGUSR1, sigusr1)

            # get the masterid so other services can use it in
            # startup/reconfig.  This goes directly to the DB since the data
            # API isn't initialized yet, and anyway, this method is aware of
            # the DB API since it just called its setup function
            self.masterid = yield self.db.masters.findMasterId(
                name=self.name)

            # mark this master as stopped, in case it crashed before
            yield self.data.updates.masterStopped(name=self.name,
                                                  masterid=self.masterid)

            # call the parent method
            yield service.AsyncMultiService.startService(self)

            # We make sure the housekeeping is done before configuring in order to cleanup
            # any remaining claimed schedulers or change sources from zombie
            # masters
            yield self.data.updates.expireMasters(forceHouseKeeping=True)

            # give all services a chance to load the new configuration, rather
            # than the base configuration
            yield self.reconfigServiceWithBuildbotConfig(self.config)

            # Mark the master as active now that mq is running
            yield self.data.updates.masterActive(name=self.name,
                                                 masterid=self.masterid)

            # Start the heartbeat timer
            yield self.masterHeartbeatService.setServiceParent(self)

            # send the statistics to buildbot.net, without waiting
            self.sendBuildbotNetUsageData()
            startup_succeed = True
        except Exception:
            f = failure.Failure()
            log.err(f, 'while starting BuildMaster')
            self.reactor.stop()

        finally:
            if startup_succeed:
                log.msg("BuildMaster is running")
            else:
                log.msg("BuildMaster startup failed")

            yield self.initLock.release()
            self._master_initialized = True

    def sendBuildbotNetUsageData(self):
        if "TRIAL_PYTHONPATH" in os.environ and self.config.buildbotNetUsageData is not None:
            raise RuntimeError(
                "Should not enable buildbotNetUsageData in trial tests!")
        sendBuildbotNetUsageData(self)

    @defer.inlineCallbacks
    def stopService(self):
        try:
            yield self.initLock.acquire()
            if self.masterid is not None:
                yield self.data.updates.masterStopped(
                    name=self.name, masterid=self.masterid)
            if self.running:
                yield self.botmaster.cleanShutdown(
                    quickMode=True, stopReactor=False)
                yield service.AsyncMultiService.stopService(self)

            log.msg("BuildMaster is stopped")
            self._master_initialized = False
        finally:
            yield self.initLock.release()

    def reconfig(self):
        # this method wraps doConfig, ensuring it is only ever called once at
        # a time, and alerting the user if the reconfig takes too long
        if self.reconfig_active:
            log.msg("reconfig already active; will reconfig again after")
            self.reconfig_requested = True
            return

        self.reconfig_active = self.reactor.seconds()
        metrics.MetricCountEvent.log("loaded_config", 1)

        # notify every 10 seconds that the reconfig is still going on, although
        # reconfigs should not take that long!
        self.reconfig_notifier = task.LoopingCall(lambda:
                                                  log.msg("reconfig is ongoing for %d s" %
                                                          (self.reactor.seconds() - self.reconfig_active)))
        self.reconfig_notifier.start(10, now=False)

        timer = metrics.Timer("BuildMaster.reconfig")
        timer.start()

        d = self.doReconfig()

        @d.addBoth
        def cleanup(res):
            timer.stop()
            self.reconfig_notifier.stop()
            self.reconfig_notifier = None
            self.reconfig_active = False
            if self.reconfig_requested:
                self.reconfig_requested = False
                self.reconfig()
            return res

        d.addErrback(log.err, 'while reconfiguring')

        return d  # for tests

    @defer.inlineCallbacks
    def doReconfig(self):
        log.msg("beginning configuration update")
        changes_made = False
        failed = False
        try:
            yield self.initLock.acquire()
            # Run the master.cfg in thread, so that it can use blocking code
            new_config = yield threads.deferToThreadPool(
                self.reactor, self.reactor.getThreadPool(),
                self.config_loader.loadConfig)
            changes_made = True
            self.config = new_config

            yield self.reconfigServiceWithBuildbotConfig(new_config)

        except config.ConfigErrors as e:
            for msg in e.errors:
                log.msg(msg)
            failed = True

        except Exception:
            log.err(failure.Failure(), 'during reconfig:')
            failed = True

        finally:
            yield self.initLock.release()

        if failed:
            if changes_made:
                log.msg("WARNING: reconfig partially applied; master "
                        "may malfunction")
            else:
                log.msg("reconfig aborted without making any changes")
        else:
            log.msg("configuration update complete")

    def reconfigServiceWithBuildbotConfig(self, new_config):
        if self.configured_db_url is None:
            self.configured_db_url = new_config.db['db_url']
        elif (self.configured_db_url != new_config.db['db_url']):
            config.error(
                "Cannot change c['db']['db_url'] after the master has started",
            )

        if self.config.mq['type'] != new_config.mq['type']:
            raise config.ConfigErrors([
                "Cannot change c['mq']['type'] after the master has started",
            ])

        return service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self,
                                                                                    new_config)

    # informational methods
    def allSchedulers(self):
        return list(self.scheduler_manager)

    def getStatus(self):
        """
        @rtype: L{buildbot.status.builder.Status}
        """
        return self.status

    # state maintenance (private)
    def getObjectId(self):
        """
        Return the object id for this master, for associating state with the
        master.

        @returns: ID, via Deferred
        """
        # try to get the cached value
        if self._object_id is not None:
            return defer.succeed(self._object_id)

        # failing that, get it from the DB; multiple calls to this function
        # at the same time will not hurt

        d = self.db.state.getObjectId(self.name,
                                      "buildbot.master.BuildMaster")

        @d.addCallback
        def keep(id):
            self._object_id = id
            return id
        return d

    def _getState(self, name, default=None):
        "private wrapper around C{self.db.state.getState}"
        d = self.getObjectId()

        @d.addCallback
        def get(objectid):
            return self.db.state.getState(objectid, name, default)
        return d

    def _setState(self, name, value):
        "private wrapper around C{self.db.state.setState}"
        d = self.getObjectId()

        @d.addCallback
        def set(objectid):
            return self.db.state.setState(objectid, name, value)
        return d
Esempio n. 39
0
    def create_child_services(self):
        # note that these are order-dependent.  If you get the order wrong,
        # you'll know it, as the master will fail to start.

        self.metrics = metrics.MetricLogObserver()
        self.metrics.setServiceParent(self)

        self.caches = cache.CacheManager()
        self.caches.setServiceParent(self)

        self.pbmanager = buildbot.pbmanager.PBManager()
        self.pbmanager.setServiceParent(self)

        self.workers = workermanager.WorkerManager(self)
        self.workers.setServiceParent(self)

        self.change_svc = ChangeManager()
        self.change_svc.setServiceParent(self)

        self.botmaster = BotMaster()
        self.botmaster.setServiceParent(self)

        self.scheduler_manager = SchedulerManager()
        self.scheduler_manager.setServiceParent(self)

        self.user_manager = UserManagerManager(self)
        self.user_manager.setServiceParent(self)

        self.db = dbconnector.DBConnector(self.basedir)
        self.db.setServiceParent(self)

        self.wamp = wampconnector.WampConnector()
        self.wamp.setServiceParent(self)

        self.mq = mqconnector.MQConnector()
        self.mq.setServiceParent(self)

        self.data = dataconnector.DataConnector()
        self.data.setServiceParent(self)

        self.www = wwwservice.WWWService()
        self.www.setServiceParent(self)

        self.debug = debug.DebugServices()
        self.debug.setServiceParent(self)

        self.status = Status()
        self.status.setServiceParent(self)

        self.secrets_manager = SecretManager()
        self.secrets_manager.setServiceParent(self)
        self.secrets_manager.reconfig_priority = 2000

        self.service_manager = service.BuildbotServiceManager()
        self.service_manager.setServiceParent(self)
        self.service_manager.reconfig_priority = 1000

        self.masterHouskeepingTimer = 0

        @defer.inlineCallbacks
        def heartbeat():
            if self.masterid is not None:
                yield self.data.updates.masterActive(name=self.name,
                                                     masterid=self.masterid)
            yield self.data.updates.expireMasters()
        self.masterHeartbeatService = internet.TimerService(60, heartbeat)
Esempio n. 40
0
class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService,
                  WorkerAPICompatMixin):

    # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
    # unclaimed; this should be at least 2 to avoid false positives
    UNCLAIMED_BUILD_FACTOR = 6

    def __init__(self,
                 basedir,
                 configFileName=None,
                 umask=None,
                 reactor=None,
                 config_loader=None):
        service.AsyncMultiService.__init__(self)

        if reactor is None:
            from twisted.internet import reactor
        self.reactor = reactor

        self.setName("buildmaster")

        self.umask = umask

        self.basedir = basedir
        if basedir is not None:  # None is used in tests
            assert os.path.isdir(self.basedir)

        if config_loader is not None and configFileName is not None:
            raise config.ConfigErrors([
                "Can't specify both `config_loader` and `configFilename`.",
            ])
        elif config_loader is None:
            if configFileName is None:
                configFileName = 'master.cfg'
            config_loader = config.FileLoader(self.basedir, configFileName)
        self.config_loader = config_loader
        self.configFileName = configFileName

        # flag so we don't try to do fancy things before the master is ready
        self._master_initialized = False
        self.initLock = defer.DeferredLock()

        # set up child services
        self.create_child_services()

        # db configured values
        self.configured_db_url = None

        # configuration / reconfiguration handling
        self.config = config.MasterConfig()
        self.reconfig_active = False
        self.reconfig_requested = False
        self.reconfig_notifier = None

        # this stores parameters used in the tac file, and is accessed by the
        # WebStatus to duplicate those values.
        self.log_rotation = LogRotation()

        # local cache for this master's object ID
        self._object_id = None

        # Check environment is sensible
        check_functional_environment(self.config)

        # figure out local hostname
        try:
            self.hostname = os.uname()[1]  # only on unix
        except AttributeError:
            self.hostname = socket.getfqdn()

        # public attributes
        self.name = ("%s:%s" %
                     (self.hostname, os.path.abspath(self.basedir or '.')))
        if isinstance(self.name, bytes):
            self.name = self.name.decode('ascii', 'replace')
        self.masterid = None

    def create_child_services(self):
        # note that these are order-dependent.  If you get the order wrong,
        # you'll know it, as the master will fail to start.

        self.metrics = metrics.MetricLogObserver()
        self.metrics.setServiceParent(self)

        self.caches = cache.CacheManager()
        self.caches.setServiceParent(self)

        self.pbmanager = buildbot.pbmanager.PBManager()
        self.pbmanager.setServiceParent(self)

        self.workers = workermanager.WorkerManager(self)
        self.workers.setServiceParent(self)

        self.change_svc = ChangeManager()
        self.change_svc.setServiceParent(self)

        self.botmaster = BotMaster()
        self.botmaster.setServiceParent(self)

        self.scheduler_manager = SchedulerManager()
        self.scheduler_manager.setServiceParent(self)

        self.user_manager = UserManagerManager(self)
        self.user_manager.setServiceParent(self)

        self.db = dbconnector.DBConnector(self.basedir)
        self.db.setServiceParent(self)

        self.wamp = wampconnector.WampConnector()
        self.wamp.setServiceParent(self)

        self.mq = mqconnector.MQConnector()
        self.mq.setServiceParent(self)

        self.data = dataconnector.DataConnector()
        self.data.setServiceParent(self)

        self.www = wwwservice.WWWService()
        self.www.setServiceParent(self)

        self.debug = debug.DebugServices()
        self.debug.setServiceParent(self)

        self.status = Status()
        self.status.setServiceParent(self)

        self.secrets_manager = SecretManager()
        self.secrets_manager.setServiceParent(self)
        self.secrets_manager.reconfig_priority = 2000

        self.service_manager = service.BuildbotServiceManager()
        self.service_manager.setServiceParent(self)
        self.service_manager.reconfig_priority = 1000

        self.masterHouskeepingTimer = 0

        @defer.inlineCallbacks
        def heartbeat():
            if self.masterid is not None:
                yield self.data.updates.masterActive(name=self.name,
                                                     masterid=self.masterid)
            yield self.data.updates.expireMasters()

        self.masterHeartbeatService = internet.TimerService(60, heartbeat)
        self.masterHeartbeatService.clock = self.reactor
        # we do setServiceParent only when the master is configured
        # master should advertise itself only at that time

    # setup and reconfig handling

    _already_started = False

    @defer.inlineCallbacks
    def startService(self):
        assert not self._already_started, "can only start the master once"
        self._already_started = True

        log.msg("Starting BuildMaster -- buildbot.version: %s" %
                buildbot.version)

        # Set umask
        if self.umask is not None:
            os.umask(self.umask)

        # first, apply all monkeypatches
        monkeypatches.patch_all()

        # we want to wait until the reactor is running, so we can call
        # reactor.stop() for fatal errors
        d = defer.Deferred()
        self.reactor.callWhenRunning(d.callback, None)
        yield d

        startup_succeed = False
        try:
            yield self.initLock.acquire()
            # load the configuration file, treating errors as fatal
            try:
                # run the master.cfg in thread, so that it can use blocking
                # code
                self.config = yield threads.deferToThreadPool(
                    self.reactor, self.reactor.getThreadPool(),
                    self.config_loader.loadConfig)

            except config.ConfigErrors as e:
                log.msg("Configuration Errors:")
                for msg in e.errors:
                    log.msg("  " + msg)
                log.msg("Halting master.")
                self.reactor.stop()
                return
            except Exception:
                log.err(failure.Failure(), 'while starting BuildMaster')
                self.reactor.stop()
                return

            # set up services that need access to the config before everything
            # else gets told to reconfig
            try:
                yield self.db.setup()
            except exceptions.DatabaseNotReadyError:
                # (message was already logged)
                self.reactor.stop()
                return

            self.mq.setup()

            if hasattr(signal, "SIGHUP"):

                def sighup(*args):
                    eventually(self.reconfig)

                signal.signal(signal.SIGHUP, sighup)

            if hasattr(signal, "SIGUSR1"):

                def sigusr1(*args):
                    eventually(self.botmaster.cleanShutdown)

                signal.signal(signal.SIGUSR1, sigusr1)

            # get the masterid so other services can use it in
            # startup/reconfig.  This goes directly to the DB since the data
            # API isn't initialized yet, and anyway, this method is aware of
            # the DB API since it just called its setup function
            self.masterid = yield self.db.masters.findMasterId(name=self.name)

            # mark this master as stopped, in case it crashed before
            yield self.data.updates.masterStopped(name=self.name,
                                                  masterid=self.masterid)

            # call the parent method
            yield service.AsyncMultiService.startService(self)

            # We make sure the housekeeping is done before configuring in order to cleanup
            # any remaining claimed schedulers or change sources from zombie
            # masters
            yield self.data.updates.expireMasters(forceHouseKeeping=True)

            # give all services a chance to load the new configuration, rather
            # than the base configuration
            yield self.reconfigServiceWithBuildbotConfig(self.config)

            # Mark the master as active now that mq is running
            yield self.data.updates.masterActive(name=self.name,
                                                 masterid=self.masterid)

            # Start the heartbeat timer
            yield self.masterHeartbeatService.setServiceParent(self)

            # send the statistics to buildbot.net, without waiting
            self.sendBuildbotNetUsageData()
            startup_succeed = True
        except Exception:
            f = failure.Failure()
            log.err(f, 'while starting BuildMaster')
            self.reactor.stop()

        finally:
            if startup_succeed:
                log.msg("BuildMaster is running")
            else:
                log.msg("BuildMaster startup failed")

            yield self.initLock.release()
            self._master_initialized = True

    def sendBuildbotNetUsageData(self):
        if "TRIAL_PYTHONPATH" in os.environ and self.config.buildbotNetUsageData is not None:
            raise RuntimeError(
                "Should not enable buildbotNetUsageData in trial tests!")
        sendBuildbotNetUsageData(self)

    @defer.inlineCallbacks
    def stopService(self):
        try:
            yield self.initLock.acquire()
            if self.masterid is not None:
                yield self.data.updates.masterStopped(name=self.name,
                                                      masterid=self.masterid)
            if self.running:
                yield self.botmaster.cleanShutdown(quickMode=True,
                                                   stopReactor=False)
                yield service.AsyncMultiService.stopService(self)

            log.msg("BuildMaster is stopped")
            self._master_initialized = False
        finally:
            yield self.initLock.release()

    def reconfig(self):
        # this method wraps doConfig, ensuring it is only ever called once at
        # a time, and alerting the user if the reconfig takes too long
        if self.reconfig_active:
            log.msg("reconfig already active; will reconfig again after")
            self.reconfig_requested = True
            return

        self.reconfig_active = self.reactor.seconds()
        metrics.MetricCountEvent.log("loaded_config", 1)

        # notify every 10 seconds that the reconfig is still going on, although
        # reconfigs should not take that long!
        self.reconfig_notifier = task.LoopingCall(
            lambda: log.msg("reconfig is ongoing for %d s" %
                            (self.reactor.seconds() - self.reconfig_active)))
        self.reconfig_notifier.start(10, now=False)

        timer = metrics.Timer("BuildMaster.reconfig")
        timer.start()

        d = self.doReconfig()

        @d.addBoth
        def cleanup(res):
            timer.stop()
            self.reconfig_notifier.stop()
            self.reconfig_notifier = None
            self.reconfig_active = False
            if self.reconfig_requested:
                self.reconfig_requested = False
                self.reconfig()
            return res

        d.addErrback(log.err, 'while reconfiguring')

        return d  # for tests

    @defer.inlineCallbacks
    def doReconfig(self):
        log.msg("beginning configuration update")
        changes_made = False
        failed = False
        try:
            yield self.initLock.acquire()
            # Run the master.cfg in thread, so that it can use blocking code
            new_config = yield threads.deferToThreadPool(
                self.reactor, self.reactor.getThreadPool(),
                self.config_loader.loadConfig)
            changes_made = True
            self.config = new_config

            yield self.reconfigServiceWithBuildbotConfig(new_config)

        except config.ConfigErrors as e:
            for msg in e.errors:
                log.msg(msg)
            failed = True

        except Exception:
            log.err(failure.Failure(), 'during reconfig:')
            failed = True

        finally:
            yield self.initLock.release()

        if failed:
            if changes_made:
                log.msg("WARNING: reconfig partially applied; master "
                        "may malfunction")
            else:
                log.msg("reconfig aborted without making any changes")
        else:
            log.msg("configuration update complete")

    def reconfigServiceWithBuildbotConfig(self, new_config):
        if self.configured_db_url is None:
            self.configured_db_url = new_config.db['db_url']
        elif (self.configured_db_url != new_config.db['db_url']):
            config.error(
                "Cannot change c['db']['db_url'] after the master has started",
            )

        if self.config.mq['type'] != new_config.mq['type']:
            raise config.ConfigErrors([
                "Cannot change c['mq']['type'] after the master has started",
            ])

        return service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(
            self, new_config)

    # informational methods
    def allSchedulers(self):
        return list(self.scheduler_manager)

    def getStatus(self):
        """
        @rtype: L{buildbot.status.builder.Status}
        """
        return self.status

    # state maintenance (private)
    def getObjectId(self):
        """
        Return the object id for this master, for associating state with the
        master.

        @returns: ID, via Deferred
        """
        # try to get the cached value
        if self._object_id is not None:
            return defer.succeed(self._object_id)

        # failing that, get it from the DB; multiple calls to this function
        # at the same time will not hurt

        d = self.db.state.getObjectId(self.name, "buildbot.master.BuildMaster")

        @d.addCallback
        def keep(id):
            self._object_id = id
            return id

        return d

    def _getState(self, name, default=None):
        "private wrapper around C{self.db.state.getState}"
        d = self.getObjectId()

        @d.addCallback
        def get(objectid):
            return self.db.state.getState(objectid, name, default)

        return d

    def _setState(self, name, value):
        "private wrapper around C{self.db.state.setState}"
        d = self.getObjectId()

        @d.addCallback
        def set(objectid):
            return self.db.state.setState(objectid, name, value)

        return d
Esempio n. 41
0
 def testGetNoDataManagerService(self):
     secret_service_manager = SecretManager()
     SecretManager.master = self.master
     secret_result = yield secret_service_manager.get("foo2")
     self.assertEqual(secret_result, None)