def test_failed_substantiations_get_exception(self): """ If a latent worker fails to substantiate, the result is an exception. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest self.createBuildrequest(master, [builder_id]) # The worker fails to substantiate. controller.start_instance( Failure(LatentWorkerCannotSubstantiate("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(LatentWorkerCannotSubstantiate) dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1) # When the substantiation fails, the result is an exception. self.assertEqual(EXCEPTION, dbdict['results']) controller.auto_stop(True)
def test_build_stop_with_cancelled_during_substantiation(self): """ If a build is stopping during latent worker substantiating, the build becomes cancelled """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder = master.botmaster.builders['testy'] builder_id = self.successResultOf(builder.getBuilderId()) # Trigger a buildrequest self.createBuildrequest(master, [builder_id]) # Stop the build build = builder.getBuild(0) build.stopBuild('no reason', results=CANCELLED) # Indicate that the worker can't start an instance. controller.start_instance(False) dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1) self.assertEqual(CANCELLED, dbdict['results']) controller.auto_stop(True) self.flushLoggedErrors(LatentWorkerFailedToSubstantiate)
def test_latent_max_builds(self): """ If max_builds is set, only one build is started on a latent worker at a time. """ controller = LatentController(self, 'local', max_builds=1) step_controller = StepController() config_dict = { 'builders': [ BuilderConfig( name="testy-1", workernames=["local"], factory=BuildFactory([step_controller]), ), BuilderConfig( name="testy-2", workernames=["local"], factory=BuildFactory([step_controller]), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, 'multiMaster': True, } self.master = master = yield getMaster(self, self.reactor, config_dict) builder_ids = [ (yield master.data.updates.findBuilderId('testy-1')), (yield master.data.updates.findBuilderId('testy-2')), ] started_builds = [] yield master.mq.startConsuming( lambda key, build: started_builds.append(build), ('builds', None, 'new')) # Trigger a buildrequest bsid, brids = yield master.data.updates.addBuildset( waited_for=False, builderids=builder_ids, sourcestamps=[ { 'codebase': '', 'repository': '', 'branch': None, 'revision': None, 'project': '' }, ], ) # The worker fails to substantiate. controller.start_instance(True) controller.connect_worker() self.assertEqual(len(started_builds), 1) yield controller.auto_stop(True)
def test_build_stop_with_cancelled_during_substantiation(self): """ If a build is stopping during latent worker substantiating, the build becomes cancelled """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder = master.botmaster.builders['testy'] builder_id = self.successResultOf(builder.getBuilderId()) # Trigger a buildrequest self.createBuildrequest(master, [builder_id]) # Stop the build build = builder.getBuild(0) build.stopBuild('no reason', results=CANCELLED) # Indicate that the worker can't start an instance. controller.start_instance(False) dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1) self.assertEqual(CANCELLED, dbdict['results']) controller.auto_stop(True) self.flushLoggedErrors(LatentWorkerFailedToSubstantiate)
def test_worker_multiple_substantiations_succeed(self): """ If multiple builders trigger try to substantiate a worker at the same time, if the substantiation succeeds then all of the builds proceeed. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy-1", workernames=["local"], factory=BuildFactory(), ), BuilderConfig(name="testy-2", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, 'multiMaster': True, } master = self.successResultOf( getMaster(self, self.reactor, config_dict)) builder_ids = [ self.successResultOf(master.data.updates.findBuilderId('testy-1')), self.successResultOf(master.data.updates.findBuilderId('testy-2')), ] finished_builds = [] self.successResultOf(master.mq.startConsuming( lambda key, build: finished_builds.append(build), ('builds', None, 'finished'))) # Trigger a buildrequest bsid, brids = self.successResultOf( master.data.updates.addBuildset( waited_for=False, builderids=builder_ids, sourcestamps=[ {'codebase': '', 'repository': '', 'branch': None, 'revision': None, 'project': ''}, ], ) ) # The worker fails to substantiate. controller.start_instance(True) local_workdir = FilePath(self.mktemp()) local_workdir.createDirectory() controller.connect_worker(local_workdir) # We check that there were two builds that finished, and # that they both finished with success self.assertEqual([build['results'] for build in finished_builds], [SUCCESS] * 2)
def test_failed_substantiations_get_exception(self): """ If a latent worker fails to substantiate, the result is an exception. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest self.createBuildrequest(master, [builder_id]) # The worker fails to substantiate. controller.start_instance( Failure(LatentWorkerCannotSubstantiate("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(LatentWorkerCannotSubstantiate) dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1) # When the substantiation fails, the result is an exception. self.assertEqual(EXCEPTION, dbdict['results']) controller.auto_stop(True)
def test_worker_accepts_builds_after_failure(self): """ If a latent worker fails to substantiate, the worker is still able to accept jobs. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig( name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.successResultOf( getMaster(self, self.reactor, config_dict)) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest bsid, brids = self.successResultOf( master.data.updates.addBuildset( waited_for=False, builderids=[builder_id], sourcestamps=[ { 'codebase': '', 'repository': '', 'branch': None, 'revision': None, 'project': '' }, ], )) unclaimed_build_requests = [] self.successResultOf( master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # The worker fails to substantiate. controller.start_instance( Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # If the worker started again after the failure, then the retry logic will have # already kicked in to start a new build on this (the only) worker. We check that # a new instance was requested, which indicates that the worker # accepted the build. self.assertEqual(controller.started, True)
def test_failed_substantiations_get_requeued(self): """ If a latent worker fails to substantiate, the build request becomes unclaimed. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig( name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.successResultOf( getMaster(self, self.reactor, config_dict)) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest bsid, brids = self.successResultOf( master.data.updates.addBuildset( waited_for=False, builderids=[builder_id], sourcestamps=[ { 'codebase': '', 'repository': '', 'branch': None, 'revision': None, 'project': '' }, ], )) unclaimed_build_requests = [] self.successResultOf( master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # The worker fails to substantiate. controller.start_instance( Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # When the substantiation fails, the buildrequest becomes unclaimed. self.assertEqual( set(brids), set([req['buildrequestid'] for req in unclaimed_build_requests]), )
def test_worker_accepts_builds_after_failure(self): """ If a latent worker fails to substantiate, the worker is still able to accept jobs. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) controller.auto_stop(True) # Trigger a buildrequest bsid, brids = self.createBuildrequest(master, [builder_id]) unclaimed_build_requests = [] self.successResultOf(master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # The worker fails to substantiate. controller.start_instance( Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # The retry logic should only trigger after a exponential backoff self.assertEqual(controller.starting, False) # advance the time to the point where we should retry master.reactor.advance(controller.worker.quarantine_initial_timeout) # If the worker started again after the failure, then the retry logic will have # already kicked in to start a new build on this (the only) worker. We check that # a new instance was requested, which indicates that the worker # accepted the build. self.assertEqual(controller.starting, True) # The worker fails to substantiate(again). controller.start_instance( Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # advance the time to the point where we should not retry master.reactor.advance(controller.worker.quarantine_initial_timeout) self.assertEqual(controller.starting, False) # advance the time to the point where we should retry master.reactor.advance(controller.worker.quarantine_initial_timeout) self.assertEqual(controller.starting, True)
def test_latent_max_builds(self): """ If max_builds is set, only one build is started on a latent worker at a time. """ controller = LatentController( 'local', max_builds=1, ) step_controller = StepController() config_dict = { 'builders': [ BuilderConfig(name="testy-1", workernames=["local"], factory=BuildFactory([step_controller]), ), BuilderConfig(name="testy-2", workernames=["local"], factory=BuildFactory([step_controller]), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, 'multiMaster': True, } self.master = master = self.successResultOf( getMaster(self, self.reactor, config_dict)) builder_ids = [ self.successResultOf(master.data.updates.findBuilderId('testy-1')), self.successResultOf(master.data.updates.findBuilderId('testy-2')), ] started_builds = [] self.successResultOf(master.mq.startConsuming( lambda key, build: started_builds.append(build), ('builds', None, 'new'))) # Trigger a buildrequest bsid, brids = self.successResultOf( master.data.updates.addBuildset( waited_for=False, builderids=builder_ids, sourcestamps=[ {'codebase': '', 'repository': '', 'branch': None, 'revision': None, 'project': ''}, ], ) ) # The worker fails to substantiate. controller.start_instance(True) controller.connect_worker(self) self.assertEqual(len(started_builds), 1) controller.auto_stop(True)
def test_worker_accepts_builds_after_failure(self): """ If a latent worker fails to substantiate, the worker is still able to accept jobs. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) controller.auto_stop(True) # Trigger a buildrequest bsid, brids = self.createBuildrequest(master, [builder_id]) unclaimed_build_requests = [] self.successResultOf(master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # The worker fails to substantiate. controller.start_instance( Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # The retry logic should only trigger after a exponential backoff self.assertEqual(controller.starting, False) # advance the time to the point where we should retry master.reactor.advance(controller.worker.quarantine_initial_timeout) # If the worker started again after the failure, then the retry logic will have # already kicked in to start a new build on this (the only) worker. We check that # a new instance was requested, which indicates that the worker # accepted the build. self.assertEqual(controller.starting, True) # The worker fails to substantiate(again). controller.start_instance( Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # advance the time to the point where we should not retry master.reactor.advance(controller.worker.quarantine_initial_timeout) self.assertEqual(controller.starting, False) # advance the time to the point where we should retry master.reactor.advance(controller.worker.quarantine_initial_timeout) self.assertEqual(controller.starting, True)
def test_failed_substantiations_get_requeued(self): """ If a latent worker fails to substantiate, the build request becomes unclaimed. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.successResultOf( getMaster(self, self.reactor, config_dict)) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest bsid, brids = self.successResultOf( master.data.updates.addBuildset( waited_for=False, builderids=[builder_id], sourcestamps=[ {'codebase': '', 'repository': '', 'branch': None, 'revision': None, 'project': ''}, ], ) ) unclaimed_build_requests = [] self.successResultOf(master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # The worker fails to substantiate. controller.start_instance( Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # When the substantiation fails, the buildrequest becomes unclaimed. self.assertEqual( set(brids), set([req['buildrequestid'] for req in unclaimed_build_requests]), )
def test_worker_accepts_builds_after_failure(self): """ If a latent worker fails to substantiate, the worker is still able to accept jobs. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.successResultOf( getMaster(self, self.reactor, config_dict)) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest bsid, brids = self.successResultOf( master.data.updates.addBuildset( waited_for=False, builderids=[builder_id], sourcestamps=[ {'codebase': '', 'repository': '', 'branch': None, 'revision': None, 'project': ''}, ], ) ) unclaimed_build_requests = [] self.successResultOf(master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # The worker fails to substantiate. controller.start_instance( Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # If the worker started again after the failure, then the retry logic will have # already kicked in to start a new build on this (the only) worker. We check that # a new instance was requested, which indicates that the worker # accepted the build. self.assertEqual(controller.started, True)
def test_worker_multiple_substantiations_succeed(self): """ If multiple builders trigger try to substantiate a worker at the same time, if the substantiation succeeds then all of the builds proceed. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig( name="testy-1", workernames=["local"], factory=BuildFactory(), ), BuilderConfig( name="testy-2", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, 'multiMaster': True, } master = self.getMaster(config_dict) builder_ids = [ self.successResultOf(master.data.updates.findBuilderId('testy-1')), self.successResultOf(master.data.updates.findBuilderId('testy-2')), ] finished_builds = [] self.successResultOf( master.mq.startConsuming( lambda key, build: finished_builds.append(build), ('builds', None, 'finished'))) # Trigger a buildrequest bsid, brids = self.createBuildrequest(master, builder_ids) # The worker succeeds to substantiate. controller.start_instance(True) controller.connect_worker(self) # We check that there were two builds that finished, and # that they both finished with success self.assertEqual([build['results'] for build in finished_builds], [SUCCESS] * 2) controller.auto_stop(True)
def test_worker_close_connection_while_building(self): """ If the worker close connection in the middle of the build, the next build can start correctly """ controller = LatentController('local', build_wait_timeout=0) # a step that we can finish when we want stepcontroller = BuildStepController() config_dict = { 'builders': [ BuilderConfig( name="testy", workernames=["local"], factory=BuildFactory([stepcontroller.step]), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Request two builds. for i in range(2): self.createBuildrequest(master, [builder_id]) controller.auto_stop(True) self.assertTrue(controller.starting) controller.start_instance(True) controller.connect_worker(self) builds = self.successResultOf(master.data.get(("builds", ))) self.assertEqual(builds[0]['results'], None) controller.disconnect_worker(self) builds = self.successResultOf(master.data.get(("builds", ))) self.assertEqual(builds[0]['results'], RETRY) # Request one build. self.createBuildrequest(master, [builder_id]) controller.start_instance(True) controller.connect_worker(self) builds = self.successResultOf(master.data.get(("builds", ))) self.assertEqual(builds[1]['results'], None) stepcontroller.finish_step(SUCCESS) builds = self.successResultOf(master.data.get(("builds", ))) self.assertEqual(builds[1]['results'], SUCCESS)
def test_build_stop_with_retry_during_substantiation(self): """ If master is shutting down during latent worker substantiating, the build becomes retry. """ controller = LatentController(self, 'local') config_dict = { 'builders': [ BuilderConfig( name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder = master.botmaster.builders['testy'] builder_id = self.successResultOf(builder.getBuilderId()) # Trigger a buildrequest _, brids = self.createBuildrequest(master, [builder_id]) unclaimed_build_requests = [] self.successResultOf( master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # Stop the build build = builder.getBuild(0) build.stopBuild('no reason', results=RETRY) # Indicate that the worker can't start an instance. controller.start_instance(False) dbdict = self.successResultOf( master.db.builds.getBuildByNumber(builder_id, 1)) self.assertEqual(RETRY, dbdict['results']) self.assertEqual( set(brids), {req['buildrequestid'] for req in unclaimed_build_requests}) controller.auto_stop(True) self.flushLoggedErrors(LatentWorkerFailedToSubstantiate)
def test_worker_close_connection_while_building(self): """ If the worker close connection in the middle of the build, the next build can start correctly """ controller = LatentController('local', build_wait_timeout=0) # a step that we can finish when we want stepcontroller = BuildStepController() config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory([stepcontroller.step]), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Request two builds. for i in range(2): self.createBuildrequest(master, [builder_id]) controller.auto_stop(True) self.assertTrue(controller.starting) controller.start_instance(True) controller.connect_worker(self) builds = self.successResultOf( master.data.get(("builds",))) self.assertEqual(builds[0]['results'], None) controller.disconnect_worker(self) builds = self.successResultOf( master.data.get(("builds",))) self.assertEqual(builds[0]['results'], RETRY) # Request one build. self.createBuildrequest(master, [builder_id]) controller.start_instance(True) controller.connect_worker(self) builds = self.successResultOf( master.data.get(("builds",))) self.assertEqual(builds[1]['results'], None) stepcontroller.finish_step(SUCCESS) builds = self.successResultOf( master.data.get(("builds",))) self.assertEqual(builds[1]['results'], SUCCESS)
def test_failed_substantiations_get_requeued(self): """ If a latent worker fails to substantiate, the build request becomes unclaimed. """ controller = LatentController(self, 'local') config_dict = { 'builders': [ BuilderConfig( name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest bsid, brids = self.createBuildrequest(master, [builder_id]) unclaimed_build_requests = [] self.successResultOf( master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # The worker fails to substantiate. controller.start_instance( Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # When the substantiation fails, the buildrequest becomes unclaimed. self.assertEqual( set(brids), {req['buildrequestid'] for req in unclaimed_build_requests}) controller.auto_stop(True)
def test_worker_multiple_substantiations_succeed(self): """ If multiple builders trigger try to substantiate a worker at the same time, if the substantiation succeeds then all of the builds proceeed. """ controller = LatentController("local") config_dict = { "builders": [ BuilderConfig(name="testy-1", workernames=["local"], factory=BuildFactory()), BuilderConfig(name="testy-2", workernames=["local"], factory=BuildFactory()), ], "workers": [controller.worker], "protocols": {"null": {}}, "multiMaster": True, } master = self.getMaster(config_dict) builder_ids = [ self.successResultOf(master.data.updates.findBuilderId("testy-1")), self.successResultOf(master.data.updates.findBuilderId("testy-2")), ] finished_builds = [] self.successResultOf( master.mq.startConsuming(lambda key, build: finished_builds.append(build), ("builds", None, "finished")) ) # Trigger a buildrequest bsid, brids = self.successResultOf( master.data.updates.addBuildset( waited_for=False, builderids=builder_ids, sourcestamps=[{"codebase": "", "repository": "", "branch": None, "revision": None, "project": ""}], ) ) # The worker fails to substantiate. controller.start_instance(True) local_workdir = FilePath(self.mktemp()) local_workdir.createDirectory() controller.connect_worker(local_workdir) # We check that there were two builds that finished, and # that they both finished with success self.assertEqual([build["results"] for build in finished_builds], [SUCCESS] * 2)
def test_worker_multiple_substantiations_succeed(self): """ If multiple builders trigger try to substantiate a worker at the same time, if the substantiation succeeds then all of the builds proceeed. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy-1", workernames=["local"], factory=BuildFactory(), ), BuilderConfig(name="testy-2", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, 'multiMaster': True, } master = self.getMaster(config_dict) builder_ids = [ self.successResultOf(master.data.updates.findBuilderId('testy-1')), self.successResultOf(master.data.updates.findBuilderId('testy-2')), ] finished_builds = [] self.successResultOf(master.mq.startConsuming( lambda key, build: finished_builds.append(build), ('builds', None, 'finished'))) # Trigger a buildrequest bsid, brids = self.createBuildrequest(master, builder_ids) # The worker succeeds to substantiate. controller.start_instance(True) controller.connect_worker(self) # We check that there were two builds that finished, and # that they both finished with success self.assertEqual([build['results'] for build in finished_builds], [SUCCESS] * 2) controller.auto_stop(True)
def test_build_stop_with_retry_during_substantiation(self): """ If master is shutting down during latent worker substantiating, the build becomes retry. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder = master.botmaster.builders['testy'] builder_id = self.successResultOf(builder.getBuilderId()) # Trigger a buildrequest _, brids = self.createBuildrequest(master, [builder_id]) unclaimed_build_requests = [] self.successResultOf(master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # Stop the build build = builder.getBuild(0) build.stopBuild('no reason', results=RETRY) # Indicate that the worker can't start an instance. controller.start_instance(False) dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1) self.assertEqual(RETRY, dbdict['results']) self.assertEqual( set(brids), set([req['buildrequestid'] for req in unclaimed_build_requests]), ) controller.auto_stop(True) self.flushLoggedErrors(LatentWorkerFailedToSubstantiate)
def test_failed_substantiations_get_requeued(self): """ If a latent worker fails to substantiate, the build request becomes unclaimed. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest bsid, brids = self.createBuildrequest(master, [builder_id]) unclaimed_build_requests = [] self.successResultOf(master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # The worker fails to substantiate. controller.start_instance( Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # When the substantiation fails, the buildrequest becomes unclaimed. self.assertEqual( set(brids), {req['buildrequestid'] for req in unclaimed_build_requests} ) controller.auto_stop(True)
def test_failed_substantiations_get_requeued(self): """ If a latent worker fails to substantiate, the build request becomes unclaimed. """ controller = LatentController("local") config_dict = { "builders": [BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory())], "workers": [controller.worker], "protocols": {"null": {}}, # Disable checks about missing scheduler. "multiMaster": True, } master = self.getMaster(config_dict) builder_id = self.successResultOf(master.data.updates.findBuilderId("testy")) # Trigger a buildrequest bsid, brids = self.successResultOf( master.data.updates.addBuildset( waited_for=False, builderids=[builder_id], sourcestamps=[{"codebase": "", "repository": "", "branch": None, "revision": None, "project": ""}], ) ) unclaimed_build_requests = [] self.successResultOf( master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ("buildrequests", None, "unclaimed") ) ) # The worker fails to substantiate. controller.start_instance(Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # When the substantiation fails, the buildrequest becomes unclaimed. self.assertEqual(set(brids), set([req["buildrequestid"] for req in unclaimed_build_requests]))
def test_failed_ping_get_requeued(self): """ sendBuilderList can fail due to missing permissions on the workdir, the build request becomes unclaimed """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig( name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest bsid, brids = self.createBuildrequest(master, [builder_id]) unclaimed_build_requests = [] self.successResultOf( master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) logs = [] self.successResultOf( master.mq.startConsuming(lambda key, log: logs.append(log), ('logs', None, 'new'))) # The worker succeed to substantiate def remote_print(self, msg): if msg == "ping": raise TestException("can't ping") controller.patchBot(self, 'remote_print', remote_print) controller.start_instance(True) controller.connect_worker(self) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # When the substantiation fails, the buildrequest becomes unclaimed. self.assertEqual( set(brids), set([req['buildrequestid'] for req in unclaimed_build_requests]), ) # should get 2 logs (html and txt) with proper information in there self.assertEqual(len(logs), 2) logs_by_name = {} for _log in logs: fulllog = self.successResultOf( master.data.get(("logs", str(_log['logid']), "raw"))) logs_by_name[fulllog['filename']] = fulllog['raw'] for i in ["err_text", "err_html"]: self.assertIn("can't ping", logs_by_name[i]) # make sure stacktrace is present in html self.assertIn( "buildbot.test.integration.test_latent.TestException", logs_by_name[i]) controller.auto_stop(True)
def test_worker_multiple_substantiations_succeed(self): """ If multiple builders trigger try to substantiate a worker at the same time, if the substantiation succeeds then all of the builds proceeed. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig( name="testy-1", workernames=["local"], factory=BuildFactory(), ), BuilderConfig( name="testy-2", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, 'multiMaster': True, } master = self.getMaster(config_dict) builder_ids = [ self.successResultOf(master.data.updates.findBuilderId('testy-1')), self.successResultOf(master.data.updates.findBuilderId('testy-2')), ] finished_builds = [] self.successResultOf( master.mq.startConsuming( lambda key, build: finished_builds.append(build), ('builds', None, 'finished'))) # Trigger a buildrequest bsid, brids = self.successResultOf( master.data.updates.addBuildset( waited_for=False, builderids=builder_ids, sourcestamps=[ { 'codebase': '', 'repository': '', 'branch': None, 'revision': None, 'project': '' }, ], )) # The worker fails to substantiate. controller.start_instance(True) local_workdir = FilePath(self.mktemp()) local_workdir.createDirectory() controller.connect_worker(local_workdir) # We check that there were two builds that finished, and # that they both finished with success self.assertEqual([build['results'] for build in finished_builds], [SUCCESS] * 2)
def test_failed_ping_get_requeued(self): """ sendBuilderList can fail due to missing permissions on the workdir, the build request becomes unclaimed """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest bsid, brids = self.createBuildrequest(master, [builder_id]) unclaimed_build_requests = [] self.successResultOf(master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) logs = [] self.successResultOf(master.mq.startConsuming( lambda key, log: logs.append(log), ('logs', None, 'new'))) # The worker succeed to substantiate def remote_print(self, msg): if msg == "ping": raise TestException("can't ping") controller.patchBot(self, 'remote_print', remote_print) controller.start_instance(True) controller.connect_worker(self) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # When the substantiation fails, the buildrequest becomes unclaimed. self.assertEqual( set(brids), set([req['buildrequestid'] for req in unclaimed_build_requests]), ) # should get 2 logs (html and txt) with proper information in there self.assertEqual(len(logs), 2) logs_by_name = {} for _log in logs: fulllog = self.successResultOf( master.data.get(("logs", str(_log['logid']), "raw"))) logs_by_name[fulllog['filename']] = fulllog['raw'] for i in ["err_text", "err_html"]: self.assertIn("can't ping", logs_by_name[i]) # make sure stacktrace is present in html self.assertIn(os.path.join( "integration", "test_latent.py"), logs_by_name[i]) controller.auto_stop(True)