def test_trigger_controlled_step_killing_worker_in_between(self):
stepcontroller = BuildStepController()
yield self.setupConfig(masterConfig(num_concurrent=1, extra_steps=[stepcontroller.step]),
startWorker=False)
d = self.doForceBuild()
builds = []
while len(builds) != 2:
builds = yield self.master.data.get(("builds",))
yield util.asyncSleep(.1)
while not stepcontroller.running:
yield util.asyncSleep(.1)
worker = self.master.workers.workers['hyper0']
worker.client.remove_container(worker.instance['Id'], v=True, force=True)
# wait that the build is retried
while len(builds) == 2:
builds = yield self.master.data.get(("builds",))
yield util.asyncSleep(.1)
stepcontroller.auto_finish_step(SUCCESS)
yield d
builds = yield self.master.data.get(("builds",))
self.assertEqual(len(builds), 5, msg=None)
# the two first builds were retried
self.assertEqual(builds[0]['results'], RETRY)
self.assertEqual(builds[1]['results'], RETRY)
self.assertEqual(builds[2]['results'], SUCCESS)
self.assertEqual(builds[3]['results'], SUCCESS)
self.assertEqual(builds[4]['results'], SUCCESS)
def create_single_worker_two_builder_step_lock_config(
self, lock_cls, mode):
lock = lock_cls("lock1", maxCount=1)
stepcontrollers = [
BuildStepController(locks=[lock.access(mode)]),
BuildStepController(locks=[lock.access(mode)])
]
config_dict = {
'builders': [
BuilderConfig(name='builder1',
workernames=['worker1'],
factory=BuildFactory([stepcontrollers[0].step])),
BuilderConfig(name='builder2',
workernames=['worker1'],
factory=BuildFactory([stepcontrollers[1].step])),
],
'workers': [
self.createLocalWorker('worker1'),
],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
yield self.setup_master(config_dict)
builder_ids = [
(yield self.master.data.updates.findBuilderId('builder1')),
(yield self.master.data.updates.findBuilderId('builder2')),
]
return stepcontrollers, builder_ids
def test_trigger_controlled_step_killing_worker_in_between(self):
stepcontroller = BuildStepController()
yield self.setupConfig(masterConfig(num_concurrent=1,
extra_steps=[stepcontroller.step]),
startWorker=False)
d = self.doForceBuild()
builds = []
while len(builds) != 2:
builds = yield self.master.data.get(("builds", ))
yield util.asyncSleep(.1)
while not stepcontroller.running:
yield util.asyncSleep(.1)
worker = self.master.workers.workers['hyper0']
worker.client.remove_container(worker.instance['Id'],
v=True,
force=True)
# wait that the build is retried
while len(builds) == 2:
builds = yield self.master.data.get(("builds", ))
yield util.asyncSleep(.1)
stepcontroller.auto_finish_step(SUCCESS)
yield d
builds = yield self.master.data.get(("builds", ))
self.assertEqual(len(builds), 5, msg=None)
# the two first builds were retried
self.assertEqual(builds[0]['results'], RETRY)
self.assertEqual(builds[1]['results'], RETRY)
self.assertEqual(builds[2]['results'], SUCCESS)
self.assertEqual(builds[3]['results'], SUCCESS)
self.assertEqual(builds[4]['results'], SUCCESS)
def create_two_worker_two_builder_lock_config(self, mode):
stepcontrollers = [BuildStepController(), BuildStepController()]
master_lock = util.MasterLock("lock1", maxCount=1)
config_dict = {
'builders': [
BuilderConfig(name='builder1',
workernames=['worker1'],
factory=BuildFactory([stepcontrollers[0].step]),
locks=[master_lock.access(mode)]),
BuilderConfig(name='builder2',
workernames=['worker2'],
factory=BuildFactory([stepcontrollers[1].step]),
locks=[master_lock.access(mode)]),
],
'workers': [
self.createLocalWorker('worker1'),
self.createLocalWorker('worker2'),
],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
master = yield self.getMaster(config_dict)
builder_ids = [
(yield master.data.updates.findBuilderId('builder1')),
(yield master.data.updates.findBuilderId('builder2')),
]
return stepcontrollers, master, builder_ids
def test_worker_close_connection_while_building(self):
"""
If the worker close connection in the middle of the build, the next build can start correctly
"""
controller = LatentController('local', build_wait_timeout=0)
# a step that we can finish when we want
stepcontroller = BuildStepController()
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory([stepcontroller.step]),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Request two builds.
for i in range(2):
self.createBuildrequest(master, [builder_id])
controller.auto_stop(True)
self.assertTrue(controller.starting)
controller.start_instance(True)
controller.connect_worker(self)
builds = self.successResultOf(master.data.get(("builds", )))
self.assertEqual(builds[0]['results'], None)
controller.disconnect_worker(self)
builds = self.successResultOf(master.data.get(("builds", )))
self.assertEqual(builds[0]['results'], RETRY)
# Request one build.
self.createBuildrequest(master, [builder_id])
controller.start_instance(True)
controller.connect_worker(self)
builds = self.successResultOf(master.data.get(("builds", )))
self.assertEqual(builds[1]['results'], None)
stepcontroller.finish_step(SUCCESS)
builds = self.successResultOf(master.data.get(("builds", )))
self.assertEqual(builds[1]['results'], SUCCESS)
def test_worker_close_connection_while_building(self):
"""
If the worker close connection in the middle of the build, the next build can start correctly
"""
controller = LatentController('local', build_wait_timeout=0)
# a step that we can finish when we want
stepcontroller = BuildStepController()
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory([stepcontroller.step]),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Request two builds.
for i in range(2):
self.createBuildrequest(master, [builder_id])
controller.auto_stop(True)
self.assertTrue(controller.starting)
controller.start_instance(True)
controller.connect_worker(self)
builds = self.successResultOf(
master.data.get(("builds",)))
self.assertEqual(builds[0]['results'], None)
controller.disconnect_worker(self)
builds = self.successResultOf(
master.data.get(("builds",)))
self.assertEqual(builds[0]['results'], RETRY)
# Request one build.
self.createBuildrequest(master, [builder_id])
controller.start_instance(True)
controller.connect_worker(self)
builds = self.successResultOf(
master.data.get(("builds",)))
self.assertEqual(builds[1]['results'], None)
stepcontroller.finish_step(SUCCESS)
builds = self.successResultOf(
master.data.get(("builds",)))
self.assertEqual(builds[1]['results'], SUCCESS)
def create_single_worker_config_with_step(self, controller_kwargs=None):
if not controller_kwargs:
controller_kwargs = {}
controller = LatentController(self, 'local', **controller_kwargs)
stepcontroller = BuildStepController()
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory([stepcontroller.step]),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
master = yield self.getMaster(config_dict)
builder_id = yield master.data.updates.findBuilderId('testy')
return controller, stepcontroller, master, builder_id
def test_step_with_worker_build_props_during_worker_disconnect(self):
"""
We need to handle worker disconnection and steps with worker build properties gracefully
"""
controller = WorkerController(self, 'local')
stepcontroller = BuildStepController()
config_dict = {
'builders': [
BuilderConfig(
name="builder",
workernames=['local'],
properties={'worker': Interpolate("%(worker:numcpus)s")},
factory=BuildFactory([stepcontroller.step])),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
yield self.setup_master(config_dict)
builder_id = yield self.master.data.updates.findBuilderId('builder')
yield self.create_build_request([builder_id])
yield controller.connect_worker()
self.reactor.advance(1)
yield controller.disconnect_worker()
self.reactor.advance(1)
yield self.assertBuildResults(1, RETRY)
def test_trigger_controlled_step(self):
stepcontroller = BuildStepController()
yield self.setupConfig(masterConfig(num_concurrent=1, extra_steps=[stepcontroller.step]),
startWorker=False)
d = self.doForceBuild()
builds = []
while len(builds) != 2:
builds = yield self.master.data.get(("builds",))
util.asyncSleep(.1)
while not stepcontroller.running:
yield util.asyncSleep(.1)
stepcontroller.finish_step(SUCCESS)
yield d
builds = yield self.master.data.get(("builds",))
for b in builds:
self.assertEqual(b['results'], SUCCESS)
def test_trigger_controlled_step(self):
stepcontroller = BuildStepController()
yield self.setupConfig(masterConfig(num_concurrent=1,
extra_steps=[stepcontroller.step]),
startWorker=False)
d = self.doForceBuild()
builds = []
while len(builds) != 2:
builds = yield self.master.data.get(("builds", ))
util.asyncSleep(.1)
while not stepcontroller.running:
yield util.asyncSleep(.1)
stepcontroller.finish_step(SUCCESS)
yield d
builds = yield self.master.data.get(("builds", ))
for b in builds:
self.assertEqual(b['results'], SUCCESS)
def test_rejects_build_on_instance_with_different_type_timeout_nonzero(
self):
"""
If latent worker supports getting its instance type from properties that
are rendered from build then the buildrequestdistributor must not
schedule any builds on workers that are running different instance type
than what these builds will require.
"""
controller = LatentController(self,
'local',
kind=Interpolate('%(prop:worker_kind)s'),
build_wait_timeout=5)
# a step that we can finish when we want
stepcontroller = BuildStepController()
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory([stepcontroller.step]),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# create build request
self.createBuildrequest(master, [builder_id],
properties=Properties(worker_kind='a'))
# start the build and verify the kind of the worker. Note that the
# buildmaster needs to restart the worker in order to change the worker
# kind, so we allow it both to auto start and stop
self.assertEqual(True, controller.starting)
controller.auto_connect_worker = True
controller.auto_disconnect_worker = True
controller.auto_start(True)
controller.auto_stop(True)
controller.connect_worker()
self.assertEqual('a', (yield controller.get_started_kind()))
# before the other build finished, create another build request
self.createBuildrequest(master, [builder_id],
properties=Properties(worker_kind='b'))
stepcontroller.finish_step(SUCCESS)
# give the botmaster chance to insubstantiate the worker and
# maybe substantiate it for the pending build the builds on worker
self.reactor.advance(0.1)
# verify build has not started, even though the worker is waiting
# for one
self.assertIsNone((yield master.db.builds.getBuild(2)))
self.assertTrue(controller.started)
# wait until the latent worker times out, is insubstantiated,
# is substantiated because of pending buildrequest and starts the build
self.reactor.advance(6)
self.assertIsNotNone((yield master.db.builds.getBuild(2)))
# verify that the second build restarted with the expected instance
# kind
self.assertEqual('b', (yield controller.get_started_kind()))
stepcontroller.finish_step(SUCCESS)
dbdict = yield master.db.builds.getBuild(1)
self.assertEqual(SUCCESS, dbdict['results'])
dbdict = yield master.db.builds.getBuild(2)
self.assertEqual(SUCCESS, dbdict['results'])
def create_stepcontrollers(self, count, lock, mode):
stepcontrollers = []
for _ in range(count):
locks = [lock.access(mode)] if lock is not None else []
stepcontrollers.append(BuildStepController(locks=locks))
return stepcontrollers
def test_builder_lock_release_wakes_builds_for_another_builder(self):
"""
If a builder locks a master lock then the build request distributor
must retry running any buildrequests that might have been not scheduled
due to unavailability of that lock when the lock becomes available.
"""
stepcontroller1 = BuildStepController()
stepcontroller2 = BuildStepController()
master_lock = util.MasterLock("lock1", maxCount=1)
config_dict = {
'builders': [
BuilderConfig(name='builder1',
workernames=['worker1'],
factory=BuildFactory([stepcontroller1.step]),
locks=[master_lock.access('counting')]),
BuilderConfig(name='builder2',
workernames=['worker2'],
factory=BuildFactory([stepcontroller2.step]),
locks=[master_lock.access('counting')]),
],
'workers': [
self.createLocalWorker('worker1'),
self.createLocalWorker('worker2'),
],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
master = yield self.getMaster(config_dict)
builder1_id = yield master.data.updates.findBuilderId('builder1')
builder2_id = yield master.data.updates.findBuilderId('builder2')
# start two builds and verify that a second build starts after the
# first is finished
yield self.createBuildrequest(master, [builder1_id])
yield self.createBuildrequest(master, [builder2_id])
builds = yield master.data.get(("builds", ))
self.assertEqual(len(builds), 1)
self.assertEqual(builds[0]['results'], None)
self.assertEqual(builds[0]['builderid'], builder1_id)
stepcontroller1.finish_step(SUCCESS)
# execute Build.releaseLocks which is called eventually
yield flushEventualQueue()
builds = yield master.data.get(("builds", ))
self.assertEqual(len(builds), 2)
self.assertEqual(builds[0]['results'], SUCCESS)
self.assertEqual(builds[1]['results'], None)
self.assertEqual(builds[1]['builderid'], builder2_id)
stepcontroller2.finish_step(SUCCESS)
builds = yield master.data.get(("builds", ))
self.assertEqual(len(builds), 2)
self.assertEqual(builds[0]['results'], SUCCESS)
self.assertEqual(builds[1]['results'], SUCCESS)
def test_rejects_build_on_instance_with_different_type_timeout_nonzero(self):
"""
If latent worker supports getting its instance type from properties that
are rendered from build then the buildrequestdistributor must not
schedule any builds on workers that are running different instance type
than what these builds will require.
"""
controller = LatentController(self, 'local',
kind=Interpolate('%(prop:worker_kind)s'),
build_wait_timeout=5)
# a step that we can finish when we want
stepcontroller = BuildStepController()
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory([stepcontroller.step]),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# create build request
self.createBuildrequest(master, [builder_id],
properties=Properties(worker_kind='a'))
# start the build and verify the kind of the worker. Note that the
# buildmaster needs to restart the worker in order to change the worker
# kind, so we allow it both to auto start and stop
self.assertEqual(True, controller.starting)
controller.auto_connect_worker = True
controller.auto_disconnect_worker = True
controller.auto_start(True)
controller.auto_stop(True)
controller.connect_worker()
self.assertEqual('a', (yield controller.get_started_kind()))
# before the other build finished, create another build request
self.createBuildrequest(master, [builder_id],
properties=Properties(worker_kind='b'))
stepcontroller.finish_step(SUCCESS)
# give the botmaster chance to insubstantiate the worker and
# maybe substantiate it for the pending build the builds on worker
self.reactor.advance(0.1)
# verify build has not started, even though the worker is waiting
# for one
self.assertIsNone((yield master.db.builds.getBuild(2)))
self.assertTrue(controller.started)
# wait until the latent worker times out, is insubstantiated,
# is substantiated because of pending buildrequest and starts the build
self.reactor.advance(6)
self.assertIsNotNone((yield master.db.builds.getBuild(2)))
# verify that the second build restarted with the expected instance
# kind
self.assertEqual('b', (yield controller.get_started_kind()))
stepcontroller.finish_step(SUCCESS)
dbdict = yield master.db.builds.getBuild(1)
self.assertEqual(SUCCESS, dbdict['results'])
dbdict = yield master.db.builds.getBuild(2)
self.assertEqual(SUCCESS, dbdict['results'])
