def test_failed_substantiations_get_exception(self):
"""
If a latent worker fails to substantiate, the result is an exception.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
self.createBuildrequest(master, [builder_id])
# The worker fails to substantiate.
controller.start_instance(
Failure(LatentWorkerCannotSubstantiate("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(LatentWorkerCannotSubstantiate)
dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1)
# When the substantiation fails, the result is an exception.
self.assertEqual(EXCEPTION, dbdict['results'])
controller.auto_stop(True)
def test_build_stop_with_cancelled_during_substantiation(self):
"""
If a build is stopping during latent worker substantiating, the build becomes cancelled
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder = master.botmaster.builders['testy']
builder_id = self.successResultOf(builder.getBuilderId())
# Trigger a buildrequest
self.createBuildrequest(master, [builder_id])
# Stop the build
build = builder.getBuild(0)
build.stopBuild('no reason', results=CANCELLED)
# Indicate that the worker can't start an instance.
controller.start_instance(False)
dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1)
self.assertEqual(CANCELLED, dbdict['results'])
controller.auto_stop(True)
self.flushLoggedErrors(LatentWorkerFailedToSubstantiate)
def test_latent_max_builds(self):
"""
If max_builds is set, only one build is started on a latent
worker at a time.
"""
controller = LatentController(self, 'local', max_builds=1)
step_controller = StepController()
config_dict = {
'builders': [
BuilderConfig(
name="testy-1",
workernames=["local"],
factory=BuildFactory([step_controller]),
),
BuilderConfig(
name="testy-2",
workernames=["local"],
factory=BuildFactory([step_controller]),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
self.master = master = yield getMaster(self, self.reactor, config_dict)
builder_ids = [
(yield master.data.updates.findBuilderId('testy-1')),
(yield master.data.updates.findBuilderId('testy-2')),
]
started_builds = []
yield master.mq.startConsuming(
lambda key, build: started_builds.append(build),
('builds', None, 'new'))
# Trigger a buildrequest
bsid, brids = yield master.data.updates.addBuildset(
waited_for=False,
builderids=builder_ids,
sourcestamps=[
{
'codebase': '',
'repository': '',
'branch': None,
'revision': None,
'project': ''
},
],
)
# The worker fails to substantiate.
controller.start_instance(True)
controller.connect_worker()
self.assertEqual(len(started_builds), 1)
yield controller.auto_stop(True)
def test_build_stop_with_cancelled_during_substantiation(self):
"""
If a build is stopping during latent worker substantiating, the build becomes cancelled
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder = master.botmaster.builders['testy']
builder_id = self.successResultOf(builder.getBuilderId())
# Trigger a buildrequest
self.createBuildrequest(master, [builder_id])
# Stop the build
build = builder.getBuild(0)
build.stopBuild('no reason', results=CANCELLED)
# Indicate that the worker can't start an instance.
controller.start_instance(False)
dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1)
self.assertEqual(CANCELLED, dbdict['results'])
controller.auto_stop(True)
self.flushLoggedErrors(LatentWorkerFailedToSubstantiate)
def test_worker_multiple_substantiations_succeed(self):
"""
If multiple builders trigger try to substantiate a worker at
the same time, if the substantiation succeeds then all of
the builds proceeed.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy-1",
workernames=["local"],
factory=BuildFactory(),
),
BuilderConfig(name="testy-2",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
'multiMaster': True,
}
master = self.successResultOf(
getMaster(self, self.reactor, config_dict))
builder_ids = [
self.successResultOf(master.data.updates.findBuilderId('testy-1')),
self.successResultOf(master.data.updates.findBuilderId('testy-2')),
]
finished_builds = []
self.successResultOf(master.mq.startConsuming(
lambda key, build: finished_builds.append(build),
('builds', None, 'finished')))
# Trigger a buildrequest
bsid, brids = self.successResultOf(
master.data.updates.addBuildset(
waited_for=False,
builderids=builder_ids,
sourcestamps=[
{'codebase': '',
'repository': '',
'branch': None,
'revision': None,
'project': ''},
],
)
)
# The worker fails to substantiate.
controller.start_instance(True)
local_workdir = FilePath(self.mktemp())
local_workdir.createDirectory()
controller.connect_worker(local_workdir)
# We check that there were two builds that finished, and
# that they both finished with success
self.assertEqual([build['results']
for build in finished_builds], [SUCCESS] * 2)
def test_failed_substantiations_get_exception(self):
"""
If a latent worker fails to substantiate, the result is an exception.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
self.createBuildrequest(master, [builder_id])
# The worker fails to substantiate.
controller.start_instance(
Failure(LatentWorkerCannotSubstantiate("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(LatentWorkerCannotSubstantiate)
dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1)
# When the substantiation fails, the result is an exception.
self.assertEqual(EXCEPTION, dbdict['results'])
controller.auto_stop(True)
def test_worker_accepts_builds_after_failure(self):
"""
If a latent worker fails to substantiate, the worker is still able to accept jobs.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
master = self.successResultOf(
getMaster(self, self.reactor, config_dict))
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.successResultOf(
master.data.updates.addBuildset(
waited_for=False,
builderids=[builder_id],
sourcestamps=[
{
'codebase': '',
'repository': '',
'branch': None,
'revision': None,
'project': ''
},
],
))
unclaimed_build_requests = []
self.successResultOf(
master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# The worker fails to substantiate.
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# If the worker started again after the failure, then the retry logic will have
# already kicked in to start a new build on this (the only) worker. We check that
# a new instance was requested, which indicates that the worker
# accepted the build.
self.assertEqual(controller.started, True)
def test_failed_substantiations_get_requeued(self):
"""
If a latent worker fails to substantiate, the build request becomes unclaimed.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
master = self.successResultOf(
getMaster(self, self.reactor, config_dict))
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.successResultOf(
master.data.updates.addBuildset(
waited_for=False,
builderids=[builder_id],
sourcestamps=[
{
'codebase': '',
'repository': '',
'branch': None,
'revision': None,
'project': ''
},
],
))
unclaimed_build_requests = []
self.successResultOf(
master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# The worker fails to substantiate.
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# When the substantiation fails, the buildrequest becomes unclaimed.
self.assertEqual(
set(brids),
set([req['buildrequestid'] for req in unclaimed_build_requests]),
)
def test_worker_accepts_builds_after_failure(self):
"""
If a latent worker fails to substantiate, the worker is still able to accept jobs.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
controller.auto_stop(True)
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# The worker fails to substantiate.
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# The retry logic should only trigger after a exponential backoff
self.assertEqual(controller.starting, False)
# advance the time to the point where we should retry
master.reactor.advance(controller.worker.quarantine_initial_timeout)
# If the worker started again after the failure, then the retry logic will have
# already kicked in to start a new build on this (the only) worker. We check that
# a new instance was requested, which indicates that the worker
# accepted the build.
self.assertEqual(controller.starting, True)
# The worker fails to substantiate(again).
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# advance the time to the point where we should not retry
master.reactor.advance(controller.worker.quarantine_initial_timeout)
self.assertEqual(controller.starting, False)
# advance the time to the point where we should retry
master.reactor.advance(controller.worker.quarantine_initial_timeout)
self.assertEqual(controller.starting, True)
def test_latent_max_builds(self):
"""
If max_builds is set, only one build is started on a latent
worker at a time.
"""
controller = LatentController(
'local',
max_builds=1,
)
step_controller = StepController()
config_dict = {
'builders': [
BuilderConfig(name="testy-1",
workernames=["local"],
factory=BuildFactory([step_controller]),
),
BuilderConfig(name="testy-2",
workernames=["local"],
factory=BuildFactory([step_controller]),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
'multiMaster': True,
}
self.master = master = self.successResultOf(
getMaster(self, self.reactor, config_dict))
builder_ids = [
self.successResultOf(master.data.updates.findBuilderId('testy-1')),
self.successResultOf(master.data.updates.findBuilderId('testy-2')),
]
started_builds = []
self.successResultOf(master.mq.startConsuming(
lambda key, build: started_builds.append(build),
('builds', None, 'new')))
# Trigger a buildrequest
bsid, brids = self.successResultOf(
master.data.updates.addBuildset(
waited_for=False,
builderids=builder_ids,
sourcestamps=[
{'codebase': '',
'repository': '',
'branch': None,
'revision': None,
'project': ''},
],
)
)
# The worker fails to substantiate.
controller.start_instance(True)
controller.connect_worker(self)
self.assertEqual(len(started_builds), 1)
controller.auto_stop(True)
def test_worker_accepts_builds_after_failure(self):
"""
If a latent worker fails to substantiate, the worker is still able to accept jobs.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
controller.auto_stop(True)
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# The worker fails to substantiate.
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# The retry logic should only trigger after a exponential backoff
self.assertEqual(controller.starting, False)
# advance the time to the point where we should retry
master.reactor.advance(controller.worker.quarantine_initial_timeout)
# If the worker started again after the failure, then the retry logic will have
# already kicked in to start a new build on this (the only) worker. We check that
# a new instance was requested, which indicates that the worker
# accepted the build.
self.assertEqual(controller.starting, True)
# The worker fails to substantiate(again).
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# advance the time to the point where we should not retry
master.reactor.advance(controller.worker.quarantine_initial_timeout)
self.assertEqual(controller.starting, False)
# advance the time to the point where we should retry
master.reactor.advance(controller.worker.quarantine_initial_timeout)
self.assertEqual(controller.starting, True)
def test_failed_substantiations_get_requeued(self):
"""
If a latent worker fails to substantiate, the build request becomes unclaimed.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.successResultOf(
getMaster(self, self.reactor, config_dict))
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.successResultOf(
master.data.updates.addBuildset(
waited_for=False,
builderids=[builder_id],
sourcestamps=[
{'codebase': '',
'repository': '',
'branch': None,
'revision': None,
'project': ''},
],
)
)
unclaimed_build_requests = []
self.successResultOf(master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# The worker fails to substantiate.
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# When the substantiation fails, the buildrequest becomes unclaimed.
self.assertEqual(
set(brids),
set([req['buildrequestid'] for req in unclaimed_build_requests]),
)
def test_worker_accepts_builds_after_failure(self):
"""
If a latent worker fails to substantiate, the worker is still able to accept jobs.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.successResultOf(
getMaster(self, self.reactor, config_dict))
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.successResultOf(
master.data.updates.addBuildset(
waited_for=False,
builderids=[builder_id],
sourcestamps=[
{'codebase': '',
'repository': '',
'branch': None,
'revision': None,
'project': ''},
],
)
)
unclaimed_build_requests = []
self.successResultOf(master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# The worker fails to substantiate.
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# If the worker started again after the failure, then the retry logic will have
# already kicked in to start a new build on this (the only) worker. We check that
# a new instance was requested, which indicates that the worker
# accepted the build.
self.assertEqual(controller.started, True)
def test_worker_multiple_substantiations_succeed(self):
"""
If multiple builders trigger try to substantiate a worker at
the same time, if the substantiation succeeds then all of
the builds proceed.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(
name="testy-1",
workernames=["local"],
factory=BuildFactory(),
),
BuilderConfig(
name="testy-2",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder_ids = [
self.successResultOf(master.data.updates.findBuilderId('testy-1')),
self.successResultOf(master.data.updates.findBuilderId('testy-2')),
]
finished_builds = []
self.successResultOf(
master.mq.startConsuming(
lambda key, build: finished_builds.append(build),
('builds', None, 'finished')))
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, builder_ids)
# The worker succeeds to substantiate.
controller.start_instance(True)
controller.connect_worker(self)
# We check that there were two builds that finished, and
# that they both finished with success
self.assertEqual([build['results'] for build in finished_builds],
[SUCCESS] * 2)
controller.auto_stop(True)
def test_worker_close_connection_while_building(self):
"""
If the worker close connection in the middle of the build, the next build can start correctly
"""
controller = LatentController('local', build_wait_timeout=0)
# a step that we can finish when we want
stepcontroller = BuildStepController()
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory([stepcontroller.step]),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Request two builds.
for i in range(2):
self.createBuildrequest(master, [builder_id])
controller.auto_stop(True)
self.assertTrue(controller.starting)
controller.start_instance(True)
controller.connect_worker(self)
builds = self.successResultOf(master.data.get(("builds", )))
self.assertEqual(builds[0]['results'], None)
controller.disconnect_worker(self)
builds = self.successResultOf(master.data.get(("builds", )))
self.assertEqual(builds[0]['results'], RETRY)
# Request one build.
self.createBuildrequest(master, [builder_id])
controller.start_instance(True)
controller.connect_worker(self)
builds = self.successResultOf(master.data.get(("builds", )))
self.assertEqual(builds[1]['results'], None)
stepcontroller.finish_step(SUCCESS)
builds = self.successResultOf(master.data.get(("builds", )))
self.assertEqual(builds[1]['results'], SUCCESS)
def test_build_stop_with_retry_during_substantiation(self):
"""
If master is shutting down during latent worker substantiating, the build becomes retry.
"""
controller = LatentController(self, 'local')
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder = master.botmaster.builders['testy']
builder_id = self.successResultOf(builder.getBuilderId())
# Trigger a buildrequest
_, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(
master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# Stop the build
build = builder.getBuild(0)
build.stopBuild('no reason', results=RETRY)
# Indicate that the worker can't start an instance.
controller.start_instance(False)
dbdict = self.successResultOf(
master.db.builds.getBuildByNumber(builder_id, 1))
self.assertEqual(RETRY, dbdict['results'])
self.assertEqual(
set(brids),
{req['buildrequestid']
for req in unclaimed_build_requests})
controller.auto_stop(True)
self.flushLoggedErrors(LatentWorkerFailedToSubstantiate)
def test_worker_close_connection_while_building(self):
"""
If the worker close connection in the middle of the build, the next build can start correctly
"""
controller = LatentController('local', build_wait_timeout=0)
# a step that we can finish when we want
stepcontroller = BuildStepController()
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory([stepcontroller.step]),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Request two builds.
for i in range(2):
self.createBuildrequest(master, [builder_id])
controller.auto_stop(True)
self.assertTrue(controller.starting)
controller.start_instance(True)
controller.connect_worker(self)
builds = self.successResultOf(
master.data.get(("builds",)))
self.assertEqual(builds[0]['results'], None)
controller.disconnect_worker(self)
builds = self.successResultOf(
master.data.get(("builds",)))
self.assertEqual(builds[0]['results'], RETRY)
# Request one build.
self.createBuildrequest(master, [builder_id])
controller.start_instance(True)
controller.connect_worker(self)
builds = self.successResultOf(
master.data.get(("builds",)))
self.assertEqual(builds[1]['results'], None)
stepcontroller.finish_step(SUCCESS)
builds = self.successResultOf(
master.data.get(("builds",)))
self.assertEqual(builds[1]['results'], SUCCESS)
def test_failed_substantiations_get_requeued(self):
"""
If a latent worker fails to substantiate, the build request becomes unclaimed.
"""
controller = LatentController(self, 'local')
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(
master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# The worker fails to substantiate.
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# When the substantiation fails, the buildrequest becomes unclaimed.
self.assertEqual(
set(brids),
{req['buildrequestid']
for req in unclaimed_build_requests})
controller.auto_stop(True)
def test_worker_multiple_substantiations_succeed(self):
"""
If multiple builders trigger try to substantiate a worker at
the same time, if the substantiation succeeds then all of
the builds proceeed.
"""
controller = LatentController("local")
config_dict = {
"builders": [
BuilderConfig(name="testy-1", workernames=["local"], factory=BuildFactory()),
BuilderConfig(name="testy-2", workernames=["local"], factory=BuildFactory()),
],
"workers": [controller.worker],
"protocols": {"null": {}},
"multiMaster": True,
}
master = self.getMaster(config_dict)
builder_ids = [
self.successResultOf(master.data.updates.findBuilderId("testy-1")),
self.successResultOf(master.data.updates.findBuilderId("testy-2")),
]
finished_builds = []
self.successResultOf(
master.mq.startConsuming(lambda key, build: finished_builds.append(build), ("builds", None, "finished"))
)
# Trigger a buildrequest
bsid, brids = self.successResultOf(
master.data.updates.addBuildset(
waited_for=False,
builderids=builder_ids,
sourcestamps=[{"codebase": "", "repository": "", "branch": None, "revision": None, "project": ""}],
)
)
# The worker fails to substantiate.
controller.start_instance(True)
local_workdir = FilePath(self.mktemp())
local_workdir.createDirectory()
controller.connect_worker(local_workdir)
# We check that there were two builds that finished, and
# that they both finished with success
self.assertEqual([build["results"] for build in finished_builds], [SUCCESS] * 2)
def test_worker_multiple_substantiations_succeed(self):
"""
If multiple builders trigger try to substantiate a worker at
the same time, if the substantiation succeeds then all of
the builds proceeed.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy-1",
workernames=["local"],
factory=BuildFactory(),
),
BuilderConfig(name="testy-2",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_ids = [
self.successResultOf(master.data.updates.findBuilderId('testy-1')),
self.successResultOf(master.data.updates.findBuilderId('testy-2')),
]
finished_builds = []
self.successResultOf(master.mq.startConsuming(
lambda key, build: finished_builds.append(build),
('builds', None, 'finished')))
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, builder_ids)
# The worker succeeds to substantiate.
controller.start_instance(True)
controller.connect_worker(self)
# We check that there were two builds that finished, and
# that they both finished with success
self.assertEqual([build['results']
for build in finished_builds], [SUCCESS] * 2)
controller.auto_stop(True)
def test_build_stop_with_retry_during_substantiation(self):
"""
If master is shutting down during latent worker substantiating, the build becomes retry.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder = master.botmaster.builders['testy']
builder_id = self.successResultOf(builder.getBuilderId())
# Trigger a buildrequest
_, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# Stop the build
build = builder.getBuild(0)
build.stopBuild('no reason', results=RETRY)
# Indicate that the worker can't start an instance.
controller.start_instance(False)
dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1)
self.assertEqual(RETRY, dbdict['results'])
self.assertEqual(
set(brids),
set([req['buildrequestid'] for req in unclaimed_build_requests]),
)
controller.auto_stop(True)
self.flushLoggedErrors(LatentWorkerFailedToSubstantiate)
def test_failed_substantiations_get_requeued(self):
"""
If a latent worker fails to substantiate, the build request becomes unclaimed.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
# The worker fails to substantiate.
controller.start_instance(
Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# When the substantiation fails, the buildrequest becomes unclaimed.
self.assertEqual(
set(brids),
{req['buildrequestid'] for req in unclaimed_build_requests}
)
controller.auto_stop(True)
def test_failed_substantiations_get_requeued(self):
"""
If a latent worker fails to substantiate, the build request becomes unclaimed.
"""
controller = LatentController("local")
config_dict = {
"builders": [BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory())],
"workers": [controller.worker],
"protocols": {"null": {}},
# Disable checks about missing scheduler.
"multiMaster": True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(master.data.updates.findBuilderId("testy"))
# Trigger a buildrequest
bsid, brids = self.successResultOf(
master.data.updates.addBuildset(
waited_for=False,
builderids=[builder_id],
sourcestamps=[{"codebase": "", "repository": "", "branch": None, "revision": None, "project": ""}],
)
)
unclaimed_build_requests = []
self.successResultOf(
master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request), ("buildrequests", None, "unclaimed")
)
)
# The worker fails to substantiate.
controller.start_instance(Failure(TestException("substantiation failed")))
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# When the substantiation fails, the buildrequest becomes unclaimed.
self.assertEqual(set(brids), set([req["buildrequestid"] for req in unclaimed_build_requests]))
def test_failed_ping_get_requeued(self):
"""
sendBuilderList can fail due to missing permissions on the workdir,
the build request becomes unclaimed
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(
name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
# Disable checks about missing scheduler.
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(
master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
logs = []
self.successResultOf(
master.mq.startConsuming(lambda key, log: logs.append(log),
('logs', None, 'new')))
# The worker succeed to substantiate
def remote_print(self, msg):
if msg == "ping":
raise TestException("can't ping")
controller.patchBot(self, 'remote_print', remote_print)
controller.start_instance(True)
controller.connect_worker(self)
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# When the substantiation fails, the buildrequest becomes unclaimed.
self.assertEqual(
set(brids),
set([req['buildrequestid'] for req in unclaimed_build_requests]),
)
# should get 2 logs (html and txt) with proper information in there
self.assertEqual(len(logs), 2)
logs_by_name = {}
for _log in logs:
fulllog = self.successResultOf(
master.data.get(("logs", str(_log['logid']), "raw")))
logs_by_name[fulllog['filename']] = fulllog['raw']
for i in ["err_text", "err_html"]:
self.assertIn("can't ping", logs_by_name[i])
# make sure stacktrace is present in html
self.assertIn(
"buildbot.test.integration.test_latent.TestException",
logs_by_name[i])
controller.auto_stop(True)
def test_worker_multiple_substantiations_succeed(self):
"""
If multiple builders trigger try to substantiate a worker at
the same time, if the substantiation succeeds then all of
the builds proceeed.
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(
name="testy-1",
workernames=["local"],
factory=BuildFactory(),
),
BuilderConfig(
name="testy-2",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {
'null': {}
},
'multiMaster':
True,
}
master = self.getMaster(config_dict)
builder_ids = [
self.successResultOf(master.data.updates.findBuilderId('testy-1')),
self.successResultOf(master.data.updates.findBuilderId('testy-2')),
]
finished_builds = []
self.successResultOf(
master.mq.startConsuming(
lambda key, build: finished_builds.append(build),
('builds', None, 'finished')))
# Trigger a buildrequest
bsid, brids = self.successResultOf(
master.data.updates.addBuildset(
waited_for=False,
builderids=builder_ids,
sourcestamps=[
{
'codebase': '',
'repository': '',
'branch': None,
'revision': None,
'project': ''
},
],
))
# The worker fails to substantiate.
controller.start_instance(True)
local_workdir = FilePath(self.mktemp())
local_workdir.createDirectory()
controller.connect_worker(local_workdir)
# We check that there were two builds that finished, and
# that they both finished with success
self.assertEqual([build['results'] for build in finished_builds],
[SUCCESS] * 2)
def test_failed_ping_get_requeued(self):
"""
sendBuilderList can fail due to missing permissions on the workdir,
the build request becomes unclaimed
"""
controller = LatentController('local')
config_dict = {
'builders': [
BuilderConfig(name="testy",
workernames=["local"],
factory=BuildFactory(),
),
],
'workers': [controller.worker],
'protocols': {'null': {}},
# Disable checks about missing scheduler.
'multiMaster': True,
}
master = self.getMaster(config_dict)
builder_id = self.successResultOf(
master.data.updates.findBuilderId('testy'))
# Trigger a buildrequest
bsid, brids = self.createBuildrequest(master, [builder_id])
unclaimed_build_requests = []
self.successResultOf(master.mq.startConsuming(
lambda key, request: unclaimed_build_requests.append(request),
('buildrequests', None, 'unclaimed')))
logs = []
self.successResultOf(master.mq.startConsuming(
lambda key, log: logs.append(log),
('logs', None, 'new')))
# The worker succeed to substantiate
def remote_print(self, msg):
if msg == "ping":
raise TestException("can't ping")
controller.patchBot(self, 'remote_print', remote_print)
controller.start_instance(True)
controller.connect_worker(self)
# Flush the errors logged by the failure.
self.flushLoggedErrors(TestException)
# When the substantiation fails, the buildrequest becomes unclaimed.
self.assertEqual(
set(brids),
set([req['buildrequestid'] for req in unclaimed_build_requests]),
)
# should get 2 logs (html and txt) with proper information in there
self.assertEqual(len(logs), 2)
logs_by_name = {}
for _log in logs:
fulllog = self.successResultOf(
master.data.get(("logs", str(_log['logid']), "raw")))
logs_by_name[fulllog['filename']] = fulllog['raw']
for i in ["err_text", "err_html"]:
self.assertIn("can't ping", logs_by_name[i])
# make sure stacktrace is present in html
self.assertIn(os.path.join(
"integration", "test_latent.py"), logs_by_name[i])
controller.auto_stop(True)