def test_makeSlaveFromVitals(self):
# Builder.slave is a BuilderSlave that points at the actual Builder.
# The Builder is only ever used in scripts that run outside of the
# security context.
builder = MockBuilder(virtualized=False)
vitals = extract_vitals_from_db(builder)
slave = BuilderInteractor.makeSlaveFromVitals(vitals)
self.assertEqual(builder.url, slave.url)
self.assertEqual(10, slave.timeout)
builder = MockBuilder(virtualized=True)
vitals = extract_vitals_from_db(builder)
slave = BuilderInteractor.makeSlaveFromVitals(vitals)
self.assertEqual(5, slave.timeout)
def test_makeSlaveFromVitals(self):
# Builder.slave is a BuilderSlave that points at the actual Builder.
# The Builder is only ever used in scripts that run outside of the
# security context.
builder = MockBuilder(virtualized=False)
vitals = extract_vitals_from_db(builder)
slave = BuilderInteractor.makeSlaveFromVitals(vitals)
self.assertEqual(builder.url, slave.url)
self.assertEqual(10, slave.timeout)
builder = MockBuilder(virtualized=True)
vitals = extract_vitals_from_db(builder)
slave = BuilderInteractor.makeSlaveFromVitals(vitals)
self.assertEqual(5, slave.timeout)
def test_virtual_ppa_dispatch(self):
# Make sure the builder slave gets reset before a build is
# dispatched to it.
archive = self.factory.makeArchive(virtualized=True)
slave = OkSlave()
builder = self.factory.makeBuilder(virtualized=True, vm_host="foohost")
vitals = extract_vitals_from_db(builder)
build = self.factory.makeBinaryPackageBuild(builder=builder,
archive=archive)
lf = self.factory.makeLibraryFileAlias()
transaction.commit()
build.distro_arch_series.addOrUpdateChroot(lf)
bq = build.queueBuild()
bq.markAsBuilding(builder)
interactor = BuilderInteractor()
d = interactor._startBuild(
bq, vitals, builder, slave,
interactor.getBuildBehavior(bq, builder, slave), BufferLogger())
def check_build(ignored):
# We expect the first call to the slave to be a resume call,
# followed by the rest of the usual calls we expect.
expected_resume_call = slave.call_log.pop(0)
self.assertEqual('resume', expected_resume_call)
self.assertExpectedInteraction(ignored, slave.call_log, builder,
build, lf, archive,
ArchivePurpose.PPA)
return d.addCallback(check_build)
def test_non_virtual_ppa_dispatch(self):
# When the BinaryPackageBuildBehavior dispatches PPA builds to
# non-virtual builders, it stores the chroot on the server and
# requests a binary package build, lying to say that the archive
# purpose is "PRIMARY" because this ensures that the package mangling
# tools will run over the built packages.
archive = self.factory.makeArchive(virtualized=False)
slave = OkSlave()
builder = self.factory.makeBuilder(virtualized=False)
vitals = extract_vitals_from_db(builder)
build = self.factory.makeBinaryPackageBuild(
builder=builder, archive=archive)
lf = self.factory.makeLibraryFileAlias()
transaction.commit()
build.distro_arch_series.addOrUpdateChroot(lf)
bq = build.queueBuild()
bq.markAsBuilding(builder)
interactor = BuilderInteractor()
d = interactor._startBuild(
bq, vitals, builder, slave,
interactor.getBuildBehavior(bq, builder, slave), BufferLogger())
d.addCallback(
self.assertExpectedInteraction, slave.call_log, builder, build,
lf, archive, ArchivePurpose.PRIMARY, 'universe')
return d
def test_virtual_ppa_dispatch(self):
# Make sure the builder slave gets reset before a build is
# dispatched to it.
archive = self.factory.makeArchive(virtualized=True)
slave = OkSlave()
builder = self.factory.makeBuilder(
virtualized=True, vm_host="foohost")
vitals = extract_vitals_from_db(builder)
build = self.factory.makeBinaryPackageBuild(
builder=builder, archive=archive)
lf = self.factory.makeLibraryFileAlias()
transaction.commit()
build.distro_arch_series.addOrUpdateChroot(lf)
bq = build.queueBuild()
bq.markAsBuilding(builder)
interactor = BuilderInteractor()
d = interactor._startBuild(
bq, vitals, builder, slave,
interactor.getBuildBehavior(bq, builder, slave), BufferLogger())
def check_build(ignored):
# We expect the first call to the slave to be a resume call,
# followed by the rest of the usual calls we expect.
expected_resume_call = slave.call_log.pop(0)
self.assertEqual('resume', expected_resume_call)
self.assertExpectedInteraction(
ignored, slave.call_log, builder, build, lf, archive,
ArchivePurpose.PPA)
return d.addCallback(check_build)
def test_non_virtual_ppa_dispatch(self):
# When the BinaryPackageBuildBehaviour dispatches PPA builds to
# non-virtual builders, it stores the chroot on the server and
# requests a binary package build, lying to say that the archive
# purpose is "PRIMARY" because this ensures that the package mangling
# tools will run over the built packages.
archive = self.factory.makeArchive(virtualized=False)
slave = OkSlave()
builder = self.factory.makeBuilder(virtualized=False)
builder.setCleanStatus(BuilderCleanStatus.CLEAN)
vitals = extract_vitals_from_db(builder)
build = self.factory.makeBinaryPackageBuild(builder=builder,
archive=archive)
lf = self.factory.makeLibraryFileAlias()
transaction.commit()
build.distro_arch_series.addOrUpdateChroot(lf)
bq = build.queueBuild()
bq.markAsBuilding(builder)
interactor = BuilderInteractor()
yield interactor._startBuild(
bq, vitals, builder, slave,
interactor.getBuildBehaviour(bq, builder, slave), BufferLogger())
yield self.assertExpectedInteraction(slave.call_log, builder, build,
lf, archive,
ArchivePurpose.PRIMARY,
'universe')
def test_non_virtual_ppa_dispatch_with_primary_ancestry(self):
# If there is a primary component override, it is honoured for
# non-virtual PPA builds too.
archive = self.factory.makeArchive(virtualized=False)
slave = OkSlave()
builder = self.factory.makeBuilder(virtualized=False)
builder.setCleanStatus(BuilderCleanStatus.CLEAN)
vitals = extract_vitals_from_db(builder)
build = self.factory.makeBinaryPackageBuild(builder=builder,
archive=archive)
self.factory.makeSourcePackagePublishingHistory(
distroseries=build.distro_series,
archive=archive.distribution.main_archive,
sourcepackagename=build.source_package_release.sourcepackagename,
component='main')
lf = self.factory.makeLibraryFileAlias()
transaction.commit()
build.distro_arch_series.addOrUpdateChroot(lf)
bq = build.queueBuild()
bq.markAsBuilding(builder)
interactor = BuilderInteractor()
yield interactor._startBuild(
bq, vitals, builder, slave,
interactor.getBuildBehaviour(bq, builder, slave), BufferLogger())
yield self.assertExpectedInteraction(slave.call_log, builder, build,
lf, archive,
ArchivePurpose.PRIMARY, 'main')
def test_recover_ok_slave(self):
# An idle slave that's meant to be idle is not rescued.
slave = OkSlave()
lost = yield BuilderInteractor.rescueIfLost(
extract_vitals_from_db(MockBuilder()), slave, None)
self.assertFalse(lost)
self.assertEqual([], slave.call_log)
def test_private_source_dispatch(self):
archive = self.factory.makeArchive(private=True)
slave = OkSlave()
builder = self.factory.makeBuilder()
builder.setCleanStatus(BuilderCleanStatus.CLEAN)
vitals = extract_vitals_from_db(builder)
build = self.factory.makeBinaryPackageBuild(builder=builder,
archive=archive)
sprf = build.source_package_release.addFile(
self.factory.makeLibraryFileAlias(db_only=True),
filetype=SourcePackageFileType.ORIG_TARBALL)
sprf_url = (
'http://private-ppa.launchpad.dev/%s/%s/ubuntu/pool/%s/%s' %
(archive.owner.name, archive.name,
poolify(build.source_package_release.sourcepackagename.name,
'main'), sprf.libraryfile.filename))
lf = self.factory.makeLibraryFileAlias()
transaction.commit()
build.distro_arch_series.addOrUpdateChroot(lf)
bq = build.queueBuild()
bq.markAsBuilding(builder)
interactor = BuilderInteractor()
yield interactor._startBuild(
bq, vitals, builder, slave,
interactor.getBuildBehaviour(bq, builder, slave), BufferLogger())
yield self.assertExpectedInteraction(
slave.call_log,
builder,
build,
lf,
archive,
ArchivePurpose.PPA,
extra_uploads=[(sprf_url, 'buildd', 'sekrit')],
filemap_names=[sprf.libraryfile.filename])
def test_recover_building_slave_with_bad_id(self):
# If a slave is BUILDING with a build id we don't recognize, then we
# abort the build, thus stopping it in its tracks.
building_slave = BuildingSlave(build_id='non-trivial')
lost = yield BuilderInteractor.rescueIfLost(
extract_vitals_from_db(MockBuilder()), building_slave, 'trivial')
self.assertTrue(lost)
self.assertEqual(['status', 'abort'], building_slave.call_log)
def test_recover_building_slave_with_good_id(self):
# rescueIfLost does not attempt to abort or clean a builder that is
# BUILDING.
building_slave = BuildingSlave(build_id='trivial')
lost = yield BuilderInteractor.rescueIfLost(
extract_vitals_from_db(MockBuilder()), building_slave, 'trivial')
self.assertFalse(lost)
self.assertEqual(['status'], building_slave.call_log)
def assertQuerylessVitals(comparator):
expected_vitals = extract_vitals_from_db(builder)
transaction.commit()
with StormStatementRecorder() as recorder:
got_vitals = pbf.getVitals(name)
comparator(expected_vitals, got_vitals)
comparator(expected_vitals.build_queue, got_vitals.build_queue)
self.assertThat(recorder, HasQueryCount(Equals(0)))
return got_vitals
def test_recover_idle_slave(self):
# An idle slave is not rescued, even if it's not meant to be
# idle. SlaveScanner.scan() will clean up the DB side, because
# we still report that it's lost.
slave = OkSlave()
lost = yield BuilderInteractor.rescueIfLost(
extract_vitals_from_db(MockBuilder()), slave, 'trivial')
self.assertTrue(lost)
self.assertEqual([], slave.call_log)
def test_findAndStartJob_dirties_slave(self):
# findAndStartJob marks its builder DIRTY before dispatching.
builder, build = self._setupBinaryBuildAndBuilder()
candidate = build.queueBuild()
removeSecurityProxy(builder)._findBuildCandidate = FakeMethod(
result=candidate)
vitals = extract_vitals_from_db(builder)
yield BuilderInteractor.findAndStartJob(vitals, builder, OkSlave())
self.assertEqual(BuilderCleanStatus.DIRTY, builder.clean_status)
def assertQuerylessVitals(comparator):
expected_vitals = extract_vitals_from_db(builder)
transaction.commit()
with StormStatementRecorder() as recorder:
got_vitals = pbf.getVitals(name)
comparator(expected_vitals, got_vitals)
comparator(expected_vitals.build_queue, got_vitals.build_queue)
self.assertThat(recorder, HasQueryCount(Equals(0)))
return got_vitals
def test_recover_waiting_slave_with_bad_id(self):
# If a slave is WAITING with a build for us to get, and the build
# cookie cannot be verified, which means we don't recognize the build,
# then rescueBuilderIfLost should attempt to abort it, so that the
# builder is reset for a new build, and the corrupt build is
# discarded.
waiting_slave = WaitingSlave(build_id='non-trivial')
lost = yield BuilderInteractor.rescueIfLost(
extract_vitals_from_db(MockBuilder()), waiting_slave, 'trivial')
self.assertTrue(lost)
self.assertEqual(['status', 'clean'], waiting_slave.call_log)
def update(self):
"""See `BuilderFactory`."""
transaction.abort()
builders_and_bqs = IStore(Builder).using(
Builder, LeftJoin(BuildQueue, BuildQueue.builderID == Builder.id)
).find((Builder, BuildQueue))
self.vitals_map = dict(
(b.name, extract_vitals_from_db(b, bq))
for b, bq in builders_and_bqs)
transaction.abort()
self.date_updated = datetime.datetime.utcnow()
def update(self):
"""See `BuilderFactory`."""
transaction.abort()
builders_and_bqs = IStore(Builder).using(
Builder, LeftJoin(BuildQueue,
BuildQueue.builderID == Builder.id)).find(
(Builder, BuildQueue))
self.vitals_map = dict((b.name, extract_vitals_from_db(b, bq))
for b, bq in builders_and_bqs)
transaction.abort()
self.date_updated = datetime.datetime.utcnow()
def test_findAndStartJob_requires_clean_slave(self):
# findAndStartJob ensures that its slave starts CLEAN.
builder, build = self._setupBinaryBuildAndBuilder()
builder.setCleanStatus(BuilderCleanStatus.DIRTY)
candidate = build.queueBuild()
removeSecurityProxy(builder)._findBuildCandidate = FakeMethod(
result=candidate)
vitals = extract_vitals_from_db(builder)
with ExpectedException(BuildDaemonIsolationError,
"Attempted to start build on a dirty slave."):
yield BuilderInteractor.findAndStartJob(vitals, builder, OkSlave())
def test_virtual_no_protocol(self):
# Virtual builders fail to clean unless vm_reset_protocol is
# set.
builder = MockBuilder(virtualized=True,
clean_status=BuilderCleanStatus.DIRTY,
vm_host='lol')
builder.vm_reset_protocol = None
with ExpectedException(CannotResumeHost,
"Invalid vm_reset_protocol: None"):
yield BuilderInteractor.cleanSlave(
extract_vitals_from_db(builder), OkSlave(),
MockBuilderFactory(builder, None))
def test_findAndStartJob_returns_candidate(self):
# findAndStartJob finds the next queued job using _findBuildCandidate.
# We don't care about the type of build at all.
builder, build = self._setupRecipeBuildAndBuilder()
candidate = build.queueBuild()
# _findBuildCandidate is tested elsewhere, we just make sure that
# findAndStartJob delegates to it.
removeSecurityProxy(builder)._findBuildCandidate = FakeMethod(
result=candidate)
vitals = extract_vitals_from_db(builder)
d = BuilderInteractor.findAndStartJob(vitals, builder, OkSlave())
return d.addCallback(self.assertEqual, candidate)
def test_findAndStartJob_returns_candidate(self):
# findAndStartJob finds the next queued job using _findBuildCandidate.
# We don't care about the type of build at all.
builder, build = self._setupRecipeBuildAndBuilder()
candidate = build.queueBuild()
# _findBuildCandidate is tested elsewhere, we just make sure that
# findAndStartJob delegates to it.
removeSecurityProxy(builder)._findBuildCandidate = FakeMethod(
result=candidate)
vitals = extract_vitals_from_db(builder)
d = BuilderInteractor.findAndStartJob(vitals, builder, OkSlave())
return d.addCallback(self.assertEqual, candidate)
def test_resetOrFail_resume_failure(self):
reset_fail_config = """
[builddmaster]
vm_resume_command: /bin/false"""
config.push('reset fail', reset_fail_config)
self.addCleanup(config.pop, 'reset fail')
builder = MockBuilder(virtualized=True, vm_host="pop", builderok=True)
vitals = extract_vitals_from_db(builder)
d = BuilderInteractor.resetOrFail(
vitals, BuilderInteractor.makeSlaveFromVitals(vitals), builder,
DevNullLogger(), Exception())
return assert_fails_with(d, CannotResumeHost)
def test_nonvirtual_broken(self):
# A broken non-virtual builder is probably unrecoverable, so the
# method just crashes.
builder = MockBuilder(virtualized=False,
clean_status=BuilderCleanStatus.DIRTY)
vitals = extract_vitals_from_db(builder)
slave = LostBuildingBrokenSlave()
try:
yield BuilderInteractor.cleanSlave(
vitals, slave, MockBuilderFactory(builder, None))
except xmlrpclib.Fault:
self.assertEqual(['status', 'abort'], slave.call_log)
else:
self.fail("abort() should crash.")
def test_rescueIfLost_aborts_lost_and_broken_slave(self):
# A slave that's 'lost' should be aborted; when the slave is
# broken then abort() should also throw a fault.
slave = LostBuildingBrokenSlave()
d = BuilderInteractor.rescueIfLost(
extract_vitals_from_db(MockBuilder()), slave, 'trivial')
def check_slave_status(failure):
self.assertIn('abort', slave.call_log)
# 'Fault' comes from the LostBuildingBrokenSlave, this is
# just testing that the value is passed through.
self.assertIsInstance(failure.value, xmlrpclib.Fault)
return d.addBoth(check_slave_status)
def test_virtual_job_dispatch_pings_before_building(self):
# We need to send a ping to the builder to work around a bug
# where sometimes the first network packet sent is dropped.
builder, build = self._setupBinaryBuildAndBuilder()
candidate = build.queueBuild()
removeSecurityProxy(builder)._findBuildCandidate = FakeMethod(
result=candidate)
vitals = extract_vitals_from_db(builder)
slave = OkSlave()
d = BuilderInteractor.findAndStartJob(vitals, builder, slave)
def check_build_started(candidate):
self.assertIn(('echo', 'ping'), slave.call_log)
return d.addCallback(check_build_started)
def test_findAndStartJob_starts_job(self):
# findAndStartJob finds the next queued job using _findBuildCandidate
# and then starts it.
# We don't care about the type of build at all.
builder, build = self._setupRecipeBuildAndBuilder()
candidate = build.queueBuild()
removeSecurityProxy(builder)._findBuildCandidate = FakeMethod(
result=candidate)
vitals = extract_vitals_from_db(builder)
d = BuilderInteractor.findAndStartJob(vitals, builder, OkSlave())
def check_build_started(candidate):
self.assertEqual(candidate.builder, builder)
self.assertEqual(BuildStatus.BUILDING, build.status)
return d.addCallback(check_build_started)
def test_findAndStartJob_starts_job(self):
# findAndStartJob finds the next queued job using _findBuildCandidate
# and then starts it.
# We don't care about the type of build at all.
builder, build = self._setupRecipeBuildAndBuilder()
candidate = build.queueBuild()
removeSecurityProxy(builder)._findBuildCandidate = FakeMethod(
result=candidate)
vitals = extract_vitals_from_db(builder)
d = BuilderInteractor.findAndStartJob(vitals, builder, OkSlave())
def check_build_started(candidate):
self.assertEqual(candidate.builder, builder)
self.assertEqual(BuildStatus.BUILDING, build.status)
return d.addCallback(check_build_started)
def test_virtual_ppa_dispatch(self):
archive = self.factory.makeArchive(virtualized=True)
slave = OkSlave()
builder = self.factory.makeBuilder(virtualized=True, vm_host="foohost")
builder.setCleanStatus(BuilderCleanStatus.CLEAN)
vitals = extract_vitals_from_db(builder)
build = self.factory.makeBinaryPackageBuild(builder=builder,
archive=archive)
lf = self.factory.makeLibraryFileAlias()
transaction.commit()
build.distro_arch_series.addOrUpdateChroot(lf)
bq = build.queueBuild()
bq.markAsBuilding(builder)
interactor = BuilderInteractor()
yield interactor._startBuild(
bq, vitals, builder, slave,
interactor.getBuildBehaviour(bq, builder, slave), BufferLogger())
yield self.assertExpectedInteraction(slave.call_log, builder, build,
lf, archive, ArchivePurpose.PPA)
def test_partner_dispatch_no_publishing_history(self):
archive = self.factory.makeArchive(virtualized=False,
purpose=ArchivePurpose.PARTNER)
slave = OkSlave()
builder = self.factory.makeBuilder(virtualized=False)
vitals = extract_vitals_from_db(builder)
build = self.factory.makeBinaryPackageBuild(builder=builder,
archive=archive)
lf = self.factory.makeLibraryFileAlias()
transaction.commit()
build.distro_arch_series.addOrUpdateChroot(lf)
bq = build.queueBuild()
bq.markAsBuilding(builder)
interactor = BuilderInteractor()
d = interactor._startBuild(
bq, vitals, builder, slave,
interactor.getBuildBehavior(bq, builder, slave), BufferLogger())
d.addCallback(self.assertExpectedInteraction, slave.call_log, builder,
build, lf, archive, ArchivePurpose.PARTNER)
return d
def test_partner_dispatch_no_publishing_history(self):
archive = self.factory.makeArchive(
virtualized=False, purpose=ArchivePurpose.PARTNER)
slave = OkSlave()
builder = self.factory.makeBuilder(virtualized=False)
vitals = extract_vitals_from_db(builder)
build = self.factory.makeBinaryPackageBuild(
builder=builder, archive=archive)
lf = self.factory.makeLibraryFileAlias()
transaction.commit()
build.distro_arch_series.addOrUpdateChroot(lf)
bq = build.queueBuild()
bq.markAsBuilding(builder)
interactor = BuilderInteractor()
d = interactor._startBuild(
bq, vitals, builder, slave,
interactor.getBuildBehavior(bq, builder, slave), BufferLogger())
d.addCallback(
self.assertExpectedInteraction, slave.call_log, builder, build,
lf, archive, ArchivePurpose.PARTNER)
return d
def test_resetOrFail_nonvirtual(self):
builder = MockBuilder(virtualized=False, builderok=True)
vitals = extract_vitals_from_db(builder)
yield BuilderInteractor().resetOrFail(
vitals, None, builder, DevNullLogger(), Exception())
self.assertFalse(builder.builderok)
def assertCleanCalls(self, builder, slave, calls, done):
actually_done = yield BuilderInteractor.cleanSlave(
extract_vitals_from_db(builder), slave,
MockBuilderFactory(builder, None))
self.assertEqual(done, actually_done)
self.assertEqual(calls, slave.method_log)
def getVitals(self, name):
self.getVitals_call_count += 1
return extract_vitals_from_db(self._builder, self._build_queue)
def resumeSlaveHost(self, builder):
vitals = extract_vitals_from_db(builder)
return BuilderInteractor.resumeSlaveHost(
vitals, BuilderInteractor.makeSlaveFromVitals(vitals))
def vitals(self):
return extract_vitals_from_db(self.builder)
def _assessFailureCounts(self, fail_notes):
# Helper for assessFailureCounts boilerplate.
return assessFailureCounts(
BufferLogger(), extract_vitals_from_db(self.builder), self.builder,
self.slave, BuilderInteractor(), Exception(fail_notes))
def iterVitals(self):
"""Iterate over all `BuilderVitals` objects."""
return (
extract_vitals_from_db(b)
for b in getUtility(IBuilderSet).__iter__())
def _assessFailureCounts(self, fail_notes):
# Helper for assessFailureCounts boilerplate.
return assessFailureCounts(BufferLogger(),
extract_vitals_from_db(self.builder),
self.builder, self.slave,
BuilderInteractor(), Exception(fail_notes))
def getVitals(self, name):
self.getVitals_call_count += 1
return extract_vitals_from_db(self._builder, self._build_queue)
def resumeSlaveHost(self, builder):
vitals = extract_vitals_from_db(builder)
return BuilderInteractor.resumeSlaveHost(
vitals, BuilderInteractor.makeSlaveFromVitals(vitals))
def getVitals(self, name):
"""Get the named `BuilderVitals` object."""
return extract_vitals_from_db(self[name])
def getVitals(self, name):
"""Get the named `BuilderVitals` object."""
return extract_vitals_from_db(self[name])
def iterVitals(self):
"""Iterate over all `BuilderVitals` objects."""
return (extract_vitals_from_db(b)
for b in getUtility(IBuilderSet).__iter__())
def vitals(self):
return extract_vitals_from_db(self.builder)
