def test_dispatchBuildToSlave(self):
# Ensure dispatchBuildToSlave will make the right calls to the slave
job = self.makeJob()
test_publisher = SoyuzTestPublisher()
test_publisher.addFakeChroots(job.build.distroseries)
slave = OkSlave()
builder = MockBuilder("bob-de-bouwer")
builder.processor = getUtility(IProcessorSet).getByName("386")
job.setBuilder(builder, slave)
logger = BufferLogger()
d = defer.maybeDeferred(job.dispatchBuildToSlave, "someid", logger)
def check_dispatch(ignored):
self.assertThat(
logger.getLogBuffer(),
StartsWith(
dedent(
"""\
INFO Sending chroot file for recipe build to bob-de-bouwer
INFO Initiating build 1-someid on http://fake:0000
"""
)
),
)
self.assertEquals(["ensurepresent", "build"], [call[0] for call in slave.call_log])
build_args = slave.call_log[1][1:]
self.assertEquals(build_args[0], job.getBuildCookie())
self.assertEquals(build_args[1], "sourcepackagerecipe")
self.assertEquals(build_args[3], [])
distroarchseries = job.build.distroseries.architectures[0]
self.assertEqual(build_args[4], job._extraBuildArgs(distroarchseries))
return d.addCallback(check_dispatch)
def test_dispatchBuildToSlave(self):
# Ensure dispatchBuildToSlave will make the right calls to the slave
job = self.makeJob()
test_publisher = SoyuzTestPublisher()
test_publisher.addFakeChroots(job.build.distroseries)
slave = OkSlave()
builder = MockBuilder("bob-de-bouwer")
builder.processor = getUtility(IProcessorSet).getByName('386')
job.setBuilder(builder, slave)
logger = BufferLogger()
d = defer.maybeDeferred(job.dispatchBuildToSlave, "someid", logger)
def check_dispatch(ignored):
self.assertThat(
logger.getLogBuffer(),
StartsWith(
dedent("""\
INFO Sending chroot file for recipe build to bob-de-bouwer
INFO Initiating build 1-someid on http://fake:0000
""")))
self.assertEquals(["ensurepresent", "build"],
[call[0] for call in slave.call_log])
build_args = slave.call_log[1][1:]
self.assertEquals(build_args[0], job.getBuildCookie())
self.assertEquals(build_args[1], "sourcepackagerecipe")
self.assertEquals(build_args[3], [])
distroarchseries = job.build.distroseries.architectures[0]
self.assertEqual(build_args[4],
job._extraBuildArgs(distroarchseries))
return d.addCallback(check_dispatch)
def makeJob(self,
archive=None,
pocket=PackagePublishingPocket.RELEASE,
with_builder=False,
**kwargs):
"""Create a sample `ILiveFSBuildBehaviour`."""
if archive is None:
distribution = self.factory.makeDistribution(name="distro")
else:
distribution = archive.distribution
distroseries = self.factory.makeDistroSeries(distribution=distribution,
name="unstable")
processor = getUtility(IProcessorSet).getByName("386")
distroarchseries = self.factory.makeDistroArchSeries(
distroseries=distroseries,
architecturetag="i386",
processor=processor)
build = self.factory.makeLiveFSBuild(archive=archive,
distroarchseries=distroarchseries,
pocket=pocket,
name="test-livefs",
**kwargs)
job = IBuildFarmJobBehaviour(build)
if with_builder:
builder = MockBuilder()
builder.processor = processor
job.setBuilder(builder, None)
return job
def test_verifyBuildRequest_non_virtual(self):
# verifyBuildRequest will raise if a non-virtual builder is proposed.
job = self.makeJob()
builder = MockBuilder("non-virtual builder")
builder.virtualized = False
job.setBuilder(builder, OkSlave())
logger = BufferLogger()
e = self.assertRaises(AssertionError, job.verifyBuildRequest, logger)
self.assertEqual("Attempt to build virtual item on a non-virtual builder.", str(e))
def test_verifyBuildRequest_non_virtual(self):
# verifyBuildRequest will raise if a non-virtual builder is proposed.
job = self.makeJob()
builder = MockBuilder('non-virtual builder')
builder.virtualized = False
job.setBuilder(builder, OkSlave())
logger = BufferLogger()
e = self.assertRaises(AssertionError, job.verifyBuildRequest, logger)
self.assertEqual(
'Attempt to build virtual item on a non-virtual builder.', str(e))
def makeJob(self, **kwargs):
# We need a builder slave in these tests, in order that requesting a
# proxy token can piggyback on its reactor and pool.
job = super(TestAsyncSnapBuildBehaviour, self).makeJob(**kwargs)
builder = MockBuilder()
builder.processor = job.build.processor
slave = self.useFixture(SlaveTestHelpers()).getClientSlave()
job.setBuilder(builder, slave)
self.addCleanup(slave.pool.closeCachedConnections)
return job
def test_dispatchBuildToSlave_nochroot(self):
# dispatchBuildToSlave will fail when there is not chroot tarball
# available for the distroseries to build for.
job = self.makeJob()
#test_publisher = SoyuzTestPublisher()
builder = MockBuilder("bob-de-bouwer")
builder.processor = getUtility(IProcessorSet).getByName('386')
job.setBuilder(builder, OkSlave())
logger = BufferLogger()
d = defer.maybeDeferred(job.dispatchBuildToSlave, "someid", logger)
return assert_fails_with(d, CannotBuild)
def test_dispatchBuildToSlave_nochroot(self):
# dispatchBuildToSlave will fail when there is not chroot tarball
# available for the distroseries to build for.
job = self.makeJob()
# test_publisher = SoyuzTestPublisher()
builder = MockBuilder("bob-de-bouwer")
builder.processor = getUtility(IProcessorSet).getByName("386")
job.setBuilder(builder, OkSlave())
logger = BufferLogger()
d = defer.maybeDeferred(job.dispatchBuildToSlave, "someid", logger)
return assert_fails_with(d, CannotBuild)
def test_virtual_no_protocol(self):
# Virtual builders fail to clean unless vm_reset_protocol is
# set.
builder = MockBuilder(virtualized=True,
clean_status=BuilderCleanStatus.DIRTY,
vm_host='lol')
builder.vm_reset_protocol = None
with ExpectedException(CannotResumeHost,
"Invalid vm_reset_protocol: None"):
yield BuilderInteractor.cleanSlave(
extract_vitals_from_db(builder), OkSlave(),
MockBuilderFactory(builder, None))
def test_dispatchBuildToSlave_falls_back_to_chroot(self):
job = self.makeJob(allow_internet=False)
builder = MockBuilder()
builder.processor = job.build.processor
slave = OkSlave()
job.setBuilder(builder, slave)
chroot_lfa = self.factory.makeLibraryFileAlias(db_only=True)
job.build.distro_arch_series.addOrUpdateChroot(
chroot_lfa, image_type=BuildBaseImageType.CHROOT)
yield job.dispatchBuildToSlave(DevNullLogger())
self.assertEqual(('ensurepresent', chroot_lfa.http_url, '', ''),
slave.call_log[0])
def test_makeSlaveFromVitals(self):
# Builder.slave is a BuilderSlave that points at the actual Builder.
# The Builder is only ever used in scripts that run outside of the
# security context.
builder = MockBuilder(virtualized=False)
vitals = extract_vitals_from_db(builder)
slave = BuilderInteractor.makeSlaveFromVitals(vitals)
self.assertEqual(builder.url, slave.url)
self.assertEqual(10, slave.timeout)
builder = MockBuilder(virtualized=True)
vitals = extract_vitals_from_db(builder)
slave = BuilderInteractor.makeSlaveFromVitals(vitals)
self.assertEqual(5, slave.timeout)
def test_scan_aborts_lost_slave_with_job(self):
# SlaveScanner.scan uses BuilderInteractor.rescueIfLost to abort
# slaves that don't have the expected job.
slave = BuildingSlave('nontrivial')
bq = FakeBuildQueue()
# Instrument updateBuild.
interactor = BuilderInteractor()
interactor.updateBuild = FakeMethod()
scanner = SlaveScanner('mock',
MockBuilderFactory(MockBuilder(), bq),
BufferLogger(),
interactor_factory=FakeMethod(interactor),
slave_factory=FakeMethod(slave),
behavior_factory=FakeMethod(TrivialBehavior()))
# XXX: checkCancellation needs more than a FakeBuildQueue.
scanner.checkCancellation = FakeMethod(defer.succeed(False))
# A single scan will call status(), notice that the slave is
# lost, abort() the slave, then reset() the job without calling
# updateBuild().
yield scanner.scan()
self.assertEqual(['status', 'abort'], slave.call_log)
self.assertEqual(0, interactor.updateBuild.call_count)
self.assertEqual(1, bq.reset.call_count)
def test_recover_ok_slave(self):
# An idle slave that's meant to be idle is not rescued.
slave = OkSlave()
lost = yield BuilderInteractor.rescueIfLost(
extract_vitals_from_db(MockBuilder()), slave, None)
self.assertFalse(lost)
self.assertEqual([], slave.call_log)
def test_nonvirtual_aborting(self):
# An ABORTING non-virtual slave must be waited out. It should
# hit WAITING eventually.
yield self.assertCleanCalls(
MockBuilder(virtualized=False,
clean_status=BuilderCleanStatus.DIRTY),
AbortingSlave(), ['status'], False)
def test_nonvirtual_building(self):
# A BUILDING non-virtual slave needs to be aborted. It'll go
# through ABORTING and eventually be picked up from WAITING.
yield self.assertCleanCalls(
MockBuilder(virtualized=False,
clean_status=BuilderCleanStatus.DIRTY),
BuildingSlave(), ['status', 'abort'], False)
def test_recover_building_slave_with_good_id(self):
# rescueIfLost does not attempt to abort or clean a builder that is
# BUILDING.
building_slave = BuildingSlave(build_id='trivial')
lost = yield BuilderInteractor.rescueIfLost(
extract_vitals_from_db(MockBuilder()), building_slave, 'trivial')
self.assertFalse(lost)
self.assertEqual(['status'], building_slave.call_log)
def test_recover_building_slave_with_bad_id(self):
# If a slave is BUILDING with a build id we don't recognize, then we
# abort the build, thus stopping it in its tracks.
building_slave = BuildingSlave(build_id='non-trivial')
lost = yield BuilderInteractor.rescueIfLost(
extract_vitals_from_db(MockBuilder()), building_slave, 'trivial')
self.assertTrue(lost)
self.assertEqual(['status', 'abort'], building_slave.call_log)
def test_resumeSlaveHost_command_failed(self):
reset_fail_config = """
[builddmaster]
vm_resume_command: /bin/false"""
config.push('reset fail', reset_fail_config)
self.addCleanup(config.pop, 'reset fail')
d = self.resumeSlaveHost(MockBuilder(virtualized=True, vm_host="pop"))
return assert_fails_with(d, CannotResumeHost)
def test_verifyBuildRequest_no_chroot(self):
# verifyBuildRequest raises when the DAS has no chroot.
job = self.makeJob()
builder = MockBuilder()
job.setBuilder(builder, OkSlave())
logger = BufferLogger()
e = self.assertRaises(CannotBuild, job.verifyBuildRequest, logger)
self.assertIn("Missing chroot", str(e))
def test_recover_idle_slave(self):
# An idle slave is not rescued, even if it's not meant to be
# idle. SlaveScanner.scan() will clean up the DB side, because
# we still report that it's lost.
slave = OkSlave()
lost = yield BuilderInteractor.rescueIfLost(
extract_vitals_from_db(MockBuilder()), slave, 'trivial')
self.assertTrue(lost)
self.assertEqual([], slave.call_log)
def test_verifyBuildRequest_bad_pocket(self):
# verifyBuildRequest will raise if a bad pocket is proposed.
build = self.factory.makeSourcePackageRecipeBuild(
pocket=PackagePublishingPocket.SECURITY)
job = IBuildFarmJobBehaviour(build)
job.setBuilder(MockBuilder("bob-de-bouwer"), OkSlave())
e = self.assertRaises(
AssertionError, job.verifyBuildRequest, BufferLogger())
self.assertIn('invalid pocket due to the series status of', str(e))
def test_virtual_1_1(self):
# Virtual builders using protocol 1.1 get reset, and once the
# trigger completes we're happy that it's clean.
builder = MockBuilder(virtualized=True,
clean_status=BuilderCleanStatus.DIRTY,
vm_host='lol',
vm_reset_protocol=BuilderResetProtocol.PROTO_1_1)
yield self.assertCleanCalls(builder, OkSlave(), ['resume', 'echo'],
True)
def test_verifyBuildRequest_valid(self):
# VerifyBuildRequest won't raise any exceptions when called with a
# valid builder set.
job = self.makeJob()
builder = MockBuilder("bob-de-bouwer")
job.setBuilder(builder, OkSlave())
logger = BufferLogger()
job.verifyBuildRequest(logger)
self.assertEquals("", logger.getLogBuffer())
def test_virtual_2_0_dirty(self):
# Virtual builders using protocol 2.0 get reset and set to
# CLEANING. It's then up to the non-Launchpad reset code to set
# the builder back to CLEAN using the webservice.
builder = MockBuilder(virtualized=True,
clean_status=BuilderCleanStatus.DIRTY,
vm_host='lol',
vm_reset_protocol=BuilderResetProtocol.PROTO_2_0)
yield self.assertCleanCalls(builder, OkSlave(), ['resume'], False)
self.assertEqual(BuilderCleanStatus.CLEANING, builder.clean_status)
def test_dispatchBuildToSlave(self):
behaviour = self.makeBehaviour(FakeDistroArchSeries())
builder = MockBuilder()
slave = OkSlave()
logger = BufferLogger()
behaviour.setBuilder(builder, slave)
yield behaviour.dispatchBuildToSlave(logger)
self.assertDispatched(slave, logger, 'chroot-fooix-bar-y86.tar.gz',
'chroot')
def test_virtual_2_0_cleaning(self):
# Virtual builders using protocol 2.0 only get touched when
# they're DIRTY. Once they're cleaning, they're not our problem
# until they return to CLEAN, so we ignore them.
builder = MockBuilder(virtualized=True,
clean_status=BuilderCleanStatus.CLEANING,
vm_host='lol',
vm_reset_protocol=BuilderResetProtocol.PROTO_2_0)
yield self.assertCleanCalls(builder, OkSlave(), [], False)
self.assertEqual(BuilderCleanStatus.CLEANING, builder.clean_status)
def test_recover_waiting_slave_with_bad_id(self):
# If a slave is WAITING with a build for us to get, and the build
# cookie cannot be verified, which means we don't recognize the build,
# then rescueBuilderIfLost should attempt to abort it, so that the
# builder is reset for a new build, and the corrupt build is
# discarded.
waiting_slave = WaitingSlave(build_id='non-trivial')
lost = yield BuilderInteractor.rescueIfLost(
extract_vitals_from_db(MockBuilder()), waiting_slave, 'trivial')
self.assertTrue(lost)
self.assertEqual(['status', 'clean'], waiting_slave.call_log)
def makeJob(self, recipe_registrant=None, recipe_owner=None,
archive=None, git=False, with_builder=False):
"""Create a sample `ISourcePackageRecipeBuild`."""
spn = self.factory.makeSourcePackageName("apackage")
if archive is None:
distro = self.factory.makeDistribution(name="distro")
else:
distro = archive.distribution
distroseries = self.factory.makeDistroSeries(
name="mydistro", distribution=distro)
processor = getUtility(IProcessorSet).getByName('386')
distroseries.nominatedarchindep = distroseries.newArch(
'i386', processor, True, self.factory.makePerson())
sourcepackage = self.factory.makeSourcePackage(spn, distroseries)
if recipe_registrant is None:
recipe_registrant = self.factory.makePerson(
email="*****@*****.**",
name="joe", displayname="Joe User")
if recipe_owner is None:
recipe_owner = recipe_registrant
if git:
[somebranch] = self.factory.makeGitRefs(
owner=recipe_owner, name="pkg",
target=self.factory.makeProduct("someapp"),
paths=["refs/heads/packaging"])
else:
somebranch = self.factory.makeBranch(
owner=recipe_owner, name="pkg",
product=self.factory.makeProduct("someapp"))
recipe = self.factory.makeSourcePackageRecipe(
recipe_registrant, recipe_owner, distroseries, "recept",
"Recipe description", branches=[somebranch])
spb = self.factory.makeSourcePackageRecipeBuild(
sourcepackage=sourcepackage, archive=archive,
recipe=recipe, requester=recipe_owner, distroseries=distroseries)
job = IBuildFarmJobBehaviour(spb)
if with_builder:
builder = MockBuilder()
builder.processor = processor
job.setBuilder(builder, None)
return job
def test_resetOrFail_resume_failure(self):
reset_fail_config = """
[builddmaster]
vm_resume_command: /bin/false"""
config.push('reset fail', reset_fail_config)
self.addCleanup(config.pop, 'reset fail')
builder = MockBuilder(virtualized=True, vm_host="pop", builderok=True)
vitals = extract_vitals_from_db(builder)
d = BuilderInteractor.resetOrFail(
vitals, BuilderInteractor.makeSlaveFromVitals(vitals), builder,
DevNullLogger(), Exception())
return assert_fails_with(d, CannotResumeHost)
def test_verifyBuildRequest_archive_disabled(self):
archive = self.factory.makeArchive(enabled=False,
displayname="Disabled Archive")
job = self.makeJob(archive=archive)
lfa = self.factory.makeLibraryFileAlias()
transaction.commit()
job.build.distro_arch_series.addOrUpdateChroot(lfa)
builder = MockBuilder()
job.setBuilder(builder, OkSlave())
logger = BufferLogger()
e = self.assertRaises(ArchiveDisabled, job.verifyBuildRequest, logger)
self.assertEqual("Disabled Archive is disabled.", str(e))
def test_verifyBuildRequest_valid(self):
# verifyBuildRequest doesn't raise any exceptions when called with a
# valid builder set.
job = self.makeJob()
lfa = self.factory.makeLibraryFileAlias()
transaction.commit()
job.build.distro_arch_series.addOrUpdateChroot(lfa)
builder = MockBuilder()
job.setBuilder(builder, OkSlave())
logger = BufferLogger()
job.verifyBuildRequest(logger)
self.assertEqual("", logger.getLogBuffer())
def test_verifyBuildRequest_archive_private_owners_match(self):
archive = self.factory.makeArchive(private=True)
job = self.makeJob(archive=archive,
registrant=archive.owner,
owner=archive.owner)
lfa = self.factory.makeLibraryFileAlias()
transaction.commit()
job.build.distro_arch_series.addOrUpdateChroot(lfa)
builder = MockBuilder()
job.setBuilder(builder, OkSlave())
logger = BufferLogger()
job.verifyBuildRequest(logger)
self.assertEqual("", logger.getLogBuffer())
def test_verifyBuildRequest_virtual_mismatch(self):
# verifyBuildRequest raises on an attempt to build a virtualized
# build on a non-virtual builder.
job = self.makeJob()
lfa = self.factory.makeLibraryFileAlias()
transaction.commit()
job.build.distro_arch_series.addOrUpdateChroot(lfa)
builder = MockBuilder(virtualized=False)
job.setBuilder(builder, OkSlave())
logger = BufferLogger()
e = self.assertRaises(AssertionError, job.verifyBuildRequest, logger)
self.assertEqual(
"Attempt to build virtual item on a non-virtual builder.", str(e))