def test_dont_dispatch_release_builds(self): archive = self.factory.makeArchive(purpose=ArchivePurpose.PRIMARY) builder = self.factory.makeBuilder() distroseries = self.factory.makeDistroSeries( status=SeriesStatus.CURRENT, distribution=archive.distribution) distro_arch_series = self.factory.makeDistroArchSeries( distroseries=distroseries) build = self.factory.makeBinaryPackageBuild( builder=builder, archive=archive, distroarchseries=distro_arch_series, pocket=PackagePublishingPocket.RELEASE) lf = self.factory.makeLibraryFileAlias() transaction.commit() build.distro_arch_series.addOrUpdateChroot(lf) candidate = build.queueBuild() behavior = IBuildFarmJobBehavior(candidate.specific_job) behavior.setBuilder(builder, None) e = self.assertRaises(AssertionError, behavior.verifyBuildRequest, BufferLogger()) expected_message = ( "%s (%s) can not be built for pocket %s: invalid pocket due " "to the series status of %s." % (build.title, build.id, build.pocket.name, build.distro_series.name)) self.assertEqual(expected_message, str(e))
def test_dont_dispatch_release_builds(self): archive = self.factory.makeArchive(purpose=ArchivePurpose.PRIMARY) builder = self.factory.makeBuilder() distroseries = self.factory.makeDistroSeries( status=SeriesStatus.CURRENT, distribution=archive.distribution) distro_arch_series = self.factory.makeDistroArchSeries( distroseries=distroseries) build = self.factory.makeBinaryPackageBuild( builder=builder, archive=archive, distroarchseries=distro_arch_series, pocket=PackagePublishingPocket.RELEASE) lf = self.factory.makeLibraryFileAlias() transaction.commit() build.distro_arch_series.addOrUpdateChroot(lf) candidate = build.queueBuild() behavior = IBuildFarmJobBehavior(candidate.specific_job) behavior.setBuilder(builder, None) e = self.assertRaises( AssertionError, behavior.verifyBuildRequest, BufferLogger()) expected_message = ( "%s (%s) can not be built for pocket %s: invalid pocket due " "to the series status of %s." % ( build.title, build.id, build.pocket.name, build.distro_series.name)) self.assertEqual(expected_message, str(e))
def test_verifyBuildRequest_bad_pocket(self): # verifyBuildRequest will raise if a bad pocket is proposed. build = self.factory.makeSourcePackageRecipeBuild(pocket=PackagePublishingPocket.SECURITY) job = self.factory.makeSourcePackageRecipeBuildJob(recipe_build=build) job = IBuildFarmJobBehavior(job.specific_job) job.setBuilder(MockBuilder("bob-de-bouwer"), OkSlave()) e = self.assertRaises(AssertionError, job.verifyBuildRequest, BufferLogger()) self.assertIn("invalid pocket due to the series status of", str(e))
def test_verifyBuildRequest_bad_pocket(self): # verifyBuildRequest will raise if a bad pocket is proposed. build = self.factory.makeSourcePackageRecipeBuild( pocket=PackagePublishingPocket.SECURITY) job = self.factory.makeSourcePackageRecipeBuildJob(recipe_build=build) job = IBuildFarmJobBehavior(job.specific_job) job.setBuilder(MockBuilder("bob-de-bouwer"), OkSlave()) e = self.assertRaises(AssertionError, job.verifyBuildRequest, BufferLogger()) self.assertIn('invalid pocket due to the series status of', str(e))
def test_verifyBuildRequest_no_chroot(self): # Don't dispatch a build when the DAS has no chroot. archive = self.factory.makeArchive(purpose=ArchivePurpose.PRIMARY) builder = self.factory.makeBuilder() build = self.factory.makeBinaryPackageBuild( builder=builder, archive=archive) candidate = build.queueBuild() behavior = IBuildFarmJobBehavior(candidate.specific_job) behavior.setBuilder(builder, None) e = self.assertRaises( CannotBuild, behavior.verifyBuildRequest, BufferLogger()) self.assertIn("Missing CHROOT", str(e))
def test_verifyBuildRequest_no_chroot(self): # Don't dispatch a build when the DAS has no chroot. archive = self.factory.makeArchive(purpose=ArchivePurpose.PRIMARY) builder = self.factory.makeBuilder() build = self.factory.makeBinaryPackageBuild(builder=builder, archive=archive) candidate = build.queueBuild() behavior = IBuildFarmJobBehavior(candidate.specific_job) behavior.setBuilder(builder, None) e = self.assertRaises(CannotBuild, behavior.verifyBuildRequest, BufferLogger()) self.assertIn("Missing CHROOT", str(e))
def test_cancelling_a_build(self): # When scanning an in-progress build, if its state is CANCELLING # then the build should be aborted, and eventually stopped and moved # to the CANCELLED state if it does not abort by itself. # Set up a mock building slave. slave = BuildingSlave() # Set the sample data builder building with the slave from above. builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] login('*****@*****.**') builder.builderok = True # For now, we can only cancel virtual builds. builder.virtualized = True builder.vm_host = "fake_vm_host" self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(slave)) transaction.commit() login(ANONYMOUS) buildqueue = builder.currentjob behavior = IBuildFarmJobBehavior(buildqueue.specific_job) slave.build_id = behavior.getBuildCookie() self.assertBuildingJob(buildqueue, builder) # Now set the build to CANCELLING. build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(buildqueue) build.updateStatus(BuildStatus.CANCELLING) # Run 'scan' and check its results. switch_dbuser(config.builddmaster.dbuser) clock = task.Clock() scanner = self._getScanner(clock=clock) yield scanner.scan() # An abort request should be sent. self.assertEqual(1, slave.call_log.count("abort")) self.assertEqual(BuildStatus.CANCELLING, build.status) # Advance time a little. Nothing much should happen. clock.advance(1) yield scanner.scan() self.assertEqual(1, slave.call_log.count("abort")) self.assertEqual(BuildStatus.CANCELLING, build.status) # Advance past the timeout. The build state should be cancelled and # we should have also called the resume() method on the slave that # resets the virtual machine. clock.advance(SlaveScanner.CANCEL_TIMEOUT) yield scanner.scan() self.assertEqual(1, slave.call_log.count("abort")) self.assertEqual(1, slave.call_log.count("resume")) self.assertEqual(BuildStatus.CANCELLED, build.status)
def test_log_file_collection(self): self.build.updateStatus(BuildStatus.FULLYBUILT) old_tmps = sorted(os.listdir('/tmp')) slave = WaitingSlave('BuildStatus.OK') def got_log(logfile_lfa_id): # Grabbing logs should not leave new files in /tmp (bug #172798) logfile_lfa = getUtility(ILibraryFileAliasSet)[logfile_lfa_id] new_tmps = sorted(os.listdir('/tmp')) self.assertEqual(old_tmps, new_tmps) # The new librarian file is stored compressed with a .gz # extension and text/plain file type for easy viewing in # browsers, as it decompresses and displays the file inline. self.assertTrue( logfile_lfa.filename.endswith('_FULLYBUILT.txt.gz')) self.assertEqual('text/plain', logfile_lfa.mimetype) self.layer.txn.commit() # LibrarianFileAlias does not implement tell() or seek(), which # are required by gzip.open(), so we need to read the file out # of the librarian first. fd, fname = tempfile.mkstemp() self.addCleanup(os.remove, fname) tmp = os.fdopen(fd, 'wb') tmp.write(logfile_lfa.read()) tmp.close() uncompressed_file = gzip.open(fname).read() # Now make a temp filename that getFile() can write to. fd, tmp_orig_file_name = tempfile.mkstemp() self.addCleanup(os.remove, tmp_orig_file_name) # Check that the original file from the slave matches the # uncompressed file in the librarian. def got_orig_log(ignored): orig_file_content = open(tmp_orig_file_name).read() self.assertEqual(orig_file_content, uncompressed_file) d = removeSecurityProxy(slave).getFile('buildlog', tmp_orig_file_name) return d.addCallback(got_orig_log) behavior = IBuildFarmJobBehavior(self.candidate.specific_job) behavior.setBuilder(self.builder, slave) d = behavior.getLogFromSlave(self.build.buildqueue_record) return d.addCallback(got_log)
def test_log_file_collection(self): self.build.updateStatus(BuildStatus.FULLYBUILT) old_tmps = sorted(os.listdir('/tmp')) slave = WaitingSlave('BuildStatus.OK') def got_log(logfile_lfa_id): # Grabbing logs should not leave new files in /tmp (bug #172798) logfile_lfa = getUtility(ILibraryFileAliasSet)[logfile_lfa_id] new_tmps = sorted(os.listdir('/tmp')) self.assertEqual(old_tmps, new_tmps) # The new librarian file is stored compressed with a .gz # extension and text/plain file type for easy viewing in # browsers, as it decompresses and displays the file inline. self.assertTrue( logfile_lfa.filename.endswith('_FULLYBUILT.txt.gz')) self.assertEqual('text/plain', logfile_lfa.mimetype) self.layer.txn.commit() # LibrarianFileAlias does not implement tell() or seek(), which # are required by gzip.open(), so we need to read the file out # of the librarian first. fd, fname = tempfile.mkstemp() self.addCleanup(os.remove, fname) tmp = os.fdopen(fd, 'wb') tmp.write(logfile_lfa.read()) tmp.close() uncompressed_file = gzip.open(fname).read() # Now make a temp filename that getFile() can write to. fd, tmp_orig_file_name = tempfile.mkstemp() self.addCleanup(os.remove, tmp_orig_file_name) # Check that the original file from the slave matches the # uncompressed file in the librarian. def got_orig_log(ignored): orig_file_content = open(tmp_orig_file_name).read() self.assertEqual(orig_file_content, uncompressed_file) d = removeSecurityProxy(slave).getFile( 'buildlog', tmp_orig_file_name) return d.addCallback(got_orig_log) behavior = IBuildFarmJobBehavior(self.candidate.specific_job) behavior.setBuilder(self.builder, slave) d = behavior.getLogFromSlave(self.build.buildqueue_record) return d.addCallback(got_log)
def findAndStartJob(cls, vitals, builder, slave): """Find a job to run and send it to the buildd slave. :return: A Deferred whose value is the `IBuildQueue` instance found or None if no job was found. """ logger = cls._getSlaveScannerLogger() # XXX This method should be removed in favour of two separately # called methods that find and dispatch the job. It will # require a lot of test fixing. candidate = builder.acquireBuildCandidate() if candidate is None: logger.debug("No build candidates available for builder.") defer.returnValue(None) new_behavior = cls.getBuildBehavior(candidate, builder, slave) needed_bfjb = type( removeSecurityProxy(IBuildFarmJobBehavior(candidate.specific_job))) if not zope_isinstance(new_behavior, needed_bfjb): raise AssertionError( "Inappropriate IBuildFarmJobBehavior: %r is not a %r" % (new_behavior, needed_bfjb)) yield cls._startBuild(candidate, vitals, builder, slave, new_behavior, logger) defer.returnValue(candidate)
def makeBehavior(self, branch=None, use_fake_chroot=True): """Create a TranslationTemplatesBuildBehavior. Anything that might communicate with build slaves and such (which we can't really do here) is mocked up. """ specific_job = self.factory.makeTranslationTemplatesBuildJob( branch=branch) behavior = IBuildFarmJobBehavior(specific_job) slave = WaitingSlave() behavior.setBuilder(self.factory.makeBuilder(), slave) if use_fake_chroot: lf = self.factory.makeLibraryFileAlias() self.layer.txn.commit() behavior._getChroot = lambda: lf return behavior
def test_dont_dispatch_security_builds(self): archive = self.factory.makeArchive(purpose=ArchivePurpose.PRIMARY) builder = self.factory.makeBuilder() build = self.factory.makeBinaryPackageBuild( builder=builder, archive=archive, pocket=PackagePublishingPocket.SECURITY) lf = self.factory.makeLibraryFileAlias() transaction.commit() build.distro_arch_series.addOrUpdateChroot(lf) candidate = build.queueBuild() behavior = IBuildFarmJobBehavior(candidate.specific_job) behavior.setBuilder(builder, None) e = self.assertRaises(AssertionError, behavior.verifyBuildRequest, BufferLogger()) self.assertEqual( 'Soyuz is not yet capable of building SECURITY uploads.', str(e))
def test_dont_dispatch_security_builds(self): archive = self.factory.makeArchive(purpose=ArchivePurpose.PRIMARY) builder = self.factory.makeBuilder() build = self.factory.makeBinaryPackageBuild( builder=builder, archive=archive, pocket=PackagePublishingPocket.SECURITY) lf = self.factory.makeLibraryFileAlias() transaction.commit() build.distro_arch_series.addOrUpdateChroot(lf) candidate = build.queueBuild() behavior = IBuildFarmJobBehavior(candidate.specific_job) behavior.setBuilder(builder, None) e = self.assertRaises( AssertionError, behavior.verifyBuildRequest, BufferLogger()) self.assertEqual( 'Soyuz is not yet capable of building SECURITY uploads.', str(e))
def test_getBuildCookie(self): # A build cookie is made up of the job type and record id. # The uploadprocessor relies on this format. build = self.factory.makeBinaryPackageBuild() candidate = build.queueBuild() behavior = IBuildFarmJobBehavior(candidate.specific_job) cookie = removeSecurityProxy(behavior).getBuildCookie() expected_cookie = "PACKAGEBUILD-%d" % build.id self.assertEqual(expected_cookie, cookie)
def test_verifyBuildRequest(self): # Don't allow a virtual build on a non-virtual builder. archive = self.factory.makeArchive(purpose=ArchivePurpose.PPA) builder = self.factory.makeBuilder(virtualized=False) build = self.factory.makeBinaryPackageBuild( builder=builder, archive=archive, pocket=PackagePublishingPocket.RELEASE) lf = self.factory.makeLibraryFileAlias() transaction.commit() build.distro_arch_series.addOrUpdateChroot(lf) candidate = build.queueBuild() behavior = IBuildFarmJobBehavior(candidate.specific_job) behavior.setBuilder(builder, None) e = self.assertRaises(AssertionError, behavior.verifyBuildRequest, BufferLogger()) self.assertEqual( 'Attempt to build virtual item on a non-virtual builder.', str(e))
def test_getBuildCookie(self): # A build cookie is made up of the job type and record id. # The uploadprocessor relies on this format. build = self.factory.makeSourcePackageRecipeBuild() job = self.factory.makeSourcePackageRecipeBuildJob(recipe_build=build) job = IBuildFarmJobBehavior(job.specific_job) cookie = removeSecurityProxy(job).getBuildCookie() expected_cookie = "RECIPEBRANCHBUILD-%d" % build.id self.assertEquals(expected_cookie, cookie)
def test_verifyBuildRequest(self): # Don't allow a virtual build on a non-virtual builder. archive = self.factory.makeArchive(purpose=ArchivePurpose.PPA) builder = self.factory.makeBuilder(virtualized=False) build = self.factory.makeBinaryPackageBuild( builder=builder, archive=archive, pocket=PackagePublishingPocket.RELEASE) lf = self.factory.makeLibraryFileAlias() transaction.commit() build.distro_arch_series.addOrUpdateChroot(lf) candidate = build.queueBuild() behavior = IBuildFarmJobBehavior(candidate.specific_job) behavior.setBuilder(builder, None) e = self.assertRaises( AssertionError, behavior.verifyBuildRequest, BufferLogger()) self.assertEqual( 'Attempt to build virtual item on a non-virtual builder.', str(e))
def makeExpectedInteraction(self, builder, build, chroot, archive, archive_purpose, component=None, extra_urls=None, filemap_names=None): """Build the log of calls that we expect to be made to the slave. :param builder: The builder we are using to build the binary package. :param build: The build being done on the builder. :param chroot: The `LibraryFileAlias` for the chroot in which we are building. :param archive: The `IArchive` into which we are building. :param archive_purpose: The ArchivePurpose we are sending to the builder. We specify this separately from the archive because sometimes the behavior object has to give a different purpose in order to trick the slave into building correctly. :return: A list of the calls we expect to be made. """ cookie = IBuildFarmJobBehavior( build.buildqueue_record.specific_job).getBuildCookie() ds_name = build.distro_arch_series.distroseries.name suite = ds_name + pocketsuffix[build.pocket] archives = get_sources_list_for_building( build, build.distro_arch_series, build.source_package_release.name) arch_indep = build.distro_arch_series.isNominatedArchIndep if component is None: component = build.current_component.name if filemap_names is None: filemap_names = [] if extra_urls is None: extra_urls = [] upload_logs = [('ensurepresent', url, '', '') for url in [chroot.http_url] + extra_urls] extra_args = { 'arch_indep': arch_indep, 'arch_tag': build.distro_arch_series.architecturetag, 'archive_private': archive.private, 'archive_purpose': archive_purpose.name, 'archives': archives, 'build_debug_symbols': archive.build_debug_symbols, 'ogrecomponent': component, 'suite': suite, } build_log = [('build', cookie, 'binarypackage', chroot.content.sha1, filemap_names, extra_args)] if builder.virtualized: result = [('echo', 'ping')] + upload_logs + build_log else: result = upload_logs + build_log return result
class TestGetUploadMethodsMixin: """Tests for `IPackageBuild` that need objects from the rest of LP.""" layer = LaunchpadZopelessLayer def makeBuild(self): """Allow classes to override the build with which the test runs.""" raise NotImplemented def setUp(self): super(TestGetUploadMethodsMixin, self).setUp() self.build = self.makeBuild() self.behavior = IBuildFarmJobBehavior( self.build.buildqueue_record.specific_job) def test_getUploadDirLeafCookie_parseable(self): # getUploadDirLeaf should return a directory name # that is parseable by the upload processor. upload_leaf = self.behavior.getUploadDirLeaf( self.behavior.getBuildCookie()) (job_type, job_id) = parse_build_upload_leaf_name(upload_leaf) self.assertEqual((self.build.job_type.name, self.build.id), (job_type, job_id))
class TestGetUploadMethodsMixin: """Tests for `IPackageBuild` that need objects from the rest of LP.""" layer = LaunchpadZopelessLayer def makeBuild(self): """Allow classes to override the build with which the test runs.""" raise NotImplemented def setUp(self): super(TestGetUploadMethodsMixin, self).setUp() self.build = self.makeBuild() self.behavior = IBuildFarmJobBehavior( self.build.buildqueue_record.specific_job) def test_getUploadDirLeafCookie_parseable(self): # getUploadDirLeaf should return a directory name # that is parseable by the upload processor. upload_leaf = self.behavior.getUploadDirLeaf( self.behavior.getBuildCookie()) (job_type, job_id) = parse_build_upload_leaf_name(upload_leaf) self.assertEqual( (self.build.job_type.name, self.build.id), (job_type, job_id))
def makeJob(self, recipe_registrant=None, recipe_owner=None, archive=None): """Create a sample `ISourcePackageRecipeBuildJob`.""" spn = self.factory.makeSourcePackageName("apackage") distro = self.factory.makeDistribution(name="distro") distroseries = self.factory.makeDistroSeries(name="mydistro", distribution=distro) processor = getUtility(IProcessorSet).getByName('386') distroseries.newArch('i386', processor, True, self.factory.makePerson()) sourcepackage = self.factory.makeSourcePackage(spn, distroseries) if recipe_registrant is None: recipe_registrant = self.factory.makePerson( email="*****@*****.**", name="joe", displayname="Joe User") if recipe_owner is None: recipe_owner = recipe_registrant somebranch = self.factory.makeBranch( owner=recipe_owner, name="pkg", product=self.factory.makeProduct("someapp")) recipe = self.factory.makeSourcePackageRecipe(recipe_registrant, recipe_owner, distroseries, u"recept", u"Recipe description", branches=[somebranch]) spb = self.factory.makeSourcePackageRecipeBuild( sourcepackage=sourcepackage, archive=archive, recipe=recipe, requester=recipe_owner, distroseries=distroseries) job = spb.makeJob() job_id = removeSecurityProxy(job.job).id BuildQueue(job_type=BuildFarmJobType.RECIPEBRANCHBUILD, job=job_id) job = IBuildFarmJobBehavior(job) return job
def setUp(self): super(TestGetUploadMethodsMixin, self).setUp() self.build = self.makeBuild() self.behavior = IBuildFarmJobBehavior( self.build.buildqueue_record.specific_job)
def getBuildBehavior(queue_item, builder, slave): if queue_item is None: return None behavior = IBuildFarmJobBehavior(queue_item.specific_job) behavior.setBuilder(builder, slave) return behavior
def test_adapts_ISourcePackageRecipeBuildJob(self): # IBuildFarmJobBehavior adapts a ISourcePackageRecipeBuildJob job = self.factory.makeSourcePackageRecipeBuild().makeJob() job = IBuildFarmJobBehavior(job) self.assertProvides(job, IBuildFarmJobBehavior)