def _requestProxyToken(self): admin_username = config.snappy.builder_proxy_auth_api_admin_username if not admin_username: raise CannotBuild( "builder_proxy_auth_api_admin_username is not configured.") secret = config.snappy.builder_proxy_auth_api_admin_secret if not secret: raise CannotBuild( "builder_proxy_auth_api_admin_secret is not configured.") url = config.snappy.builder_proxy_auth_api_endpoint if not secret: raise CannotBuild( "builder_proxy_auth_api_endpoint is not configured.") timestamp = int(time.time()) proxy_username = '******'.format( build_id=self.build.build_cookie, timestamp=timestamp) auth_string = '{}:{}'.format(admin_username, secret).strip() auth_header = b'Basic ' + base64.b64encode(auth_string) response = yield treq.post( url, headers={'Authorization': auth_header}, json={'username': proxy_username}, reactor=self._slave.reactor, pool=self._slave.pool) response = yield check_status(response) token = yield treq.json_content(response) defer.returnValue(token)
def dispatchBuildToSlave(self, build_queue_id, logger): """See `IBuildFarmJobBehavior`.""" distroseries = self.build.distroseries # Start the binary package build on the slave builder. First # we send the chroot. distroarchseries = distroseries.getDistroArchSeriesByProcessor( self._builder.processor) if distroarchseries is None: raise CannotBuild("Unable to find distroarchseries for %s in %s" % (self._builder.processor.name, self.build.distroseries.displayname)) args = self._extraBuildArgs(distroarchseries, logger) chroot = distroarchseries.getChroot() if chroot is None: raise CannotBuild("Unable to find a chroot for %s" % distroarchseries.displayname) logger.info("Sending chroot file for recipe build to %s" % self._builder.name) d = self._slave.cacheFile(logger, chroot) def got_cache_file(ignored): # Generate a string which can be used to cross-check when # obtaining results so we know we are referring to the right # database object in subsequent runs. buildid = "%s-%s" % (self.build.id, build_queue_id) cookie = self.getBuildCookie() chroot_sha1 = chroot.content.sha1 logger.info("Initiating build %s on %s" % (buildid, self._builder.url)) return self._slave.build(cookie, "sourcepackagerecipe", chroot_sha1, {}, args) def log_build_result((status, info)): message = """%s (%s): ***** RESULT ***** %s %s: %s ****************** """ % ( self._builder.name, self._builder.url, args, status, info, ) logger.info(message) return d.addCallback(got_cache_file).addCallback(log_build_result)
def dispatchBuildToSlave(self, logger): """See `IBuildFarmJobBehaviour`.""" cookie = self.build.build_cookie logger.info("Preparing job %s (%s) on %s." % (cookie, self.build.title, self._builder.url)) builder_type, das, pocket, files, args = yield ( self.composeBuildRequest(logger)) # First cache the chroot and any other files that the job needs. pocket_chroot = None for image_type in self.image_types: pocket_chroot = das.getPocketChroot(pocket=pocket, image_type=image_type) if pocket_chroot is not None: break if pocket_chroot is None: raise CannotBuild("Unable to find a chroot for %s" % das.displayname) chroot = pocket_chroot.chroot args["image_type"] = pocket_chroot.image_type.name.lower() filename_to_sha1 = {} dl = [] dl.append( self._slave.sendFileToSlave(logger=logger, url=chroot.http_url, sha1=chroot.content.sha1)) for filename, params in files.items(): filename_to_sha1[filename] = params['sha1'] dl.append(self._slave.sendFileToSlave(logger=logger, **params)) yield defer.gatherResults(dl) combined_args = { 'builder_type': builder_type, 'chroot_sha1': chroot.content.sha1, 'filemap': filename_to_sha1, 'args': args } logger.info("Dispatching job %s (%s) to %s:\n%s" % (cookie, self.build.title, self._builder.url, sanitise_urls(repr(combined_args)))) (status, info) = yield self._slave.build(cookie, builder_type, chroot.content.sha1, filename_to_sha1, args) logger.info( "Job %s (%s) started on %s: %s %s" % (cookie, self.build.title, self._builder.url, status, info))
def extraBuildArgs(self, logger=None): """ Return the extra arguments required by the slave for the given build. """ if self.distro_arch_series is None: raise CannotBuild("Unable to find distroarchseries for %s in %s" % (self._builder.processor.name, self.build.distroseries.displayname)) # Build extra arguments. args = yield super(RecipeBuildBehaviour, self).extraBuildArgs(logger=logger) args['suite'] = self.build.distroseries.getSuite(self.build.pocket) requester = self.build.requester if requester.preferredemail is None: # Use a constant, known, name and email. args["author_name"] = 'Launchpad Package Builder' args["author_email"] = config.canonical.noreply_from_address else: args["author_name"] = requester.displayname # We have to remove the security proxy here b/c there's not a # logged in entity, and anonymous email lookups aren't allowed. # Don't keep the naked requester around though. args["author_email"] = removeSecurityProxy( requester).preferredemail.email args["recipe_text"] = self.build.recipe.getRecipeText(validate=True) args['archive_purpose'] = self.build.archive.purpose.name args["ogrecomponent"] = get_primary_current_component( self.build.archive, self.build.distroseries, None).name args['archives'], args['trusted_keys'] = ( yield get_sources_list_for_building( self.build, self.distro_arch_series, None, tools_source=config.builddmaster.bzr_builder_sources_list, logger=logger)) # XXX cjwatson 2017-07-26: This duplicates "series", which is common # to all build types; this name for it is deprecated and should be # removed once launchpad-buildd no longer requires it. args['distroseries_name'] = self.build.distroseries.name if self.build.recipe.base_git_repository is not None: args['git'] = True defer.returnValue(args)
def verifyBuildRequest(self, logger): """Assert some pre-build checks. The build request is checked: * Virtualized builds can't build on a non-virtual builder * Ensure that we have a chroot * Ensure that the build pocket allows builds for the current distroseries state. """ build = self.build if build.archive.require_virtualized and not self._builder.virtualized: raise AssertionError( "Attempt to build virtual archive on a non-virtual builder.") # Assert that we are not silently building SECURITY jobs. # See findBuildCandidates. Once we start building SECURITY # correctly from EMBARGOED archive this assertion can be removed. # XXX Julian 2007-12-18 spec=security-in-soyuz: This is being # addressed in the work on the blueprint: # https://blueprints.launchpad.net/soyuz/+spec/security-in-soyuz target_pocket = build.pocket assert target_pocket != PackagePublishingPocket.SECURITY, ( "Soyuz is not yet capable of building SECURITY uploads.") # Ensure build has the needed chroot chroot = build.distro_arch_series.getChroot(pocket=build.pocket) if chroot is None: raise CannotBuild("Missing CHROOT for %s/%s/%s" % (build.distro_series.distribution.name, build.distro_series.name, build.distro_arch_series.architecturetag)) # This should already have been checked earlier, but just check again # here in case of programmer errors. reason = build.archive.checkUploadToPocket(build.distro_series, build.pocket) assert reason is None, ( "%s (%s) can not be built for pocket %s: invalid pocket due " "to the series status of %s." % (build.title, build.id, build.pocket.name, build.distro_series.name))
def dispatchBuildToSlave(self, build_queue_item, logger): """See `IBuildFarmJobBehavior`.""" chroot = self._getChroot() if chroot is None: distroarchseries = self._getDistroArchSeries() raise CannotBuild("Unable to find a chroot for %s" % distroarchseries.displayname) chroot_sha1 = chroot.content.sha1 d = self._slave.cacheFile(logger, chroot) def got_cache_file(ignored): args = { 'arch_tag': self._getDistroArchSeries().architecturetag, 'branch_url': self.buildfarmjob.branch.composePublicURL(), } filemap = {} return self._slave.build( self.getBuildCookie(), self.build_type, chroot_sha1, filemap, args) return d.addCallback(got_cache_file)
def verifyBuildRequest(self, logger): """Assert some pre-build checks. The build request is checked: * Virtualized builds can't build on a non-virtual builder * The source archive may not be disabled * If the source archive is private, the snap owner must match the archive owner (see `SnapBuildArchiveOwnerMismatch` docstring) * Ensure that we have a chroot """ build = self.build if build.virtualized and not self._builder.virtualized: raise AssertionError( "Attempt to build virtual item on a non-virtual builder.") if not build.archive.enabled: raise ArchiveDisabled(build.archive.displayname) if build.archive.private and build.snap.owner != build.archive.owner: raise SnapBuildArchiveOwnerMismatch() chroot = build.distro_arch_series.getChroot(pocket=build.pocket) if chroot is None: raise CannotBuild( "Missing chroot for %s" % build.distro_arch_series.displayname)
def extraBuildArgs(self, logger=None): """ Return the extra arguments required by the slave for the given build. """ build = self.build args = yield super(SnapBuildBehaviour, self).extraBuildArgs( logger=logger) if config.snappy.builder_proxy_host and build.snap.allow_internet: token = yield self._requestProxyToken() args["proxy_url"] = ( "http://{username}:{password}@{host}:{port}".format( username=token['username'], password=token['secret'], host=config.snappy.builder_proxy_host, port=config.snappy.builder_proxy_port)) args["revocation_endpoint"] = ( "{endpoint}/{token}".format( endpoint=config.snappy.builder_proxy_auth_api_endpoint, token=token['username'])) args["name"] = build.snap.store_name or build.snap.name # XXX cjwatson 2015-08-03: Allow tools_source to be overridden at # some more fine-grained level. args["archives"], args["trusted_keys"] = ( yield get_sources_list_for_building( build, build.distro_arch_series, None, tools_source=config.snappy.tools_source, tools_fingerprint=config.snappy.tools_fingerprint, logger=logger)) channels = build.channels or {} if "snapcraft" not in channels: channels["snapcraft"] = ( getFeatureFlag(SNAP_SNAPCRAFT_CHANNEL_FEATURE_FLAG) or "apt") if channels.get("snapcraft") != "apt": # We have to remove the security proxy that Zope applies to this # dict, since otherwise we'll be unable to serialise it to # XML-RPC. args["channels"] = removeSecurityProxy(channels) if build.snap.branch is not None: args["branch"] = build.snap.branch.bzr_identity elif build.snap.git_ref is not None: if build.snap.git_ref.repository_url is not None: args["git_repository"] = build.snap.git_ref.repository_url elif build.snap.git_repository.private: macaroon_raw = yield cancel_on_timeout( self._authserver.callRemote( "issueMacaroon", "snap-build", "SnapBuild", build.id), config.builddmaster.authentication_timeout) # XXX cjwatson 2019-03-07: This is ugly and needs # refactoring once we support more general HTTPS # authentication; see also comment in # GitRepository.git_https_url. split = urlsplit(build.snap.git_repository.getCodebrowseUrl()) netloc = ":%s@%s" % (macaroon_raw, split.hostname) if split.port: netloc += ":%s" % split.port args["git_repository"] = urlunsplit([ split.scheme, netloc, split.path, "", ""]) else: args["git_repository"] = ( build.snap.git_repository.git_https_url) # "git clone -b" doesn't accept full ref names. If this becomes # a problem then we could change launchpad-buildd to do "git # clone" followed by "git checkout" instead. if build.snap.git_path != u"HEAD": args["git_path"] = build.snap.git_ref.name else: raise CannotBuild( "Source branch/repository for ~%s/%s has been deleted." % (build.snap.owner.name, build.snap.name)) args["build_source_tarball"] = build.snap.build_source_tarball args["private"] = build.is_private build_request = build.build_request if build_request is not None: args["build_request_id"] = build_request.id # RFC3339 format for timestamp # (matching snapd, SAS and snapcraft representation) timestamp = format_as_rfc3339(build_request.date_requested) args["build_request_timestamp"] = timestamp defer.returnValue(args)