def _requestProxyToken(self):
admin_username = config.snappy.builder_proxy_auth_api_admin_username
if not admin_username:
raise CannotBuild(
"builder_proxy_auth_api_admin_username is not configured.")
secret = config.snappy.builder_proxy_auth_api_admin_secret
if not secret:
raise CannotBuild(
"builder_proxy_auth_api_admin_secret is not configured.")
url = config.snappy.builder_proxy_auth_api_endpoint
if not secret:
raise CannotBuild(
"builder_proxy_auth_api_endpoint is not configured.")
timestamp = int(time.time())
proxy_username = '******'.format(
build_id=self.build.build_cookie,
timestamp=timestamp)
auth_string = '{}:{}'.format(admin_username, secret).strip()
auth_header = b'Basic ' + base64.b64encode(auth_string)
response = yield treq.post(
url, headers={'Authorization': auth_header},
json={'username': proxy_username},
reactor=self._slave.reactor,
pool=self._slave.pool)
response = yield check_status(response)
token = yield treq.json_content(response)
defer.returnValue(token)
def dispatchBuildToSlave(self, build_queue_id, logger):
"""See `IBuildFarmJobBehavior`."""
distroseries = self.build.distroseries
# Start the binary package build on the slave builder. First
# we send the chroot.
distroarchseries = distroseries.getDistroArchSeriesByProcessor(
self._builder.processor)
if distroarchseries is None:
raise CannotBuild("Unable to find distroarchseries for %s in %s" %
(self._builder.processor.name,
self.build.distroseries.displayname))
args = self._extraBuildArgs(distroarchseries, logger)
chroot = distroarchseries.getChroot()
if chroot is None:
raise CannotBuild("Unable to find a chroot for %s" %
distroarchseries.displayname)
logger.info("Sending chroot file for recipe build to %s" %
self._builder.name)
d = self._slave.cacheFile(logger, chroot)
def got_cache_file(ignored):
# Generate a string which can be used to cross-check when
# obtaining results so we know we are referring to the right
# database object in subsequent runs.
buildid = "%s-%s" % (self.build.id, build_queue_id)
cookie = self.getBuildCookie()
chroot_sha1 = chroot.content.sha1
logger.info("Initiating build %s on %s" %
(buildid, self._builder.url))
return self._slave.build(cookie, "sourcepackagerecipe",
chroot_sha1, {}, args)
def log_build_result((status, info)):
message = """%s (%s):
***** RESULT *****
%s
%s: %s
******************
""" % (
self._builder.name,
self._builder.url,
args,
status,
info,
)
logger.info(message)
return d.addCallback(got_cache_file).addCallback(log_build_result)
def dispatchBuildToSlave(self, logger):
"""See `IBuildFarmJobBehaviour`."""
cookie = self.build.build_cookie
logger.info("Preparing job %s (%s) on %s." %
(cookie, self.build.title, self._builder.url))
builder_type, das, pocket, files, args = yield (
self.composeBuildRequest(logger))
# First cache the chroot and any other files that the job needs.
pocket_chroot = None
for image_type in self.image_types:
pocket_chroot = das.getPocketChroot(pocket=pocket,
image_type=image_type)
if pocket_chroot is not None:
break
if pocket_chroot is None:
raise CannotBuild("Unable to find a chroot for %s" %
das.displayname)
chroot = pocket_chroot.chroot
args["image_type"] = pocket_chroot.image_type.name.lower()
filename_to_sha1 = {}
dl = []
dl.append(
self._slave.sendFileToSlave(logger=logger,
url=chroot.http_url,
sha1=chroot.content.sha1))
for filename, params in files.items():
filename_to_sha1[filename] = params['sha1']
dl.append(self._slave.sendFileToSlave(logger=logger, **params))
yield defer.gatherResults(dl)
combined_args = {
'builder_type': builder_type,
'chroot_sha1': chroot.content.sha1,
'filemap': filename_to_sha1,
'args': args
}
logger.info("Dispatching job %s (%s) to %s:\n%s" %
(cookie, self.build.title, self._builder.url,
sanitise_urls(repr(combined_args))))
(status, info) = yield self._slave.build(cookie, builder_type,
chroot.content.sha1,
filename_to_sha1, args)
logger.info(
"Job %s (%s) started on %s: %s %s" %
(cookie, self.build.title, self._builder.url, status, info))
def extraBuildArgs(self, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
if self.distro_arch_series is None:
raise CannotBuild("Unable to find distroarchseries for %s in %s" %
(self._builder.processor.name,
self.build.distroseries.displayname))
# Build extra arguments.
args = yield super(RecipeBuildBehaviour,
self).extraBuildArgs(logger=logger)
args['suite'] = self.build.distroseries.getSuite(self.build.pocket)
requester = self.build.requester
if requester.preferredemail is None:
# Use a constant, known, name and email.
args["author_name"] = 'Launchpad Package Builder'
args["author_email"] = config.canonical.noreply_from_address
else:
args["author_name"] = requester.displayname
# We have to remove the security proxy here b/c there's not a
# logged in entity, and anonymous email lookups aren't allowed.
# Don't keep the naked requester around though.
args["author_email"] = removeSecurityProxy(
requester).preferredemail.email
args["recipe_text"] = self.build.recipe.getRecipeText(validate=True)
args['archive_purpose'] = self.build.archive.purpose.name
args["ogrecomponent"] = get_primary_current_component(
self.build.archive, self.build.distroseries, None).name
args['archives'], args['trusted_keys'] = (
yield get_sources_list_for_building(
self.build,
self.distro_arch_series,
None,
tools_source=config.builddmaster.bzr_builder_sources_list,
logger=logger))
# XXX cjwatson 2017-07-26: This duplicates "series", which is common
# to all build types; this name for it is deprecated and should be
# removed once launchpad-buildd no longer requires it.
args['distroseries_name'] = self.build.distroseries.name
if self.build.recipe.base_git_repository is not None:
args['git'] = True
defer.returnValue(args)
def verifyBuildRequest(self, logger):
"""Assert some pre-build checks.
The build request is checked:
* Virtualized builds can't build on a non-virtual builder
* Ensure that we have a chroot
* Ensure that the build pocket allows builds for the current
distroseries state.
"""
build = self.build
if build.archive.require_virtualized and not self._builder.virtualized:
raise AssertionError(
"Attempt to build virtual archive on a non-virtual builder.")
# Assert that we are not silently building SECURITY jobs.
# See findBuildCandidates. Once we start building SECURITY
# correctly from EMBARGOED archive this assertion can be removed.
# XXX Julian 2007-12-18 spec=security-in-soyuz: This is being
# addressed in the work on the blueprint:
# https://blueprints.launchpad.net/soyuz/+spec/security-in-soyuz
target_pocket = build.pocket
assert target_pocket != PackagePublishingPocket.SECURITY, (
"Soyuz is not yet capable of building SECURITY uploads.")
# Ensure build has the needed chroot
chroot = build.distro_arch_series.getChroot(pocket=build.pocket)
if chroot is None:
raise CannotBuild("Missing CHROOT for %s/%s/%s" %
(build.distro_series.distribution.name,
build.distro_series.name,
build.distro_arch_series.architecturetag))
# This should already have been checked earlier, but just check again
# here in case of programmer errors.
reason = build.archive.checkUploadToPocket(build.distro_series,
build.pocket)
assert reason is None, (
"%s (%s) can not be built for pocket %s: invalid pocket due "
"to the series status of %s." %
(build.title, build.id, build.pocket.name,
build.distro_series.name))
def dispatchBuildToSlave(self, build_queue_item, logger):
"""See `IBuildFarmJobBehavior`."""
chroot = self._getChroot()
if chroot is None:
distroarchseries = self._getDistroArchSeries()
raise CannotBuild("Unable to find a chroot for %s" %
distroarchseries.displayname)
chroot_sha1 = chroot.content.sha1
d = self._slave.cacheFile(logger, chroot)
def got_cache_file(ignored):
args = {
'arch_tag': self._getDistroArchSeries().architecturetag,
'branch_url': self.buildfarmjob.branch.composePublicURL(),
}
filemap = {}
return self._slave.build(
self.getBuildCookie(), self.build_type, chroot_sha1, filemap,
args)
return d.addCallback(got_cache_file)
def verifyBuildRequest(self, logger):
"""Assert some pre-build checks.
The build request is checked:
* Virtualized builds can't build on a non-virtual builder
* The source archive may not be disabled
* If the source archive is private, the snap owner must match the
archive owner (see `SnapBuildArchiveOwnerMismatch` docstring)
* Ensure that we have a chroot
"""
build = self.build
if build.virtualized and not self._builder.virtualized:
raise AssertionError(
"Attempt to build virtual item on a non-virtual builder.")
if not build.archive.enabled:
raise ArchiveDisabled(build.archive.displayname)
if build.archive.private and build.snap.owner != build.archive.owner:
raise SnapBuildArchiveOwnerMismatch()
chroot = build.distro_arch_series.getChroot(pocket=build.pocket)
if chroot is None:
raise CannotBuild(
"Missing chroot for %s" % build.distro_arch_series.displayname)
def extraBuildArgs(self, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
build = self.build
args = yield super(SnapBuildBehaviour, self).extraBuildArgs(
logger=logger)
if config.snappy.builder_proxy_host and build.snap.allow_internet:
token = yield self._requestProxyToken()
args["proxy_url"] = (
"http://{username}:{password}@{host}:{port}".format(
username=token['username'],
password=token['secret'],
host=config.snappy.builder_proxy_host,
port=config.snappy.builder_proxy_port))
args["revocation_endpoint"] = (
"{endpoint}/{token}".format(
endpoint=config.snappy.builder_proxy_auth_api_endpoint,
token=token['username']))
args["name"] = build.snap.store_name or build.snap.name
# XXX cjwatson 2015-08-03: Allow tools_source to be overridden at
# some more fine-grained level.
args["archives"], args["trusted_keys"] = (
yield get_sources_list_for_building(
build, build.distro_arch_series, None,
tools_source=config.snappy.tools_source,
tools_fingerprint=config.snappy.tools_fingerprint,
logger=logger))
channels = build.channels or {}
if "snapcraft" not in channels:
channels["snapcraft"] = (
getFeatureFlag(SNAP_SNAPCRAFT_CHANNEL_FEATURE_FLAG) or "apt")
if channels.get("snapcraft") != "apt":
# We have to remove the security proxy that Zope applies to this
# dict, since otherwise we'll be unable to serialise it to
# XML-RPC.
args["channels"] = removeSecurityProxy(channels)
if build.snap.branch is not None:
args["branch"] = build.snap.branch.bzr_identity
elif build.snap.git_ref is not None:
if build.snap.git_ref.repository_url is not None:
args["git_repository"] = build.snap.git_ref.repository_url
elif build.snap.git_repository.private:
macaroon_raw = yield cancel_on_timeout(
self._authserver.callRemote(
"issueMacaroon", "snap-build", "SnapBuild", build.id),
config.builddmaster.authentication_timeout)
# XXX cjwatson 2019-03-07: This is ugly and needs
# refactoring once we support more general HTTPS
# authentication; see also comment in
# GitRepository.git_https_url.
split = urlsplit(build.snap.git_repository.getCodebrowseUrl())
netloc = ":%s@%s" % (macaroon_raw, split.hostname)
if split.port:
netloc += ":%s" % split.port
args["git_repository"] = urlunsplit([
split.scheme, netloc, split.path, "", ""])
else:
args["git_repository"] = (
build.snap.git_repository.git_https_url)
# "git clone -b" doesn't accept full ref names. If this becomes
# a problem then we could change launchpad-buildd to do "git
# clone" followed by "git checkout" instead.
if build.snap.git_path != u"HEAD":
args["git_path"] = build.snap.git_ref.name
else:
raise CannotBuild(
"Source branch/repository for ~%s/%s has been deleted." %
(build.snap.owner.name, build.snap.name))
args["build_source_tarball"] = build.snap.build_source_tarball
args["private"] = build.is_private
build_request = build.build_request
if build_request is not None:
args["build_request_id"] = build_request.id
# RFC3339 format for timestamp
# (matching snapd, SAS and snapcraft representation)
timestamp = format_as_rfc3339(build_request.date_requested)
args["build_request_timestamp"] = timestamp
defer.returnValue(args)