def test_extraBuildArgs(self):
# _extraBuildArgs will return a sane set of additional arguments
self._setBuilderConfig()
job = self.makeJob()
distroarchseries = job.build.distroseries.architectures[0]
expected_archives = get_sources_list_for_building(
job.build, distroarchseries, None)
expected_archives.append("deb http://foo %s main" %
job.build.distroseries.name)
self.assertEqual(
{
'archive_private':
False,
'arch_tag':
'i386',
'author_email':
u'*****@*****.**',
'suite':
u'mydistro',
'author_name':
u'Joe User',
'archive_purpose':
'PPA',
'ogrecomponent':
'universe',
'recipe_text':
'# bzr-builder format 0.3 deb-version {debupstream}-0~{revno}\n'
'lp://dev/~joe/someapp/pkg\n',
'archives':
expected_archives,
'distroseries_name':
job.build.distroseries.name,
}, job._extraBuildArgs(distroarchseries))
def test_extraBuildArgs(self):
# extraBuildArgs returns a reasonable set of additional arguments.
job = self.makeJob(date_created=datetime(2014,
4,
25,
10,
38,
0,
tzinfo=pytz.UTC),
metadata={
"project": "distro",
"subproject": "special"
},
with_builder=True)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(job.build,
job.build.distro_arch_series,
None))
extra_args = yield job.extraBuildArgs()
self.assertEqual(
{
"archive_private": False,
"archives": expected_archives,
"arch_tag": "i386",
"build_url": canonical_url(job.build),
"datestamp": "20140425-103800",
"fast_cleanup": True,
"pocket": "release",
"project": "distro",
"subproject": "special",
"series": "unstable",
"trusted_keys": expected_trusted_keys,
}, extra_args)
def makeExpectedInteraction(self,
builder,
build,
chroot,
archive,
archive_purpose,
component=None,
extra_uploads=None,
filemap_names=None):
"""Build the log of calls that we expect to be made to the slave.
:param builder: The builder we are using to build the binary package.
:param build: The build being done on the builder.
:param chroot: The `LibraryFileAlias` for the chroot in which we are
building.
:param archive: The `IArchive` into which we are building.
:param archive_purpose: The ArchivePurpose we are sending to the
builder. We specify this separately from the archive because
sometimes the behaviour object has to give a different purpose
in order to trick the slave into building correctly.
:return: A list of the calls we expect to be made.
"""
das = build.distro_arch_series
ds_name = das.distroseries.name
suite = ds_name + pocketsuffix[build.pocket]
archives, trusted_keys = yield get_sources_list_for_building(
build, das, build.source_package_release.name)
arch_indep = das.isNominatedArchIndep
if component is None:
component = build.current_component.name
if filemap_names is None:
filemap_names = []
if extra_uploads is None:
extra_uploads = []
upload_logs = [
('ensurepresent', ) + upload
for upload in [(chroot.http_url, '', '')] + extra_uploads
]
extra_args = {
'arch_indep': arch_indep,
'arch_tag': das.architecturetag,
'archive_private': archive.private,
'archive_purpose': archive_purpose.name,
'archives': archives,
'build_debug_symbols': archive.build_debug_symbols,
'build_url': canonical_url(build),
'distribution': das.distroseries.distribution.name,
'fast_cleanup': builder.virtualized,
'image_type': 'chroot',
'ogrecomponent': component,
'series': ds_name,
'suite': suite,
'trusted_keys': trusted_keys,
}
build_log = [('build', build.build_cookie, 'binarypackage',
chroot.content.sha1, filemap_names, extra_args)]
result = upload_logs + build_log
defer.returnValue(result)
def test_extraBuildArgs_withBadConfigForBzrBuilderPPA(self):
# Ensure _extraBuildArgs doesn't blow up with a badly formatted
# bzr_builder_sources_list in the config.
self.pushConfig("builddmaster", bzr_builder_sources_list="deb http://foo %(series) main")
# (note the missing 's' in %(series)
job = self.makeJob()
distroarchseries = job.build.distroseries.architectures[0]
expected_archives = get_sources_list_for_building(job.build, distroarchseries, None)
logger = BufferLogger()
self.assertEqual(
{
"archive_private": False,
"arch_tag": "i386",
"author_email": u"*****@*****.**",
"suite": u"mydistro",
"author_name": u"Joe User",
"archive_purpose": "PPA",
"ogrecomponent": "universe",
"recipe_text": "# bzr-builder format 0.3 deb-version {debupstream}-0~{revno}\n"
"lp://dev/~joe/someapp/pkg\n",
"archives": expected_archives,
"distroseries_name": job.build.distroseries.name,
},
job._extraBuildArgs(distroarchseries, logger),
)
self.assertIn("Exception processing bzr_builder_sources_list:", logger.getLogBuffer())
def test_extraBuildArgs_git(self):
job = self.makeJob(git=True, with_builder=True)
distroarchseries = job.build.distroseries.architectures[0]
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, distroarchseries, None))
extra_args = yield job.extraBuildArgs()
self.assertEqual({
'arch_tag': 'i386',
'archive_private': False,
'archive_purpose': 'PPA',
'archives': expected_archives,
'author_email': '*****@*****.**',
'author_name': 'Joe User',
'build_url': canonical_url(job.build),
'distroseries_name': job.build.distroseries.name,
'fast_cleanup': True,
'git': True,
'ogrecomponent': 'universe',
'recipe_text':
'# git-build-recipe format 0.4 deb-version '
'{debupstream}-0~{revtime}\n'
'lp:~joe/someapp/+git/pkg packaging\n',
'series': job.build.distroseries.name,
'suite': 'mydistro',
'trusted_keys': expected_trusted_keys,
}, extra_args)
def test_extraBuildArgs(self):
# extraBuildArgs will return a sane set of additional arguments.
self._setBuilderConfig()
job = self.makeJob(with_builder=True)
distroarchseries = job.build.distroseries.architectures[0]
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, distroarchseries, None))
expected_archives.insert(
0, "deb http://foo %s main" % job.build.distroseries.name)
args = yield job.extraBuildArgs()
self.assertEqual({
'arch_tag': 'i386',
'archive_private': False,
'archive_purpose': 'PPA',
'archives': expected_archives,
'author_email': '*****@*****.**',
'author_name': 'Joe User',
'build_url': canonical_url(job.build),
'distroseries_name': job.build.distroseries.name,
'fast_cleanup': True,
'ogrecomponent': 'universe',
'recipe_text':
'# bzr-builder format 0.3 '
'deb-version {debupstream}-0~{revno}\n'
'lp://dev/~joe/someapp/pkg\n',
'series': job.build.distroseries.name,
'suite': 'mydistro',
'trusted_keys': expected_trusted_keys,
}, args)
def test_extraBuildArgs_withNoBZrBuilderConfigSet(self):
# Ensure _extraBuildArgs doesn't blow up when
# bzr_builder_sources_list isn't set.
job = self.makeJob()
distroarchseries = job.build.distroseries.architectures[0]
args = job._extraBuildArgs(distroarchseries)
expected_archives = get_sources_list_for_building(job.build, distroarchseries, None)
self.assertEqual(args["archives"], expected_archives)
def test_extraBuildArgs_withNoBZrBuilderConfigSet(self):
# Ensure _extraBuildArgs doesn't blow up when
# bzr_builder_sources_list isn't set.
job = self.makeJob()
distroarchseries = job.build.distroseries.architectures[0]
args = job._extraBuildArgs(distroarchseries)
expected_archives = get_sources_list_for_building(
job.build, distroarchseries, None)
self.assertEqual(args["archives"], expected_archives)
def _extraBuildArgs(self, distroarchseries, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
# Build extra arguments.
args = {}
suite = self.build.distroseries.name
if self.build.pocket != PackagePublishingPocket.RELEASE:
suite += "-%s" % (self.build.pocket.name.lower())
args['suite'] = suite
args['arch_tag'] = distroarchseries.architecturetag
requester = self.build.requester
if requester.preferredemail is None:
# Use a constant, known, name and email.
args["author_name"] = 'Launchpad Package Builder'
args["author_email"] = config.canonical.noreply_from_address
else:
args["author_name"] = requester.displayname
# We have to remove the security proxy here b/c there's not a
# logged in entity, and anonymous email lookups aren't allowed.
# Don't keep the naked requester around though.
args["author_email"] = removeSecurityProxy(
requester).preferredemail.email
args["recipe_text"] = str(self.build.recipe.builder_recipe)
args['archive_purpose'] = self.build.archive.purpose.name
args["ogrecomponent"] = get_primary_current_component(
self.build.archive, self.build.distroseries, None)
args['archives'] = get_sources_list_for_building(
self.build, distroarchseries, None)
args['archive_private'] = self.build.archive.private
# config.builddmaster.bzr_builder_sources_list can contain a
# sources.list entry for an archive that will contain a
# bzr-builder package that needs to be used to build this
# recipe.
try:
extra_archive = config.builddmaster.bzr_builder_sources_list
except AttributeError:
extra_archive = None
if extra_archive is not None:
try:
sources_line = extra_archive % ({
'series':
self.build.distroseries.name
})
args['archives'].append(sources_line)
except StandardError:
# Someone messed up the config, don't add it.
if logger:
logger.error(
"Exception processing bzr_builder_sources_list:\n%s" %
traceback.format_exc())
args['distroseries_name'] = self.build.distroseries.name
return args
def makeExpectedInteraction(self,
builder,
build,
chroot,
archive,
archive_purpose,
component=None,
extra_urls=None,
filemap_names=None):
"""Build the log of calls that we expect to be made to the slave.
:param builder: The builder we are using to build the binary package.
:param build: The build being done on the builder.
:param chroot: The `LibraryFileAlias` for the chroot in which we are
building.
:param archive: The `IArchive` into which we are building.
:param archive_purpose: The ArchivePurpose we are sending to the
builder. We specify this separately from the archive because
sometimes the behavior object has to give a different purpose
in order to trick the slave into building correctly.
:return: A list of the calls we expect to be made.
"""
cookie = IBuildFarmJobBehavior(
build.buildqueue_record.specific_job).getBuildCookie()
ds_name = build.distro_arch_series.distroseries.name
suite = ds_name + pocketsuffix[build.pocket]
archives = get_sources_list_for_building(
build, build.distro_arch_series, build.source_package_release.name)
arch_indep = build.distro_arch_series.isNominatedArchIndep
if component is None:
component = build.current_component.name
if filemap_names is None:
filemap_names = []
if extra_urls is None:
extra_urls = []
upload_logs = [('ensurepresent', url, '', '')
for url in [chroot.http_url] + extra_urls]
extra_args = {
'arch_indep': arch_indep,
'arch_tag': build.distro_arch_series.architecturetag,
'archive_private': archive.private,
'archive_purpose': archive_purpose.name,
'archives': archives,
'build_debug_symbols': archive.build_debug_symbols,
'ogrecomponent': component,
'suite': suite,
}
build_log = [('build', cookie, 'binarypackage', chroot.content.sha1,
filemap_names, extra_args)]
if builder.virtualized:
result = [('echo', 'ping')] + upload_logs + build_log
else:
result = upload_logs + build_log
return result
def test_extraBuildArgs_channels_apt(self):
# {"snapcraft": "apt"} causes snapcraft to be installed from apt.
job = self.makeJob(channels={"snapcraft": "apt"})
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(job.build,
job.build.distro_arch_series,
None))
with dbuser(config.builddmaster.dbuser):
args = yield job.extraBuildArgs()
self.assertNotIn("channels", args)
def _extraBuildArgs(self, distroarchseries, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
# Build extra arguments.
args = {}
suite = self.build.distroseries.name
if self.build.pocket != PackagePublishingPocket.RELEASE:
suite += "-%s" % (self.build.pocket.name.lower())
args['suite'] = suite
args['arch_tag'] = distroarchseries.architecturetag
requester = self.build.requester
if requester.preferredemail is None:
# Use a constant, known, name and email.
args["author_name"] = 'Launchpad Package Builder'
args["author_email"] = config.canonical.noreply_from_address
else:
args["author_name"] = requester.displayname
# We have to remove the security proxy here b/c there's not a
# logged in entity, and anonymous email lookups aren't allowed.
# Don't keep the naked requester around though.
args["author_email"] = removeSecurityProxy(
requester).preferredemail.email
args["recipe_text"] = str(self.build.recipe.builder_recipe)
args['archive_purpose'] = self.build.archive.purpose.name
args["ogrecomponent"] = get_primary_current_component(
self.build.archive, self.build.distroseries,
None)
args['archives'] = get_sources_list_for_building(self.build,
distroarchseries, None)
args['archive_private'] = self.build.archive.private
# config.builddmaster.bzr_builder_sources_list can contain a
# sources.list entry for an archive that will contain a
# bzr-builder package that needs to be used to build this
# recipe.
try:
extra_archive = config.builddmaster.bzr_builder_sources_list
except AttributeError:
extra_archive = None
if extra_archive is not None:
try:
sources_line = extra_archive % (
{'series': self.build.distroseries.name})
args['archives'].append(sources_line)
except StandardError:
# Someone messed up the config, don't add it.
if logger:
logger.error(
"Exception processing bzr_builder_sources_list:\n%s"
% traceback.format_exc())
args['distroseries_name'] = self.build.distroseries.name
return args
def test_extraBuildArgs_withNoBzrBuilderConfigSet(self):
# Ensure extraBuildArgs doesn't blow up when
# bzr_builder_sources_list isn't set.
job = self.makeJob(with_builder=True)
distroarchseries = job.build.distroseries.architectures[0]
args = yield job.extraBuildArgs()
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, distroarchseries, None))
self.assertEqual(args["archives"], expected_archives)
self.assertEqual(args["trusted_keys"], expected_trusted_keys)
def test_extraBuildArgs_channels(self):
# If the build needs particular channels, extraBuildArgs sends them.
job = self.makeJob(channels={"snapcraft": "edge"})
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(job.build,
job.build.distro_arch_series,
None))
with dbuser(config.builddmaster.dbuser):
args = yield job.extraBuildArgs()
self.assertFalse(isProxy(args["channels"]))
self.assertEqual({"snapcraft": "edge"}, args["channels"])
def makeExpectedInteraction(self, builder, build, chroot, archive,
archive_purpose, component=None,
extra_urls=None, filemap_names=None):
"""Build the log of calls that we expect to be made to the slave.
:param builder: The builder we are using to build the binary package.
:param build: The build being done on the builder.
:param chroot: The `LibraryFileAlias` for the chroot in which we are
building.
:param archive: The `IArchive` into which we are building.
:param archive_purpose: The ArchivePurpose we are sending to the
builder. We specify this separately from the archive because
sometimes the behavior object has to give a different purpose
in order to trick the slave into building correctly.
:return: A list of the calls we expect to be made.
"""
cookie = IBuildFarmJobBehavior(
build.buildqueue_record.specific_job).getBuildCookie()
ds_name = build.distro_arch_series.distroseries.name
suite = ds_name + pocketsuffix[build.pocket]
archives = get_sources_list_for_building(
build, build.distro_arch_series,
build.source_package_release.name)
arch_indep = build.distro_arch_series.isNominatedArchIndep
if component is None:
component = build.current_component.name
if filemap_names is None:
filemap_names = []
if extra_urls is None:
extra_urls = []
upload_logs = [
('ensurepresent', url, '', '')
for url in [chroot.http_url] + extra_urls]
extra_args = {
'arch_indep': arch_indep,
'arch_tag': build.distro_arch_series.architecturetag,
'archive_private': archive.private,
'archive_purpose': archive_purpose.name,
'archives': archives,
'build_debug_symbols': archive.build_debug_symbols,
'ogrecomponent': component,
'suite': suite,
}
build_log = [
('build', cookie, 'binarypackage', chroot.content.sha1,
filemap_names, extra_args)]
if builder.virtualized:
result = [('echo', 'ping')] + upload_logs + build_log
else:
result = upload_logs + build_log
return result
def test_extraBuildArgs_channels_feature_flag_overridden(self):
# The snap.channels.snapcraft feature flag can be overridden by
# explicit configuration.
self.useFixture(
FeatureFixture({SNAP_SNAPCRAFT_CHANNEL_FEATURE_FLAG: "stable"}))
job = self.makeJob(channels={"snapcraft": "apt"})
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(job.build,
job.build.distro_arch_series,
None))
with dbuser(config.builddmaster.dbuser):
args = yield job.extraBuildArgs()
self.assertNotIn("channels", args)
def test_extraBuildArgs_channels_feature_flag_real_channel(self):
# If the snap.channels.snapcraft feature flag is set, it identifies
# the default channel to be used for snapcraft.
self.useFixture(
FeatureFixture({SNAP_SNAPCRAFT_CHANNEL_FEATURE_FLAG: "stable"}))
job = self.makeJob()
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(job.build,
job.build.distro_arch_series,
None))
with dbuser(config.builddmaster.dbuser):
args = yield job.extraBuildArgs()
self.assertFalse(isProxy(args["channels"]))
self.assertEqual({"snapcraft": "stable"}, args["channels"])
def assertSourcesListAndKeys(self, expected_sources_list,
expected_key_names, build, **kwargs):
expected_lines = []
for archive_or_prefix, suffixes in expected_sources_list:
if IArchive.providedBy(archive_or_prefix):
prefix = "deb %s " % archive_or_prefix.archive_url
else:
prefix = archive_or_prefix + " "
expected_lines.extend([prefix + suffix for suffix in suffixes])
sources_list, trusted_keys = yield get_sources_list_for_building(
build, build.distro_arch_series, build.source_package_release.name,
**kwargs)
self.assertEqual(expected_lines, sources_list)
key_matchers = [
Base64KeyMatches(self.fingerprints[key_name])
for key_name in expected_key_names]
self.assertThat(trusted_keys, MatchesSetwise(*key_matchers))
def test_extraBuildArgs_git_url(self):
# extraBuildArgs returns appropriate arguments if asked to build a
# job for a Git branch backed by a URL for an external repository.
url = "https://git.example.org/foo"
ref = self.factory.makeGitRefRemote(repository_url=url,
path="refs/heads/master")
job = self.makeJob(git_ref=ref)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(job.build,
job.build.distro_arch_series,
None))
with dbuser(config.builddmaster.dbuser):
args = yield job.extraBuildArgs()
self.assertThat(
args,
MatchesDict({
"archive_private":
Is(False),
"archives":
Equals(expected_archives),
"arch_tag":
Equals("i386"),
"build_source_tarball":
Is(False),
"build_url":
Equals(canonical_url(job.build)),
"fast_cleanup":
Is(True),
"git_repository":
Equals(url),
"git_path":
Equals("master"),
"name":
Equals("test-snap"),
"private":
Is(False),
"proxy_url":
self.getProxyURLMatcher(job),
"revocation_endpoint":
self.getRevocationEndpointMatcher(job),
"series":
Equals("unstable"),
"trusted_keys":
Equals(expected_trusted_keys),
}))
def extraBuildArgs(self, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
if self.distro_arch_series is None:
raise CannotBuild("Unable to find distroarchseries for %s in %s" %
(self._builder.processor.name,
self.build.distroseries.displayname))
# Build extra arguments.
args = yield super(RecipeBuildBehaviour,
self).extraBuildArgs(logger=logger)
args['suite'] = self.build.distroseries.getSuite(self.build.pocket)
requester = self.build.requester
if requester.preferredemail is None:
# Use a constant, known, name and email.
args["author_name"] = 'Launchpad Package Builder'
args["author_email"] = config.canonical.noreply_from_address
else:
args["author_name"] = requester.displayname
# We have to remove the security proxy here b/c there's not a
# logged in entity, and anonymous email lookups aren't allowed.
# Don't keep the naked requester around though.
args["author_email"] = removeSecurityProxy(
requester).preferredemail.email
args["recipe_text"] = self.build.recipe.getRecipeText(validate=True)
args['archive_purpose'] = self.build.archive.purpose.name
args["ogrecomponent"] = get_primary_current_component(
self.build.archive, self.build.distroseries, None).name
args['archives'], args['trusted_keys'] = (
yield get_sources_list_for_building(
self.build,
self.distro_arch_series,
None,
tools_source=config.builddmaster.bzr_builder_sources_list,
logger=logger))
# XXX cjwatson 2017-07-26: This duplicates "series", which is common
# to all build types; this name for it is deprecated and should be
# removed once launchpad-buildd no longer requires it.
args['distroseries_name'] = self.build.distroseries.name
if self.build.recipe.base_git_repository is not None:
args['git'] = True
defer.returnValue(args)
def test_extraBuildArgs_git_HEAD(self):
# extraBuildArgs returns appropriate arguments if asked to build a
# job for the default branch in a Launchpad-hosted Git repository.
[ref] = self.factory.makeGitRefs()
removeSecurityProxy(ref.repository)._default_branch = ref.path
job = self.makeJob(git_ref=ref.repository.getRefByPath("HEAD"))
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(job.build,
job.build.distro_arch_series,
None))
with dbuser(config.builddmaster.dbuser):
args = yield job.extraBuildArgs()
self.assertThat(
args,
MatchesDict({
"archive_private":
Is(False),
"archives":
Equals(expected_archives),
"arch_tag":
Equals("i386"),
"build_source_tarball":
Is(False),
"build_url":
Equals(canonical_url(job.build)),
"fast_cleanup":
Is(True),
"git_repository":
Equals(ref.repository.git_https_url),
"name":
Equals("test-snap"),
"private":
Is(False),
"proxy_url":
self.getProxyURLMatcher(job),
"revocation_endpoint":
self.getRevocationEndpointMatcher(job),
"series":
Equals("unstable"),
"trusted_keys":
Equals(expected_trusted_keys),
}))
def test_extraBuildArgs_bzr(self):
# extraBuildArgs returns appropriate arguments if asked to build a
# job for a Bazaar branch.
branch = self.factory.makeBranch()
job = self.makeJob(branch=branch)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(job.build,
job.build.distro_arch_series,
None))
with dbuser(config.builddmaster.dbuser):
args = yield job.extraBuildArgs()
self.assertThat(
args,
MatchesDict({
"archive_private":
Is(False),
"archives":
Equals(expected_archives),
"arch_tag":
Equals("i386"),
"branch":
Equals(branch.bzr_identity),
"build_source_tarball":
Is(False),
"build_url":
Equals(canonical_url(job.build)),
"fast_cleanup":
Is(True),
"name":
Equals("test-snap"),
"private":
Is(False),
"proxy_url":
self.getProxyURLMatcher(job),
"revocation_endpoint":
self.getRevocationEndpointMatcher(job),
"series":
Equals("unstable"),
"trusted_keys":
Equals(expected_trusted_keys),
}))
def extraBuildArgs(self, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
build = self.build
base_args = yield super(LiveFSBuildBehaviour, self).extraBuildArgs(
logger=logger)
# Non-trivial metadata values may have been security-wrapped, which
# is pointless here and just gets in the way of xmlrpclib
# serialisation.
args = dict(removeSecurityProxy(build.livefs.metadata))
if build.metadata_override is not None:
args.update(removeSecurityProxy(build.metadata_override))
# Everything else overrides anything in the metadata.
args.update(base_args)
args["pocket"] = build.pocket.name.lower()
args["datestamp"] = build.version
args["archives"], args["trusted_keys"] = (
yield get_sources_list_for_building(
build, build.distro_arch_series, None, logger=logger))
defer.returnValue(args)
def _extraBuildArgs(self, build):
"""
Return the extra arguments required by the slave for the given build.
"""
# Build extra arguments.
args = {}
# turn 'arch_indep' ON only if build is archindep or if
# the specific architecture is the nominatedarchindep for
# this distroseries (in case it requires any archindep source)
args['arch_indep'] = build.distro_arch_series.isNominatedArchIndep
suite = build.distro_arch_series.distroseries.name
if build.pocket != PackagePublishingPocket.RELEASE:
suite += "-%s" % (build.pocket.name.lower())
args['suite'] = suite
args['arch_tag'] = build.distro_arch_series.architecturetag
archive_purpose = build.archive.purpose
if (archive_purpose == ArchivePurpose.PPA and
not build.archive.require_virtualized):
# If we're building a non-virtual PPA, override the purpose
# to PRIMARY and use the primary component override.
# This ensures that the package mangling tools will run over
# the built packages.
args['archive_purpose'] = ArchivePurpose.PRIMARY.name
args["ogrecomponent"] = (
get_primary_current_component(build.archive,
build.distro_series, build.source_package_release.name))
else:
args['archive_purpose'] = archive_purpose.name
args["ogrecomponent"] = (
build.current_component.name)
args['archives'] = get_sources_list_for_building(build,
build.distro_arch_series, build.source_package_release.name)
args['archive_private'] = build.archive.private
args['build_debug_symbols'] = build.archive.build_debug_symbols
return args
def _extraBuildArgs(self, build):
"""
Return the extra arguments required by the slave for the given build.
"""
# Build extra arguments.
args = {}
# turn 'arch_indep' ON only if build is archindep or if
# the specific architecture is the nominatedarchindep for
# this distroseries (in case it requires any archindep source)
args['arch_indep'] = build.distro_arch_series.isNominatedArchIndep
suite = build.distro_arch_series.distroseries.name
if build.pocket != PackagePublishingPocket.RELEASE:
suite += "-%s" % (build.pocket.name.lower())
args['suite'] = suite
args['arch_tag'] = build.distro_arch_series.architecturetag
archive_purpose = build.archive.purpose
if (archive_purpose == ArchivePurpose.PPA
and not build.archive.require_virtualized):
# If we're building a non-virtual PPA, override the purpose
# to PRIMARY and use the primary component override.
# This ensures that the package mangling tools will run over
# the built packages.
args['archive_purpose'] = ArchivePurpose.PRIMARY.name
args["ogrecomponent"] = (get_primary_current_component(
build.archive, build.distro_series,
build.source_package_release.name))
else:
args['archive_purpose'] = archive_purpose.name
args["ogrecomponent"] = (build.current_component.name)
args['archives'] = get_sources_list_for_building(
build, build.distro_arch_series, build.source_package_release.name)
args['archive_private'] = build.archive.private
args['build_debug_symbols'] = build.archive.build_debug_symbols
return args
def test_extraBuildArgs_withBadConfigForBzrBuilderPPA(self):
# Ensure _extraBuildArgs doesn't blow up with a badly formatted
# bzr_builder_sources_list in the config.
self.pushConfig(
"builddmaster",
bzr_builder_sources_list="deb http://foo %(series) main")
# (note the missing 's' in %(series)
job = self.makeJob()
distroarchseries = job.build.distroseries.architectures[0]
expected_archives = get_sources_list_for_building(
job.build, distroarchseries, None)
logger = BufferLogger()
self.assertEqual(
{
'archive_private':
False,
'arch_tag':
'i386',
'author_email':
u'*****@*****.**',
'suite':
u'mydistro',
'author_name':
u'Joe User',
'archive_purpose':
'PPA',
'ogrecomponent':
'universe',
'recipe_text':
'# bzr-builder format 0.3 deb-version {debupstream}-0~{revno}\n'
'lp://dev/~joe/someapp/pkg\n',
'archives':
expected_archives,
'distroseries_name':
job.build.distroseries.name,
}, job._extraBuildArgs(distroarchseries, logger))
self.assertIn("Exception processing bzr_builder_sources_list:",
logger.getLogBuffer())
def extraBuildArgs(self, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
build = self.build
das = build.distro_arch_series
# Build extra arguments.
args = yield super(BinaryPackageBuildBehaviour,
self).extraBuildArgs(logger=logger)
args['arch_indep'] = build.arch_indep
args['distribution'] = das.distroseries.distribution.name
args['suite'] = das.distroseries.getSuite(build.pocket)
archive_purpose = build.archive.purpose
if (archive_purpose == ArchivePurpose.PPA
and not build.archive.require_virtualized):
# If we're building a non-virtual PPA, override the purpose
# to PRIMARY and use the primary component override.
# This ensures that the package mangling tools will run over
# the built packages.
args['archive_purpose'] = ArchivePurpose.PRIMARY.name
args["ogrecomponent"] = (get_primary_current_component(
build.archive, build.distro_series,
build.source_package_release.name)).name
else:
args['archive_purpose'] = archive_purpose.name
args["ogrecomponent"] = (build.current_component.name)
args['archives'], args['trusted_keys'] = (
yield
get_sources_list_for_building(build,
das,
build.source_package_release.name,
logger=logger))
args['build_debug_symbols'] = build.archive.build_debug_symbols
defer.returnValue(args)
def test_extraBuildArgs(self):
# _extraBuildArgs will return a sane set of additional arguments
self._setBuilderConfig()
job = self.makeJob()
distroarchseries = job.build.distroseries.architectures[0]
expected_archives = get_sources_list_for_building(job.build, distroarchseries, None)
expected_archives.append("deb http://foo %s main" % job.build.distroseries.name)
self.assertEqual(
{
"archive_private": False,
"arch_tag": "i386",
"author_email": u"*****@*****.**",
"suite": u"mydistro",
"author_name": u"Joe User",
"archive_purpose": "PPA",
"ogrecomponent": "universe",
"recipe_text": "# bzr-builder format 0.3 deb-version {debupstream}-0~{revno}\n"
"lp://dev/~joe/someapp/pkg\n",
"archives": expected_archives,
"distroseries_name": job.build.distroseries.name,
},
job._extraBuildArgs(distroarchseries),
)
def test_extraBuildArgs_withBadConfigForBzrBuilderPPA(self):
# Ensure extraBuildArgs doesn't blow up with a badly formatted
# bzr_builder_sources_list in the config.
self.pushConfig(
"builddmaster",
bzr_builder_sources_list="deb http://foo %(series) main")
# (note the missing 's' in %(series)
job = self.makeJob(with_builder=True)
distroarchseries = job.build.distroseries.architectures[0]
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(
job.build, distroarchseries, None))
logger = BufferLogger()
extra_args = yield job.extraBuildArgs(logger)
self.assertEqual({
'arch_tag': 'i386',
'archive_private': False,
'archive_purpose': 'PPA',
'archives': expected_archives,
'author_email': '*****@*****.**',
'author_name': 'Joe User',
'build_url': canonical_url(job.build),
'distroseries_name': job.build.distroseries.name,
'fast_cleanup': True,
'ogrecomponent': 'universe',
'recipe_text':
'# bzr-builder format 0.3 '
'deb-version {debupstream}-0~{revno}\n'
'lp://dev/~joe/someapp/pkg\n',
'series': job.build.distroseries.name,
'suite': 'mydistro',
'trusted_keys': expected_trusted_keys,
}, extra_args)
self.assertIn(
"Exception processing build tools sources.list entry:",
logger.getLogBuffer())
def extraBuildArgs(self, logger=None):
"""
Return the extra arguments required by the slave for the given build.
"""
build = self.build
args = yield super(SnapBuildBehaviour, self).extraBuildArgs(
logger=logger)
if config.snappy.builder_proxy_host and build.snap.allow_internet:
token = yield self._requestProxyToken()
args["proxy_url"] = (
"http://{username}:{password}@{host}:{port}".format(
username=token['username'],
password=token['secret'],
host=config.snappy.builder_proxy_host,
port=config.snappy.builder_proxy_port))
args["revocation_endpoint"] = (
"{endpoint}/{token}".format(
endpoint=config.snappy.builder_proxy_auth_api_endpoint,
token=token['username']))
args["name"] = build.snap.store_name or build.snap.name
# XXX cjwatson 2015-08-03: Allow tools_source to be overridden at
# some more fine-grained level.
args["archives"], args["trusted_keys"] = (
yield get_sources_list_for_building(
build, build.distro_arch_series, None,
tools_source=config.snappy.tools_source,
tools_fingerprint=config.snappy.tools_fingerprint,
logger=logger))
channels = build.channels or {}
if "snapcraft" not in channels:
channels["snapcraft"] = (
getFeatureFlag(SNAP_SNAPCRAFT_CHANNEL_FEATURE_FLAG) or "apt")
if channels.get("snapcraft") != "apt":
# We have to remove the security proxy that Zope applies to this
# dict, since otherwise we'll be unable to serialise it to
# XML-RPC.
args["channels"] = removeSecurityProxy(channels)
if build.snap.branch is not None:
args["branch"] = build.snap.branch.bzr_identity
elif build.snap.git_ref is not None:
if build.snap.git_ref.repository_url is not None:
args["git_repository"] = build.snap.git_ref.repository_url
elif build.snap.git_repository.private:
macaroon_raw = yield cancel_on_timeout(
self._authserver.callRemote(
"issueMacaroon", "snap-build", "SnapBuild", build.id),
config.builddmaster.authentication_timeout)
# XXX cjwatson 2019-03-07: This is ugly and needs
# refactoring once we support more general HTTPS
# authentication; see also comment in
# GitRepository.git_https_url.
split = urlsplit(build.snap.git_repository.getCodebrowseUrl())
netloc = ":%s@%s" % (macaroon_raw, split.hostname)
if split.port:
netloc += ":%s" % split.port
args["git_repository"] = urlunsplit([
split.scheme, netloc, split.path, "", ""])
else:
args["git_repository"] = (
build.snap.git_repository.git_https_url)
# "git clone -b" doesn't accept full ref names. If this becomes
# a problem then we could change launchpad-buildd to do "git
# clone" followed by "git checkout" instead.
if build.snap.git_path != u"HEAD":
args["git_path"] = build.snap.git_ref.name
else:
raise CannotBuild(
"Source branch/repository for ~%s/%s has been deleted." %
(build.snap.owner.name, build.snap.name))
args["build_source_tarball"] = build.snap.build_source_tarball
args["private"] = build.is_private
build_request = build.build_request
if build_request is not None:
args["build_request_id"] = build_request.id
# RFC3339 format for timestamp
# (matching snapd, SAS and snapcraft representation)
timestamp = format_as_rfc3339(build_request.date_requested)
args["build_request_timestamp"] = timestamp
defer.returnValue(args)
def test_extraBuildArgs_git_private(self):
# extraBuildArgs returns appropriate arguments if asked to build a
# job for a private Git branch.
self.useFixture(FeatureFixture({SNAP_PRIVATE_FEATURE_FLAG: "on"}))
self.useFixture(InProcessAuthServerFixture())
self.pushConfig("launchpad",
internal_macaroon_secret_key="some-secret")
[
ref
] = self.factory.makeGitRefs(information_type=InformationType.USERDATA)
job = self.makeJob(git_ref=ref, private=True)
expected_archives, expected_trusted_keys = (
yield get_sources_list_for_building(job.build,
job.build.distro_arch_series,
None))
args = yield job.extraBuildArgs()
split_browse_root = urlsplit(config.codehosting.git_browse_root)
self.assertThat(
args,
MatchesDict({
"archive_private":
Is(False),
"archives":
Equals(expected_archives),
"arch_tag":
Equals("i386"),
"build_source_tarball":
Is(False),
"build_url":
Equals(canonical_url(job.build)),
"fast_cleanup":
Is(True),
"git_repository":
AfterPreprocessing(
urlsplit,
MatchesStructure(
scheme=Equals(split_browse_root.scheme),
username=Equals(""),
password=AfterPreprocessing(
Macaroon.deserialize,
MatchesStructure(
location=Equals(
config.vhost.mainsite.hostname),
identifier=Equals("snap-build"),
caveats=MatchesListwise([
MatchesStructure.byEquality(
caveat_id="lp.snap-build %s" %
job.build.id),
]))),
hostname=Equals(split_browse_root.hostname),
port=Equals(split_browse_root.port))),
"git_path":
Equals(ref.name),
"name":
Equals("test-snap"),
"private":
Is(True),
"proxy_url":
self.getProxyURLMatcher(job),
"revocation_endpoint":
self.getRevocationEndpointMatcher(job),
"series":
Equals("unstable"),
"trusted_keys":
Equals(expected_trusted_keys),
}))
