def test_metaroot(self):
# The metadata directory structure doesn't include a distro
# name, and it's only use by Ubuntu Software Center, so only
# Ubuntu PPAs have a metaroot.
ubuntu_ppa = self.factory.makeArchive(purpose=ArchivePurpose.PPA)
test_ppa = self.factory.makeArchive(
distribution=self.ubuntutest, purpose=ArchivePurpose.PPA)
self.assertEqual(
"/var/tmp/ppa.test/%s/meta/%s" % (
ubuntu_ppa.owner.name, ubuntu_ppa.name),
getPubConfig(ubuntu_ppa).metaroot)
self.assertIs(None, getPubConfig(test_ppa).metaroot)
def test_primary_config_compat(self):
# Primary archive configuration is correct.
archiveroot = self.root + "/ubuntutest"
self.addCleanup(os.rmdir, archiveroot + "-uefi")
os.makedirs(archiveroot + "-uefi")
primary_config = getPubConfig(self.ubuntutest.main_archive)
self.assertEqual(archiveroot + "-uefi", primary_config.signingroot)
def getDeathRow(archive, log, pool_root_override):
"""Return a Deathrow object for the archive supplied.
:param archive: Use the publisher config for this archive to derive the
DeathRow object.
:param log: Use this logger for script debug logging.
:param pool_root_override: Use this pool root for the archive instead of
the one provided by the publishing-configuration, it will be only
used for PRIMARY archives.
"""
log.debug("Grab publisher config.")
pubconf = getPubConfig(archive)
if (pool_root_override is not None and
archive.purpose == ArchivePurpose.PRIMARY):
pool_root = pool_root_override
else:
pool_root = pubconf.poolroot
log.debug("Preparing on-disk pool representation.")
diskpool_log = logging.getLogger("DiskPool")
# Set the diskpool's log level to INFO to suppress debug output
diskpool_log.setLevel(20)
dp = DiskPool(pool_root, pubconf.temproot, diskpool_log)
log.debug("Preparing death row.")
return DeathRow(archive, dp, log)
def testReplaceUpdatedHtpasswd(self):
"""Test that the htpasswd file is only replaced if it changes."""
FILE_CONTENT = "Kneel before Zod!"
# The publisher Config object does not have an interface, so we
# need to remove the security wrapper.
pub_config = getPubConfig(self.ppa)
filename = os.path.join(pub_config.htaccessroot, ".htpasswd")
# Write out a dummy .htpasswd
ensure_directory_exists(pub_config.htaccessroot)
write_file(filename, FILE_CONTENT)
# Write the same contents in a temp file.
fd, temp_filename = tempfile.mkstemp(dir=pub_config.htaccessroot)
file = os.fdopen(fd, "w")
file.write(FILE_CONTENT)
file.close()
# Replacement should not happen.
script = self.getScript()
self.assertFalse(script.replaceUpdatedHtpasswd(self.ppa,
temp_filename))
# Writing a different .htpasswd should see it get replaced.
write_file(filename, "Come to me, son of Jor-El!")
self.assertTrue(script.replaceUpdatedHtpasswd(self.ppa, temp_filename))
os.remove(filename)
def test_private_ppa_config(self):
# Private PPA configuration uses the separate base location.
p3a = self.factory.makeArchive(owner=self.ppa.owner,
name="myprivateppa",
distribution=self.ubuntutest,
purpose=ArchivePurpose.PPA)
p3a.private = True
p3a.buildd_secret = "secret"
p3a_config = getPubConfig(p3a)
self.assertEqual(config.personalpackagearchive.private_root,
p3a_config.distroroot)
archiveroot = "%s/%s/%s/ubuntutest" % (p3a_config.distroroot,
p3a.owner.name, p3a.name)
self.assertEqual(archiveroot, p3a_config.archiveroot)
self.assertEqual(archiveroot + "/pool", p3a_config.poolroot)
self.assertEqual(archiveroot + "/dists", p3a_config.distsroot)
self.assertIsNone(p3a_config.overrideroot)
self.assertIsNone(p3a_config.cacheroot)
self.assertIsNone(p3a_config.miscroot)
self.assertIsNone(p3a_config.germinateroot)
self.assertEqual("/var/tmp/archive/ubuntutest-temp",
p3a_config.temproot)
# It's OK for the signing keys to be in the same location as for
# public PPAs, as the owner/name namespace is shared.
uefiroot = "/var/tmp/ppa-signing-keys.test/uefi/%s/%s" % (
p3a.owner.name, p3a.name)
self.assertEqual(uefiroot, p3a_config.uefiroot)
def setTargetDirectory(self, archive, tarfile_path, suite):
self.archive = archive
pubconf = getPubConfig(archive)
if pubconf.signingroot is None:
if self.logger is not None:
self.logger.warning(
"No signing root configured for this archive")
self.uefi_key = None
self.uefi_cert = None
self.kmod_pem = None
self.kmod_x509 = None
self.opal_pem = None
self.opal_x509 = None
self.autokey = False
else:
self.uefi_key = os.path.join(pubconf.signingroot, "uefi.key")
self.uefi_cert = os.path.join(pubconf.signingroot, "uefi.crt")
self.kmod_pem = os.path.join(pubconf.signingroot, "kmod.pem")
self.kmod_x509 = os.path.join(pubconf.signingroot, "kmod.x509")
self.opal_pem = os.path.join(pubconf.signingroot, "opal.pem")
self.opal_x509 = os.path.join(pubconf.signingroot, "opal.x509")
self.autokey = pubconf.signingautokey
self.setComponents(tarfile_path)
dists_signed = os.path.join(pubconf.archiveroot, "dists", suite,
"main", self.dists_directory)
self.targetdir = os.path.join(dists_signed,
"%s-%s" % (self.package, self.arch))
self.archiveroot = pubconf.archiveroot
self.temproot = pubconf.temproot
self.public_keys = set()
def getDeathRow(archive, log, pool_root_override):
"""Return a Deathrow object for the archive supplied.
:param archive: Use the publisher config for this archive to derive the
DeathRow object.
:param log: Use this logger for script debug logging.
:param pool_root_override: Use this pool root for the archive instead of
the one provided by the publishing-configuration, it will be only
used for PRIMARY archives.
"""
log.debug("Grab publisher config.")
pubconf = getPubConfig(archive)
if (pool_root_override is not None
and archive.purpose == ArchivePurpose.PRIMARY):
pool_root = pool_root_override
else:
pool_root = pubconf.poolroot
log.debug("Preparing on-disk pool representation.")
diskpool_log = logging.getLogger("DiskPool")
# Set the diskpool's log level to INFO to suppress debug output
diskpool_log.setLevel(20)
dp = DiskPool(pool_root, pubconf.temproot, diskpool_log)
log.debug("Preparing death row.")
return DeathRow(archive, dp, log)
def testGenerateHtpasswd(self):
"""Given some `ArchiveAuthToken`s, test generating htpasswd."""
# Make some subscriptions and tokens.
tokens = []
for name in ['name12', 'name16']:
person = getUtility(IPersonSet).getByName(name)
self.ppa.newSubscription(person, self.ppa.owner)
tokens.append(self.ppa.newAuthToken(person))
token_usernames = [token.person.name for token in tokens]
# Generate the passwd file.
script = self.getScript()
filename = script.generateHtpasswd(self.ppa)
self.addCleanup(remove_if_exists, filename)
# It should be a temp file in the same directory as the intended
# target file when it's renamed, so that os.rename() won't
# complain about renaming across file systems.
pub_config = getPubConfig(self.ppa)
self.assertEqual(
pub_config.htaccessroot, os.path.dirname(filename))
# Read it back in.
file_contents = [
line.strip().split(':', 1) for line in open(filename, 'r')]
# First entry is buildd secret, rest are from tokens.
usernames = list(zip(*file_contents)[0])
self.assertEqual(['buildd'] + token_usernames, usernames)
# We can re-encrypt the buildd_secret and it should match the
# one in the .htpasswd file.
password = file_contents[0][1]
encrypted_secret = crypt.crypt(self.ppa.buildd_secret, password)
self.assertEqual(encrypted_secret, password)
def replaceUpdatedHtpasswd(self, ppa, temp_htpasswd_file):
"""Compare the new and the old htpasswd and replace if changed.
:return: True if the file was replaced.
"""
try:
if self.options.dryrun:
return False
# The publisher Config object does not have an
# interface, so we need to remove the security wrapper.
pub_config = getPubConfig(ppa)
if not os.path.exists(pub_config.archiveroot):
os.makedirs(pub_config.archiveroot)
htpasswd_filename = os.path.join(pub_config.archiveroot,
".htpasswd")
if (not os.path.isfile(htpasswd_filename)
or not filecmp.cmp(htpasswd_filename, temp_htpasswd_file)):
# Atomically replace the old file or create a new file.
os.rename(temp_htpasswd_file, htpasswd_filename)
self.logger.debug("Replaced htpasswd for %s" % ppa.displayname)
return True
return False
finally:
if os.path.exists(temp_htpasswd_file):
os.unlink(temp_htpasswd_file)
def testGenerateHtpasswd(self):
"""Given some `ArchiveAuthToken`s, test generating htpasswd."""
# Make some subscriptions and tokens.
tokens = []
for name in ['name12', 'name16']:
person = getUtility(IPersonSet).getByName(name)
self.ppa.newSubscription(person, self.ppa.owner)
tokens.append(self.ppa.newAuthToken(person))
token_usernames = [token.person.name for token in tokens]
# Generate the passwd file.
script = self.getScript()
filename = script.generateHtpasswd(self.ppa)
self.addCleanup(remove_if_exists, filename)
# It should be a temp file on the same filesystem as the target
# file, so os.rename() won't explode. temproot is relied on
# elsewhere for this same purpose, so it should be safe.
pub_config = getPubConfig(self.ppa)
self.assertEqual(pub_config.temproot, os.path.dirname(filename))
# Read it back in.
file_contents = [
line.strip().split(':', 1) for line in open(filename, 'r')]
# First entry is buildd secret, rest are from tokens.
usernames = list(zip(*file_contents)[0])
self.assertEqual(['buildd'] + token_usernames, usernames)
# We can re-encrypt the buildd_secret and it should match the
# one in the .htpasswd file.
password = file_contents[0][1]
encrypted_secret = crypt.crypt(self.ppa.buildd_secret, password)
self.assertEqual(encrypted_secret, password)
def testReplaceUpdatedHtpasswd(self):
"""Test that the htpasswd file is only replaced if it changes."""
FILE_CONTENT = "Kneel before Zod!"
# The publisher Config object does not have an interface, so we
# need to remove the security wrapper.
pub_config = getPubConfig(self.ppa)
filename = os.path.join(pub_config.htaccessroot, ".htpasswd")
# Write out a dummy .htpasswd
ensure_directory_exists(pub_config.htaccessroot)
write_file(filename, FILE_CONTENT)
# Write the same contents in a temp file.
fd, temp_filename = tempfile.mkstemp(dir=pub_config.htaccessroot)
file = os.fdopen(fd, "w")
file.write(FILE_CONTENT)
file.close()
# Replacement should not happen.
script = self.getScript()
self.assertFalse(
script.replaceUpdatedHtpasswd(self.ppa, temp_filename))
# Writing a different .htpasswd should see it get replaced.
write_file(filename, "Come to me, son of Jor-El!")
self.assertTrue(
script.replaceUpdatedHtpasswd(self.ppa, temp_filename))
os.remove(filename)
def test_private_ppa_config(self):
# Private PPA configuration uses the separate base location.
p3a = self.factory.makeArchive(
owner=self.ppa.owner, name="myprivateppa",
distribution=self.ubuntutest, purpose=ArchivePurpose.PPA)
p3a.private = True
p3a.buildd_secret = "secret"
p3a_config = getPubConfig(p3a)
self.assertEqual(
config.personalpackagearchive.private_root, p3a_config.distroroot)
archiveroot = "%s/%s/%s/ubuntutest" % (
p3a_config.distroroot, p3a.owner.name, p3a.name)
self.assertEqual(archiveroot, p3a_config.archiveroot)
self.assertEqual(archiveroot + "/pool", p3a_config.poolroot)
self.assertEqual(archiveroot + "/dists", p3a_config.distsroot)
self.assertIsNone(p3a_config.overrideroot)
self.assertIsNone(p3a_config.cacheroot)
self.assertIsNone(p3a_config.miscroot)
self.assertIsNone(p3a_config.germinateroot)
self.assertEqual(
"/var/tmp/archive/ubuntutest-temp", p3a_config.temproot)
# It's OK for the signing keys to be in the same location as for
# public PPAs, as the owner/name namespace is shared.
uefiroot = "/var/tmp/ppa-signing-keys.test/uefi/%s/%s" % (
p3a.owner.name, p3a.name)
self.assertEqual(uefiroot, p3a_config.uefiroot)
def ensureNoFiles(self):
"""Ensure the .ht* files don't already exist."""
pub_config = getPubConfig(self.ppa)
htaccess = os.path.join(pub_config.archiveroot, ".htaccess")
htpasswd = os.path.join(pub_config.archiveroot, ".htpasswd")
remove_if_exists(htaccess)
remove_if_exists(htpasswd)
return htaccess, htpasswd
def ensureNoFiles(self):
"""Ensure the .ht* files don't already exist."""
pub_config = getPubConfig(self.ppa)
htaccess = os.path.join(pub_config.htaccessroot, ".htaccess")
htpasswd = os.path.join(pub_config.htaccessroot, ".htpasswd")
remove_if_exists(htaccess)
remove_if_exists(htpasswd)
return htaccess, htpasswd
def getInstallerPath(self, versioned_filename=None):
pubconf = getPubConfig(self.archive)
installer_path = os.path.join(pubconf.archiveroot, "dists", self.suite,
"main", "installer-%s" % self.arch)
if versioned_filename is not None:
installer_path = os.path.join(installer_path, self.version,
versioned_filename)
return installer_path
def setUp(self):
super(TestGetPubConfigPPACompatUefi, self).setUp()
self.ubuntutest = getUtility(IDistributionSet)['ubuntutest']
self.ppa = self.factory.makeArchive(
distribution=self.ubuntutest, purpose=ArchivePurpose.PPA)
signingroot = "/var/tmp/ppa-signing-keys.test/uefi"
self.addCleanup(os.rmdir, signingroot)
os.makedirs(signingroot)
self.ppa_config = getPubConfig(self.ppa)
def getConfig(self):
"""Set up a configuration object for this archive."""
archive = self.distribution.main_archive
if archive:
return getPubConfig(archive)
else:
raise LaunchpadScriptFailure(
"There is no PRIMARY archive for %s." %
self.options.distribution)
def test_getConfigs_maps_distro_and_purpose_to_matching_config(self):
distro = self.makeDistroWithPublishDirectory()
script = self.makeScript(distro)
script.setUp()
reference_config = getPubConfig(distro.main_archive)
config = script.getConfigs()[distro][ArchivePurpose.PRIMARY]
self.assertThat(
config, MatchesStructure.fromExample(
reference_config, 'temproot', 'distroroot', 'archiveroot'))
def getConfig(self):
"""Set up a configuration object for this archive."""
archive = self.distribution.main_archive
if archive:
return getPubConfig(archive)
else:
raise LaunchpadScriptFailure(
"There is no PRIMARY archive for %s." %
self.options.distribution)
def test_getConfigs_maps_distro_and_purpose_to_matching_config(self):
distro = self.makeDistroWithPublishDirectory()
script = self.makeScript(distro)
script.setUp()
reference_config = getPubConfig(distro.main_archive)
config = script.getConfigs()[distro][ArchivePurpose.PRIMARY]
self.assertThat(
config,
MatchesStructure.fromExample(reference_config, 'temproot',
'distroroot', 'archiveroot'))
def test_locateIndexesMarker_places_file_in_archive_root(self):
# The marker file for index creation is in the distribution's
# archive root.
series = self.factory.makeDistroSeries()
script = self.makeScript(series.distribution)
script.setUp()
archive_root = getPubConfig(series.main_archive).archiveroot
self.assertThat(
script.locateIndexesMarker(
series.distribution, get_a_suite(series)),
StartsWith(os.path.normpath(archive_root)))
def map_distro_pubconfigs(distro):
"""Return dict mapping archive purpose for distro's pub configs.
:param distro: `Distribution` to get publisher configs for.
:return: Dict mapping archive purposes to publisher configs, insofar as
they have publisher configs.
"""
candidates = [(archive.purpose, getPubConfig(archive))
for archive in get_publishable_archives(distro)]
return dict((purpose, config) for purpose, config in candidates
if config is not None)
def test_locateIndexesMarker_places_file_in_archive_root(self):
# The marker file for index creation is in the distribution's
# archive root.
series = self.factory.makeDistroSeries()
script = self.makeScript(series.distribution)
script.setUp()
archive_root = getPubConfig(series.main_archive).archiveroot
self.assertThat(
script.locateIndexesMarker(series.distribution,
get_a_suite(series)),
StartsWith(os.path.normpath(archive_root)))
def setUp(self):
"""Prepare configuration and filesystem state for the script's work.
This is idempotent: run it as often as you like. (For example,
a test may call `setUp` prior to calling `main` which again
invokes `setUp`).
"""
self.processOptions()
self.config = getPubConfig(self.distribution.main_archive)
self.content_archive = os.path.join(self.config.distroroot,
"contents-generation")
self.setUpContentArchive()
def setUp(self):
"""Prepare configuration and filesystem state for the script's work.
This is idempotent: run it as often as you like. (For example,
a test may call `setUp` prior to calling `main` which again
invokes `setUp`).
"""
self.processOptions()
self.config = getPubConfig(self.distribution.main_archive)
self.content_archive = os.path.join(
self.config.distroroot, "contents-generation")
self.setUpContentArchive()
def map_distro_pubconfigs(distro):
"""Return dict mapping archive purpose for distro's pub configs.
:param distro: `Distribution` to get publisher configs for.
:return: Dict mapping archive purposes to publisher configs, insofar as
they have publisher configs.
"""
candidates = [
(archive.purpose, getPubConfig(archive))
for archive in get_publishable_archives(distro)]
return dict(
(purpose, config)
for purpose, config in candidates if config is not None)
def test_archive_copy(self):
# If there is no key/cert configuration, processing succeeds but
# nothing is signed.
self.archive = self.factory.makeArchive(distribution=self.distro,
purpose=ArchivePurpose.COPY)
pubconf = getPubConfig(self.archive)
if not os.path.exists(pubconf.temproot):
os.makedirs(pubconf.temproot)
self.openArchive("test", "1.0", "amd64")
self.tarfile.add_file("1.0/empty.efi", b"")
self.tarfile.add_file("1.0/empty.ko", b"")
self.tarfile.add_file("1.0/empty.opal", b"")
upload = self.process_emulate()
self.assertContentEqual([], upload.callLog.caller_list())
def setUpPPA(self):
self.pushConfig("personalpackagearchive",
root=self.temp_dir,
signing_keys_root=self.temp_dir)
owner = self.factory.makePerson(name="signing-owner")
self.archive = self.factory.makeArchive(distribution=self.distro,
owner=owner,
name="testing",
purpose=ArchivePurpose.PPA)
self.signing_dir = os.path.join(self.temp_dir, "signing",
"signing-owner", "testing")
self.testcase_cn = '/CN=PPA signing-owner testing/'
pubconf = getPubConfig(self.archive)
if not os.path.exists(pubconf.temproot):
os.makedirs(pubconf.temproot)
def ensureHtaccess(self, ppa):
"""Generate a .htaccess for `ppa`."""
if self.options.dryrun:
return
# The publisher Config object does not have an
# interface, so we need to remove the security wrapper.
pub_config = getPubConfig(ppa)
htaccess_filename = os.path.join(pub_config.archiveroot, ".htaccess")
if not os.path.exists(htaccess_filename):
# It's not there, so create it.
if not os.path.exists(pub_config.archiveroot):
os.makedirs(pub_config.archiveroot)
write_htaccess(htaccess_filename, pub_config.archiveroot)
self.logger.debug("Created .htaccess for %s" % ppa.displayname)
def ensureHtaccess(self, ppa):
"""Generate a .htaccess for `ppa`."""
if self.options.dryrun:
return
# The publisher Config object does not have an
# interface, so we need to remove the security wrapper.
pub_config = getPubConfig(ppa)
htaccess_filename = os.path.join(pub_config.htaccessroot, ".htaccess")
if not os.path.exists(htaccess_filename):
# It's not there, so create it.
if not os.path.exists(pub_config.htaccessroot):
os.makedirs(pub_config.htaccessroot)
write_htaccess(htaccess_filename, pub_config.htaccessroot)
self.logger.debug("Created .htaccess for %s" % ppa.displayname)
def generateHtpasswd(self, ppa):
"""Generate a htpasswd file for `ppa`s `tokens`.
:param ppa: The context PPA (an `IArchive`).
:return: The filename of the htpasswd file that was generated.
"""
# Create a temporary file that will be a new .htpasswd.
pub_config = getPubConfig(ppa)
if not os.path.exists(pub_config.temproot):
os.makedirs(pub_config.temproot)
fd, temp_filename = tempfile.mkstemp(dir=pub_config.temproot)
os.close(fd)
write_htpasswd(temp_filename, htpasswd_credentials_for_archive(ppa))
return temp_filename
def testDirtySuites(self):
"""publish-distro can be told to publish specific suites."""
archive = self.factory.makeArchive(distribution=self.ubuntutest)
self.layer.txn.commit()
# publish-distro has nothing to publish.
self.runPublishDistro(['--ppa'])
breezy_release_path = os.path.join(
getPubConfig(archive).distsroot, 'breezy-autotest', 'Release')
self.assertThat(breezy_release_path, Not(PathExists()))
# ... but it will publish a suite anyway if it is marked as dirty.
archive.markSuiteDirty(
archive.distribution.getSeries('breezy-autotest'),
PackagePublishingPocket.RELEASE)
self.runPublishDistro(['--ppa'])
self.assertThat(breezy_release_path, PathExists())
def setUp(self):
super(TestSignableArchiveWithSigningKey, self).setUp()
self.temp_dir = self.makeTemporaryDirectory()
self.distro = self.factory.makeDistribution()
db_pubconf = getUtility(IPublisherConfigSet).getByDistribution(
self.distro)
db_pubconf.root_dir = unicode(self.temp_dir)
self.archive = self.factory.makeArchive(distribution=self.distro,
purpose=ArchivePurpose.PRIMARY)
self.archive_root = getPubConfig(self.archive).archiveroot
self.suite = "distroseries"
with InProcessKeyServerFixture() as keyserver:
yield keyserver.start()
key_path = os.path.join(gpgkeysdir, '*****@*****.**')
yield IArchiveSigningKey(self.archive).setSigningKey(
key_path, async_keyserver=True)
def test_primary_config(self):
# Primary archive configuration is correct.
primary_config = getPubConfig(self.ubuntutest.main_archive)
self.assertEqual(self.root, primary_config.distroroot)
archiveroot = self.root + "/ubuntutest"
self.assertEqual(archiveroot, primary_config.archiveroot)
self.assertEqual(archiveroot + "/pool", primary_config.poolroot)
self.assertEqual(archiveroot + "/dists", primary_config.distsroot)
self.assertEqual(
archiveroot + "-overrides", primary_config.overrideroot)
self.assertEqual(archiveroot + "-cache", primary_config.cacheroot)
self.assertEqual(archiveroot + "-misc", primary_config.miscroot)
self.assertEqual(
archiveroot + "-germinate", primary_config.germinateroot)
self.assertEqual(
self.root + "/ubuntutest-temp", primary_config.temproot)
self.assertEqual(archiveroot + "-uefi", primary_config.uefiroot)
def test_recoverWorkingDists_recovers_working_directory(self):
distro = self.makeDistroWithPublishDirectory()
script = self.makeScript(distro)
script.setUp()
script.logger = BufferLogger()
script.logger.setLevel(logging.INFO)
script.setUpDirs()
archive_config = getPubConfig(distro.main_archive)
backup_dists = os.path.join(archive_config.archiveroot + "-distscopy",
"dists")
working_dists = get_working_dists(archive_config)
os.rename(backup_dists, working_dists)
write_marker_file([working_dists, "marker"], "Recovered")
script.recoverWorkingDists()
self.assertEqual("Recovered",
read_marker_file([backup_dists, "marker"]))
self.assertNotEqual('', script.logger.getLogBuffer())
def test_recoverWorkingDists_recovers_working_directory(self):
distro = self.makeDistroWithPublishDirectory()
script = self.makeScript(distro)
script.setUp()
script.logger = BufferLogger()
script.logger.setLevel(logging.INFO)
script.setUpDirs()
archive_config = getPubConfig(distro.main_archive)
backup_dists = os.path.join(
archive_config.archiveroot + "-distscopy", "dists")
working_dists = get_working_dists(archive_config)
os.rename(backup_dists, working_dists)
write_marker_file([working_dists, "marker"], "Recovered")
script.recoverWorkingDists()
self.assertEqual(
"Recovered", read_marker_file([backup_dists, "marker"]))
self.assertNotEqual('', script.logger.getLogBuffer())
def test_script_creates_indexes(self):
# End-to-end test: the script creates indexes for distroseries
# that need them.
test_publisher = SoyuzTestPublisher()
series = test_publisher.setUpDefaultDistroSeries()
series.status = SeriesStatus.FROZEN
self.factory.makeComponentSelection(distroseries=series,
component="main")
self.layer.txn.commit()
self.setUpForScriptRun(series.distribution)
script = self.makeScript(series.distribution)
script.main()
self.assertEqual([], script.listSuitesNeedingIndexes(series))
sources = os.path.join(
getPubConfig(series.main_archive).distsroot, series.name, "main",
"source", "Sources")
self.assertTrue(file_exists(sources))
def generateHtpasswd(self, ppa):
"""Generate a htpasswd file for `ppa`s `tokens`.
:param ppa: The context PPA (an `IArchive`).
:return: The filename of the htpasswd file that was generated.
"""
# Create a temporary file that will be a new .htpasswd.
pub_config = getPubConfig(ppa)
if not os.path.exists(pub_config.htaccessroot):
os.makedirs(pub_config.htaccessroot)
fd, temp_filename = tempfile.mkstemp(dir=pub_config.htaccessroot)
os.close(fd)
write_htpasswd(
temp_filename, htpasswd_credentials_for_archive(ppa))
return temp_filename
def test_script_creates_indexes(self):
# End-to-end test: the script creates indexes for distroseries
# that need them.
test_publisher = SoyuzTestPublisher()
series = test_publisher.setUpDefaultDistroSeries()
series.status = SeriesStatus.FROZEN
self.factory.makeComponentSelection(
distroseries=series, component="main")
self.layer.txn.commit()
self.setUpForScriptRun(series.distribution)
script = self.makeScript(series.distribution)
script.main()
self.assertEqual([], script.listSuitesNeedingIndexes(series))
sources = os.path.join(
getPubConfig(series.main_archive).distsroot,
series.name, "main", "source", "Sources")
self.assertTrue(file_exists(sources))
def test_primary_config(self):
# Primary archive configuration is correct.
primary_config = getPubConfig(self.ubuntutest.main_archive)
self.assertEqual(self.root, primary_config.distroroot)
archiveroot = self.root + "/ubuntutest"
self.assertEqual(archiveroot, primary_config.archiveroot)
self.assertEqual(archiveroot + "/pool", primary_config.poolroot)
self.assertEqual(archiveroot + "/dists", primary_config.distsroot)
self.assertEqual(archiveroot + "-overrides",
primary_config.overrideroot)
self.assertEqual(archiveroot + "-cache", primary_config.cacheroot)
self.assertEqual(archiveroot + "-misc", primary_config.miscroot)
self.assertEqual(archiveroot + "-germinate",
primary_config.germinateroot)
self.assertEqual(self.root + "/ubuntutest-temp",
primary_config.temproot)
self.assertEqual(archiveroot + "-uefi", primary_config.uefiroot)
def test_runFinalizeParts_quotes_archiveroots(self):
# Passing ARCHIVEROOTS to the finalize.d scripts is a bit
# difficult because the variable holds multiple values in a
# single, double-quoted string. Escaping and quoting a sequence
# of escaped and quoted items won't work.
# This test establishes how a script can sanely deal with the
# list. It'll probably go wrong if the configured archive root
# contains spaces and such, but should work with Unix-sensible
# paths.
distro = self.makeDistroWithPublishDirectory()
self.factory.makeArchive(distribution=distro,
purpose=ArchivePurpose.PARTNER)
script = self.makeScript(distro)
script.setUp()
script.setUpDirs()
# Create a run-parts script that creates marker files in each of
# the archive roots, and writes an expected string to them.
# Doesn't write to a marker file that already exists, because it
# might be a sign that the path it received is ridiculously
# wrong. Don't want to go overwriting random files now do we?
self.installRunPartsScript(
distro, "finalize.d",
dedent("""\
#!/bin/sh -e
MARKER_NAME="marker file"
for DIRECTORY in $ARCHIVEROOTS
do
MARKER="$DIRECTORY/$MARKER_NAME"
if [ -e "$MARKER" ]
then
echo "Marker file $MARKER already exists." >&2
exit 1
fi
echo "This is an archive root." >"$MARKER"
done
"""))
script.runFinalizeParts(distro)
for archive in [distro.main_archive, distro.getArchive("partner")]:
archive_root = getPubConfig(archive).archiveroot
self.assertEqual(
"This is an archive root.",
read_marker_file([archive_root, "marker file"]).rstrip(),
"Did not find expected marker for %s." % archive.purpose.title)
def test_runFinalizeParts_quotes_archiveroots(self):
# Passing ARCHIVEROOTS to the finalize.d scripts is a bit
# difficult because the variable holds multiple values in a
# single, double-quoted string. Escaping and quoting a sequence
# of escaped and quoted items won't work.
# This test establishes how a script can sanely deal with the
# list. It'll probably go wrong if the configured archive root
# contains spaces and such, but should work with Unix-sensible
# paths.
distro = self.makeDistroWithPublishDirectory()
self.factory.makeArchive(
distribution=distro, purpose=ArchivePurpose.PARTNER)
script = self.makeScript(distro)
script.setUp()
script.setUpDirs()
# Create a run-parts script that creates marker files in each of
# the archive roots, and writes an expected string to them.
# Doesn't write to a marker file that already exists, because it
# might be a sign that the path it received is ridiculously
# wrong. Don't want to go overwriting random files now do we?
self.installRunPartsScript(distro, "finalize.d", dedent("""\
#!/bin/sh -e
MARKER_NAME="marker file"
for DIRECTORY in $ARCHIVEROOTS
do
MARKER="$DIRECTORY/$MARKER_NAME"
if [ -e "$MARKER" ]
then
echo "Marker file $MARKER already exists." >&2
exit 1
fi
echo "This is an archive root." >"$MARKER"
done
"""))
script.runFinalizeParts(distro)
for archive in [distro.main_archive, distro.getArchive("partner")]:
archive_root = getPubConfig(archive).archiveroot
self.assertEqual(
"This is an archive root.",
read_marker_file([archive_root, "marker file"]).rstrip(),
"Did not find expected marker for %s."
% archive.purpose.title)
def test_sign_with_signing_key(self):
filename = os.path.join(getPubConfig(self.archive).archiveroot, "file")
write_file(filename, "contents")
self.assertIsNone(self.archive.signing_key)
self.useFixture(InProcessKeyServerFixture()).start()
key_path = os.path.join(gpgkeysdir, '*****@*****.**')
yield IArchiveSigningKey(self.archive).setSigningKey(
key_path, async_keyserver=True)
self.assertIsNotNone(self.archive.signing_key)
custom_processor = CustomUpload()
custom_processor.sign(self.archive, "suite", filename)
with open(filename) as cleartext_file:
cleartext = cleartext_file.read()
with open("%s.gpg" % filename) as signature_file:
signature = getUtility(IGPGHandler).getVerifiedSignature(
cleartext, signature_file.read())
self.assertEqual(self.archive.signing_key.fingerprint,
signature.fingerprint)
def test_primary_config(self):
# Primary archive configuration is correct.
primary_config = getPubConfig(self.ubuntutest.main_archive)
self.assertEqual(self.root, primary_config.distroroot)
archiveroot = self.root + "/ubuntutest"
self.assertEqual(archiveroot, primary_config.archiveroot)
self.assertEqual(archiveroot + "/pool", primary_config.poolroot)
self.assertEqual(archiveroot + "/dists", primary_config.distsroot)
self.assertEqual(
archiveroot + "-overrides", primary_config.overrideroot)
self.assertEqual(archiveroot + "-cache", primary_config.cacheroot)
self.assertEqual(archiveroot + "-misc", primary_config.miscroot)
self.assertEqual(
self.root + "/ubuntutest-temp", primary_config.temproot)
self.assertEqual(archiveroot + "-signing", primary_config.signingroot)
self.assertFalse(primary_config.signingautokey)
self.assertIs(None, primary_config.metaroot)
self.assertEqual(archiveroot + "-staging", primary_config.stagingroot)
def test_non_ubuntu_ppa(self):
"""A meta-data upload to a non-Ubuntu PPA is not published.
The meta-data directory is currently only defined for Ubuntu PPAs.
"""
archive = self.factory.makeArchive(
distribution=self.factory.makeDistribution())
packageupload = self.factory.makePackageUpload(archive=archive)
libraryfilealias = self.factory.makeLibraryFileAlias(db_only=True)
logger = BufferLogger()
MetaDataUpload(logger=logger).process(packageupload, libraryfilealias)
self.assertEqual(
"DEBUG Skipping meta-data for archive without metaroot.\n",
logger.getLogBuffer())
published_file = os.path.join(
getPubConfig(archive).distroroot, archive.owner.name, "meta",
archive.name, libraryfilealias.filename)
self.assertThat(published_file, Not(PathExists()))
def setUp(self):
super(TestSigningHelpers, self).setUp()
self.temp_dir = self.makeTemporaryDirectory()
self.distro = self.factory.makeDistribution()
db_pubconf = getUtility(IPublisherConfigSet).getByDistribution(
self.distro)
db_pubconf.root_dir = unicode(self.temp_dir)
self.archive = self.factory.makeArchive(distribution=self.distro,
purpose=ArchivePurpose.PRIMARY)
self.signing_dir = os.path.join(self.temp_dir,
self.distro.name + "-signing")
self.suite = "distroseries"
pubconf = getPubConfig(self.archive)
if not os.path.exists(pubconf.temproot):
os.makedirs(pubconf.temproot)
# CustomUpload.installFiles requires a umask of 0o022.
old_umask = os.umask(0o022)
self.addCleanup(os.umask, old_umask)
def publishToArchiveWithOverriddenDistsroot(self, archive):
"""Publish a test package to the specified archive.
Publishes a test package but overrides the distsroot.
:return: A tuple of the path to the overridden distsroot and the
configured distsroot, in that order.
"""
self.getPubSource(filecontent="flangetrousers", archive=archive)
self.layer.txn.commit()
pubconf = getPubConfig(archive)
tmp_path = os.path.join(pubconf.archiveroot, "tmpdistroot")
if os.path.exists(tmp_path):
shutil.rmtree(tmp_path)
os.makedirs(tmp_path)
myargs = ['-R', tmp_path]
if archive.purpose == ArchivePurpose.PARTNER:
myargs.append('--partner')
self.runPublishDistro(myargs)
return tmp_path, pubconf.distsroot
def setUp(self):
super(TestFTPArchive, self).setUp()
switch_dbuser(config.archivepublisher.dbuser)
self._distribution = getUtility(IDistributionSet)['ubuntutest']
self._archive = self._distribution.main_archive
self._config = getPubConfig(self._archive)
self._config.setupArchiveDirs()
self._sampledir = os.path.join(
config.root, "lib", "lp", "archivepublisher", "tests",
"apt-data")
self._distsdir = self._config.distsroot
self._confdir = self._config.miscroot
self._pooldir = self._config.poolroot
self._overdir = self._config.overrideroot
self._listdir = self._config.overrideroot
self._tempdir = self._config.temproot
self._logger = BufferLogger()
self._dp = DiskPool(self._pooldir, self._tempdir, self._logger)
self._publisher = SamplePublisher(self._archive)
def testEnsureHtaccess(self):
"""Ensure that the .htaccess file is generated correctly."""
# The publisher Config object does not have an interface, so we
# need to remove the security wrapper.
pub_config = getPubConfig(self.ppa)
filename = os.path.join(pub_config.htaccessroot, ".htaccess")
remove_if_exists(filename)
script = self.getScript()
script.ensureHtaccess(self.ppa)
self.addCleanup(remove_if_exists, filename)
contents = [
"",
"AuthType Basic",
"AuthName \"Token Required\"",
"AuthUserFile %s/.htpasswd" % pub_config.htaccessroot,
"Require valid-user",
"",
]
self.assertThat(filename, FileContains('\n'.join(contents)))
def test_copy_config(self):
# In the case of copy archives (used for rebuild testing) the
# archiveroot is of the form
# DISTROROOT/DISTRONAME-ARCHIVENAME/DISTRONAME.
copy_archive = getUtility(IArchiveSet).new(
purpose=ArchivePurpose.COPY, owner=self.ubuntutest.owner,
distribution=self.ubuntutest, name="rebuildtest99")
copy_config = getPubConfig(copy_archive)
self.assertEqual(self.root, copy_config.distroroot)
archiveroot = self.root + "/ubuntutest-rebuildtest99/ubuntutest"
self.assertEqual(archiveroot, copy_config.archiveroot)
self.assertEqual(archiveroot + "/pool", copy_config.poolroot)
self.assertEqual(archiveroot + "/dists", copy_config.distsroot)
self.assertEqual(
archiveroot + "-overrides", copy_config.overrideroot)
self.assertEqual(archiveroot + "-cache", copy_config.cacheroot)
self.assertEqual(archiveroot + "-misc", copy_config.miscroot)
self.assertEqual(
archiveroot + "-germinate", copy_config.germinateroot)
self.assertEqual(archiveroot + "-temp", copy_config.temproot)
self.assertIsNone(copy_config.uefiroot)
def replaceUpdatedHtpasswd(self, ppa, temp_htpasswd_file):
"""Compare the new and the old htpasswd and replace if changed.
:return: True if the file was replaced.
"""
if self.options.dryrun:
return False
# The publisher Config object does not have an
# interface, so we need to remove the security wrapper.
pub_config = getPubConfig(ppa)
htpasswd_filename = os.path.join(pub_config.htaccessroot, ".htpasswd")
if (not os.path.isfile(htpasswd_filename) or
not filecmp.cmp(htpasswd_filename, temp_htpasswd_file)):
# Atomically replace the old file or create a new file.
os.rename(temp_htpasswd_file, htpasswd_filename)
self.logger.debug("Replaced htpasswd for %s" % ppa.displayname)
return True
return False
def test_partner_config(self):
# Partner archive configuration is correct.
# The publisher config for PARTNER contains only 'partner' in its
# components. This prevents non-partner being published in the
# partner archive.
partner_archive = getUtility(IArchiveSet).getByDistroAndName(
self.ubuntutest, "partner")
partner_config = getPubConfig(partner_archive)
self.root = "/var/tmp/archive"
self.assertEqual(self.root, partner_config.distroroot)
archiveroot = self.root + "/ubuntutest-partner"
self.assertEqual(archiveroot, partner_config.archiveroot)
self.assertEqual(archiveroot + "/pool", partner_config.poolroot)
self.assertEqual(archiveroot + "/dists", partner_config.distsroot)
self.assertIsNone(partner_config.overrideroot)
self.assertIsNone(partner_config.cacheroot)
self.assertIsNone(partner_config.miscroot)
self.assertIsNone(partner_config.germinateroot)
self.assertEqual(
self.root + "/ubuntutest-temp", partner_config.temproot)
self.assertEqual(archiveroot + "-uefi", partner_config.uefiroot)
def getPublisher(archive, allowed_suites, log, distsroot=None):
"""Return an initialized Publisher instance for the given context.
The callsites can override the location where the archive indexes will
be stored via 'distroot' argument.
"""
if archive.purpose != ArchivePurpose.PPA:
log.debug("Finding configuration for %s %s."
% (archive.distribution.name, archive.displayname))
else:
log.debug("Finding configuration for '%s' PPA."
% archive.owner.name)
pubconf = getPubConfig(archive)
disk_pool = _getDiskPool(pubconf, log)
if distsroot is not None:
log.debug("Overriding dists root with %s." % distsroot)
pubconf.distsroot = distsroot
log.debug("Preparing publisher.")
return Publisher(log, pubconf, disk_pool, archive, allowed_suites)
def _archive_root_path(self):
return getPubConfig(self.archive).archiveroot
def test_getPubConfig_returns_None_if_no_publisherconfig_found(self):
archive = self.factory.makeDistribution(no_pubconf=True).main_archive
self.assertEqual(None, getPubConfig(archive))
def setUp(self):
super(TestGetPubConfigPPA, self).setUp()
self.ubuntutest = getUtility(IDistributionSet)['ubuntutest']
self.ppa = self.factory.makeArchive(
distribution=self.ubuntutest, purpose=ArchivePurpose.PPA)
self.ppa_config = getPubConfig(self.ppa)