def differ_in_content(one_file, other_file):
"""Do the two named files have different contents?"""
one_exists = file_exists(one_file)
other_exists = file_exists(other_file)
if any([one_exists, other_exists]):
return (one_exists != other_exists
or file(one_file).read() != file(other_file).read())
else:
return False
def differ_in_content(one_file, other_file):
"""Do the two named files have different contents?"""
one_exists = file_exists(one_file)
other_exists = file_exists(other_file)
if any([one_exists, other_exists]):
return (
one_exists != other_exists or
file(one_file).read() != file(other_file).read())
else:
return False
def setUpDirs(self):
"""Create output directories if they did not already exist."""
germinateroot = self.config.germinateroot
if not file_exists(germinateroot):
self.logger.debug("Creating germinate root %s.", germinateroot)
os.makedirs(germinateroot)
miscroot = self.config.miscroot
if not file_exists(miscroot):
self.logger.debug("Creating misc root %s.", miscroot)
os.makedirs(miscroot)
def setUpDirs(self):
"""Create output directories if they did not already exist."""
germinateroot = self.config.germinateroot
if not file_exists(germinateroot):
self.logger.debug("Creating germinate root %s.", germinateroot)
os.makedirs(germinateroot)
miscroot = self.config.miscroot
if not file_exists(miscroot):
self.logger.debug("Creating misc root %s.", miscroot)
os.makedirs(miscroot)
def test_setUpDirs_creates_directory_structure(self):
distro = self.makeDistroWithPublishDirectory()
pub_config = get_pub_config(distro)
archive_root = get_archive_root(pub_config)
dists_root = get_dists_root(pub_config)
script = self.makeScript(distro)
script.setUp()
self.assertFalse(file_exists(archive_root))
script.setUpDirs()
self.assertTrue(file_exists(archive_root))
self.assertTrue(file_exists(dists_root))
self.assertTrue(file_exists(get_distscopy_root(pub_config)))
def test_setUpDirs_creates_directory_structure(self):
distro = self.makeDistroWithPublishDirectory()
pub_config = get_pub_config(distro)
archive_root = get_archive_root(pub_config)
dists_root = get_dists_root(pub_config)
script = self.makeScript(distro)
script.setUp()
self.assertFalse(file_exists(archive_root))
script.setUpDirs()
self.assertTrue(file_exists(archive_root))
self.assertTrue(file_exists(dists_root))
self.assertTrue(file_exists(get_distscopy_root(pub_config)))
def test_setUpDirs_does_not_mind_if_dist_directories_already_exist(self):
distro = self.makeDistroWithPublishDirectory()
script = self.makeScript(distro)
script.setUp()
script.setUpDirs()
script.setUpDirs()
self.assertTrue(file_exists(get_archive_root(get_pub_config(distro))))
def test_atomic_file_removes_dot_new(self):
# AtomicFile does not leave .new files lying around.
self.useTempDir()
filename = self.factory.getUniqueString()
with AtomicFile(filename):
pass
self.assertFalse(file_exists("%s.new" % filename))
def listSuitesNeedingIndexes(self, distroseries):
"""Find suites in `distroseries` that need indexes created.
Checks for the marker left by `markIndexCreationComplete`.
"""
if distroseries.status != SeriesStatus.FROZEN:
# DistroSeries are created in Frozen state. If the series
# is in any other state yet has not been marked as having
# its indexes created, that's because it predates automatic
# index creation.
return []
distro = distroseries.distribution
publisher_config_set = getUtility(IPublisherConfigSet)
if publisher_config_set.getByDistribution(distro) is None:
# We won't be able to do a thing without a publisher config,
# but that's alright: we have those for all distributions
# that we want to publish.
return []
# May need indexes for this series.
suites = [
distroseries.getSuite(pocket)
for pocket in pocketsuffix.iterkeys()
]
return [
suite for suite in suites
if not file_exists(self.locateIndexesMarker(distro, suite))
]
def getSuites(self):
"""Return suites that are actually supported in this distribution."""
for series in self.getSupportedSeries():
for pocket in PackagePublishingPocket.items:
suite = series.getSuite(pocket)
if file_exists(os.path.join(self.config.distsroot, suite)):
yield suite
def test_setUpDirs_does_not_mind_if_dist_directories_already_exist(self):
distro = self.makeDistroWithPublishDirectory()
script = self.makeScript(distro)
script.setUp()
script.setUpDirs()
script.setUpDirs()
self.assertTrue(file_exists(get_archive_root(get_pub_config(distro))))
def getSuites(self):
"""Return suites that are actually supported in this distribution."""
for series in self.getSupportedSeries():
for pocket in PackagePublishingPocket.items:
suite = series.getSuite(pocket)
if file_exists(os.path.join(self.config.distsroot, suite)):
yield suite
def test_executeShell_executes_shell_command(self):
distro = self.makeDistroWithPublishDirectory()
script = self.makeScript(distro)
marker = os.path.join(
get_pub_config(distro).root_dir, "marker")
script.executeShell("touch %s" % marker)
self.assertTrue(file_exists(marker))
def listSuitesNeedingIndexes(self, distroseries):
"""Find suites in `distroseries` that need indexes created.
Checks for the marker left by `markIndexCreationComplete`.
"""
if distroseries.status != SeriesStatus.FROZEN:
# DistroSeries are created in Frozen state. If the series
# is in any other state yet has not been marked as having
# its indexes created, that's because it predates automatic
# index creation.
return []
distro = distroseries.distribution
publisher_config_set = getUtility(IPublisherConfigSet)
if publisher_config_set.getByDistribution(distro) is None:
# We won't be able to do a thing without a publisher config,
# but that's alright: we have those for all distributions
# that we want to publish.
return []
# May need indexes for this series.
suites = [
distroseries.getSuite(pocket)
for pocket in pocketsuffix.iterkeys()]
return [
suite for suite in suites
if not file_exists(self.locateIndexesMarker(distro, suite))]
def test_execute_executes_command(self):
# execute really does execute its command. If we tell it to
# "touch" a new file, that file really gets created.
self.useTempDir()
logger = DevNullLogger()
filename = self.factory.getUniqueString()
execute(logger, "touch", [filename])
self.assertTrue(file_exists(filename))
def setUpContentArchive(self):
"""Make sure the `content_archive` directories exist."""
self.logger.debug("Ensuring that we have a private tree in place.")
for suffix in ['cache', 'misc']:
dirname = '-'.join([self.distribution.name, suffix])
path = os.path.join(self.content_archive, dirname)
if not file_exists(path):
os.makedirs(path)
def setUpContentArchive(self):
"""Make sure the `content_archive` directories exist."""
self.logger.debug("Ensuring that we have a private tree in place.")
for suffix in ['cache', 'misc']:
dirname = '-'.join([self.distribution.name, suffix])
path = os.path.join(self.content_archive, dirname)
if not file_exists(path):
os.makedirs(path)
def test_execute_executes_command(self):
# execute really does execute its command. If we tell it to
# "touch" a new file, that file really gets created.
self.useTempDir()
logger = DevNullLogger()
filename = self.factory.getUniqueString()
execute(logger, "touch", [filename])
self.assertTrue(file_exists(filename))
def setUpDirs(self):
"""Create archive roots and such if they did not yet exist."""
for distro_configs in self.configs.itervalues():
for archive_purpose, archive_config in distro_configs.iteritems():
archiveroot = archive_config.archiveroot
if not file_exists(archiveroot):
self.logger.debug("Creating archive root %s.", archiveroot)
os.makedirs(archiveroot)
dists = get_dists(archive_config)
if not file_exists(dists):
self.logger.debug("Creating dists root %s.", dists)
os.makedirs(dists)
distscopy = get_backup_dists(archive_config)
if not file_exists(distscopy):
self.logger.debug("Creating backup dists directory %s",
distscopy)
os.makedirs(distscopy)
def setUpDirs(self):
"""Create archive roots and such if they did not yet exist."""
for distro_configs in self.configs.itervalues():
for archive_purpose, archive_config in distro_configs.iteritems():
archiveroot = archive_config.archiveroot
if not file_exists(archiveroot):
self.logger.debug(
"Creating archive root %s.", archiveroot)
os.makedirs(archiveroot)
dists = get_dists(archive_config)
if not file_exists(dists):
self.logger.debug("Creating dists root %s.", dists)
os.makedirs(dists)
distscopy = get_backup_dists(archive_config)
if not file_exists(distscopy):
self.logger.debug(
"Creating backup dists directory %s", distscopy)
os.makedirs(distscopy)
def test_clearEmptyDirs_cleans_up_empty_directories(self):
distro = self.makeDistroWithPublishDirectory()
script = self.makeScript(distro)
script.setUp()
script.setUpDirs()
empty_dir = os.path.join(get_dists_root(get_pub_config(distro)),
'empty-dir')
os.makedirs(empty_dir)
script.clearEmptyDirs(distro)
self.assertFalse(file_exists(empty_dir))
def test_clearEmptyDirs_cleans_up_empty_directories(self):
distro = self.makeDistroWithPublishDirectory()
script = self.makeScript(distro)
script.setUp()
script.setUpDirs()
empty_dir = os.path.join(
get_dists_root(get_pub_config(distro)), 'empty-dir')
os.makedirs(empty_dir)
script.clearEmptyDirs(distro)
self.assertFalse(file_exists(empty_dir))
def test_main(self):
# If run end-to-end, the script generates Contents.gz files, and a
# following publisher run will put those files in their final place
# and include them in the Release file.
distro = self.makeDistro()
distroseries = self.factory.makeDistroSeries(distribution=distro)
processor = self.factory.makeProcessor()
das = self.factory.makeDistroArchSeries(distroseries=distroseries,
processor=processor)
package = self.factory.makeSuiteSourcePackage(distroseries)
self.factory.makeSourcePackagePublishingHistory(
distroseries=distroseries, pocket=package.pocket)
self.factory.makeBinaryPackageBuild(distroarchseries=das,
pocket=package.pocket,
processor=processor)
suite = package.suite
script = self.makeScript(distro)
os.makedirs(os.path.join(script.config.distsroot, package.suite))
self.assertNotEqual([], list(script.getSuites()))
fake_overrides(script, distroseries)
script.process()
self.assertTrue(
file_exists(
os.path.join(script.config.stagingroot, suite,
"Contents-%s.gz" % das.architecturetag)))
publisher_script = PublishFTPMaster(test_args=["-d", distro.name])
publisher_script.txn = self.layer.txn
publisher_script.logger = DevNullLogger()
publisher_script.main()
contents_path = os.path.join(script.config.distsroot, suite,
"Contents-%s.gz" % das.architecturetag)
self.assertTrue(file_exists(contents_path))
with open(contents_path, "rb") as contents_file:
contents_bytes = contents_file.read()
release_path = os.path.join(script.config.distsroot, suite, "Release")
self.assertTrue(file_exists(release_path))
with open(release_path) as release_file:
release_lines = release_file.readlines()
self.assertIn(
" %s %16s Contents-%s.gz\n" %
(hashlib.md5(contents_bytes).hexdigest(), len(contents_bytes),
das.architecturetag), release_lines)
def find_run_parts_dir(distro, parts):
"""Find the requested run-parts directory, if it exists."""
run_parts_location = config.archivepublisher.run_parts_location
if not run_parts_location:
return
parts_dir = os.path.join(run_parts_location, distro.name, parts)
if file_exists(parts_dir):
return parts_dir
else:
return None
def test_clearEmptyDirs_does_not_clean_up_nonempty_directories(self):
distro = self.makeDistroWithPublishDirectory()
script = self.makeScript(distro)
script.setUp()
script.setUpDirs()
nonempty_dir = os.path.join(
get_dists_root(get_pub_config(distro)), 'nonempty-dir')
os.makedirs(nonempty_dir)
write_marker_file([nonempty_dir, "placeholder"], "Data here!")
script.clearEmptyDirs(distro)
self.assertTrue(file_exists(nonempty_dir))
def createComponentDirs(self, suites):
"""Create the content archive's tree for all of its components."""
for suite in suites:
for component in COMPONENTS:
for directory in self.getDirs(self.getArchs(suite)):
path = os.path.join(self.content_archive,
self.distribution.name, 'dists', suite,
component, directory)
if not file_exists(path):
self.logger.debug("Creating %s.", path)
os.makedirs(path)
def test_clearEmptyDirs_does_not_clean_up_nonempty_directories(self):
distro = self.makeDistroWithPublishDirectory()
script = self.makeScript(distro)
script.setUp()
script.setUpDirs()
nonempty_dir = os.path.join(get_dists_root(get_pub_config(distro)),
'nonempty-dir')
os.makedirs(nonempty_dir)
write_marker_file([nonempty_dir, "placeholder"], "Data here!")
script.clearEmptyDirs(distro)
self.assertTrue(file_exists(nonempty_dir))
def createComponentDirs(self, suites):
"""Create the content archive's tree for all of its components."""
for suite in suites:
for component in COMPONENTS:
for directory in self.getDirs(self.getArchs(suite)):
path = os.path.join(
self.content_archive, self.distribution.name, 'dists',
suite, component, directory)
if not file_exists(path):
self.logger.debug("Creating %s.", path)
os.makedirs(path)
def find_run_parts_dir(distro, parts):
"""Find the requested run-parts directory, if it exists."""
run_parts_location = config.archivepublisher.run_parts_location
if not run_parts_location:
return
parts_dir = os.path.join(run_parts_location, distro.name, parts)
if file_exists(parts_dir):
return parts_dir
else:
return None
def getSuites(self):
"""Return suites that need Contents files."""
# XXX cjwatson 2015-09-23: This script currently only supports the
# primary archive.
archive = self.distribution.main_archive
for series in self.distribution.getNonObsoleteSeries():
for pocket in PackagePublishingPocket.items:
suite = series.getSuite(pocket)
if cannot_modify_suite(archive, series, pocket):
continue
if file_exists(os.path.join(self.config.distsroot, suite)):
yield suite
def recoverArchiveWorkingDir(self, archive_config):
"""Recover working dists dir for `archive_config`.
If there is a dists directory for `archive_config` in the working
location, kick it back to the backup location.
"""
working_location = get_working_dists(archive_config)
if file_exists(working_location):
self.logger.info(
"Recovering working directory %s from failed run.",
working_location)
os.rename(working_location, get_backup_dists(archive_config))
def recoverArchiveWorkingDir(self, archive_config):
"""Recover working dists dir for `archive_config`.
If there is a dists directory for `archive_config` in the working
location, kick it back to the backup location.
"""
working_location = get_working_dists(archive_config)
if file_exists(working_location):
self.logger.info(
"Recovering working directory %s from failed run.",
working_location)
os.rename(working_location, get_backup_dists(archive_config))
def copyOverrides(self, override_root):
"""Copy overrides into the content archive.
This method won't access the database.
"""
if file_exists(override_root):
execute(self.logger, "cp", [
"-a",
override_root,
"%s/" % self.content_archive,
])
else:
self.logger.debug("Did not find overrides; not copying.")
def copyOverrides(self, override_root):
"""Copy overrides into the content archive.
This method won't access the database.
"""
if file_exists(override_root):
execute(self.logger, "cp", [
"-a",
override_root,
"%s/" % self.content_archive,
])
else:
self.logger.debug("Did not find overrides; not copying.")
def _writeReleaseFile(self, suite, release_data):
"""Write a Release file to the archive.
:param suite: The name of the suite whose Release file is to be
written.
:param release_data: A `debian.deb822.Release` object to write
to the filesystem.
"""
location = os.path.join(self._config.distsroot, suite)
if not file_exists(location):
os.makedirs(location)
with open(os.path.join(location, "Release"), "w") as release_file:
release_data.dump(release_file, "utf-8")
def generateListings(self, distribution):
"""Create ls-lR.gz listings."""
self.logger.debug("Creating ls-lR.gz...")
lslr = "ls-lR.gz"
lslr_new = "." + lslr + ".new"
for purpose, archive_config in self.configs[distribution].iteritems():
lslr_file = os.path.join(archive_config.archiveroot, lslr)
new_lslr_file = os.path.join(archive_config.archiveroot, lslr_new)
if file_exists(new_lslr_file):
os.remove(new_lslr_file)
self.executeShell("cd -- '%s' ; TZ=UTC ls -lR | gzip -9n >'%s'" %
(archive_config.archiveroot, lslr_new),
failure=LaunchpadScriptFailure(
"Failed to create %s for %s." %
(lslr, purpose.title)))
os.rename(new_lslr_file, lslr_file)
def generateListings(self, distribution):
"""Create ls-lR.gz listings."""
self.logger.debug("Creating ls-lR.gz...")
lslr = "ls-lR.gz"
lslr_new = "." + lslr + ".new"
for purpose, archive_config in self.configs[distribution].iteritems():
lslr_file = os.path.join(archive_config.archiveroot, lslr)
new_lslr_file = os.path.join(archive_config.archiveroot, lslr_new)
if file_exists(new_lslr_file):
os.remove(new_lslr_file)
self.executeShell(
"cd -- '%s' ; TZ=UTC ls -lR | gzip -9n >'%s'"
% (archive_config.archiveroot, lslr_new),
failure=LaunchpadScriptFailure(
"Failed to create %s for %s." % (lslr, purpose.title)))
os.rename(new_lslr_file, lslr_file)
def test_script_creates_indexes(self):
# End-to-end test: the script creates indexes for distroseries
# that need them.
test_publisher = SoyuzTestPublisher()
series = test_publisher.setUpDefaultDistroSeries()
series.status = SeriesStatus.FROZEN
self.factory.makeComponentSelection(
distroseries=series, component="main")
self.layer.txn.commit()
self.setUpForScriptRun(series.distribution)
script = self.makeScript(series.distribution)
script.main()
self.assertEqual([], script.listSuitesNeedingIndexes(series))
sources = os.path.join(
getPubConfig(series.main_archive).distsroot,
series.name, "main", "source", "Sources")
self.assertTrue(file_exists(sources))
def test_script_creates_indexes(self):
# End-to-end test: the script creates indexes for distroseries
# that need them.
test_publisher = SoyuzTestPublisher()
series = test_publisher.setUpDefaultDistroSeries()
series.status = SeriesStatus.FROZEN
self.factory.makeComponentSelection(distroseries=series,
component="main")
self.layer.txn.commit()
self.setUpForScriptRun(series.distribution)
script = self.makeScript(series.distribution)
script.main()
self.assertEqual([], script.listSuitesNeedingIndexes(series))
sources = os.path.join(
getPubConfig(series.main_archive).distsroot, series.name, "main",
"source", "Sources")
self.assertTrue(file_exists(sources))
def test_germinate_output(self):
# A single call to germinateArch produces output for all flavours on
# one architecture.
self.setUpDistroAndScript()
series_name = self.distroseries[0].name
component = self.setUpComponent()
das = self.factory.makeDistroArchSeries(
distroseries=self.distroseries[0])
arch = das.architecturetag
one = self.makePackage(component, [das])
two = self.makePackage(component, [das])
self.makeIndexFiles(self.script, self.distroseries[0])
flavour_one = self.factory.getUniqueString()
flavour_two = self.factory.getUniqueString()
seed = self.factory.getUniqueString()
self.makeSeedStructure(flavour_one, series_name, [seed])
self.makeSeed(flavour_one, series_name, seed, [one.name])
self.makeSeedStructure(flavour_two, series_name, [seed])
self.makeSeed(flavour_two, series_name, seed, [two.name])
overrides = self.fetchGerminatedOverrides(
self.script, self.distroseries[0], arch,
[flavour_one, flavour_two])
self.assertEqual([], overrides)
seed_dir_one = os.path.join(
self.seeddir, "%s.%s" % (flavour_one, series_name))
self.assertFilesEqual(
os.path.join(seed_dir_one, "STRUCTURE"),
self.script.composeOutputPath(
flavour_one, series_name, arch, "structure"))
self.assertTrue(file_exists(self.script.composeOutputPath(
flavour_one, series_name, arch, "all")))
self.assertTrue(file_exists(self.script.composeOutputPath(
flavour_one, series_name, arch, "all.sources")))
self.assertTrue(file_exists(self.script.composeOutputPath(
flavour_one, series_name, arch, seed)))
seed_dir_two = os.path.join(
self.seeddir, "%s.%s" % (flavour_two, series_name))
self.assertFilesEqual(
os.path.join(seed_dir_two, "STRUCTURE"),
self.script.composeOutputPath(
flavour_two, series_name, arch, "structure"))
self.assertTrue(file_exists(self.script.composeOutputPath(
flavour_two, series_name, arch, "all")))
self.assertTrue(file_exists(self.script.composeOutputPath(
flavour_two, series_name, arch, "all.sources")))
self.assertTrue(file_exists(self.script.composeOutputPath(
flavour_two, series_name, arch, seed)))
def test_main(self):
# If run end-to-end, the script generates Contents.gz files.
distro = self.makeDistro()
distroseries = self.factory.makeDistroSeries(distribution=distro)
processor = self.factory.makeProcessor()
das = self.factory.makeDistroArchSeries(
distroseries=distroseries, processor=processor)
package = self.factory.makeSuiteSourcePackage(distroseries)
self.factory.makeSourcePackagePublishingHistory(
distroseries=distroseries, pocket=package.pocket)
self.factory.makeBinaryPackageBuild(
distroarchseries=das, pocket=package.pocket,
processor=processor)
suite = package.suite
script = self.makeScript(distro)
os.makedirs(os.path.join(script.config.distsroot, package.suite))
self.assertNotEqual([], list(script.getSuites()))
fake_overrides(script, distroseries)
script.process()
self.assertTrue(file_exists(os.path.join(
script.config.distsroot, suite,
"Contents-%s.gz" % das.architecturetag)))
def assertFileRenamed(ignored):
self.assertFalse(file_exists(orig_filename))
self.assertTrue(file_exists(new_filename))
def assertFileRemoved(ignored):
self.assertFalse(file_exists(filename))
def test_is_not_upset_by_missing_directory(self):
self.assertFalse(
file_exists("a-nonexistent-directory/a-nonexistent-file.txt"))
def test_says_no_if_not_found(self):
self.assertFalse(file_exists("a-nonexistent-file.txt"))
def test_finds_directory(self):
os.makedirs("a-real-directory")
self.assertTrue(file_exists("a-real-directory"))
def assertFileRenamed(ignored):
self.assertFalse(file_exists(orig_filename))
self.assertTrue(file_exists(new_filename))
def test_finds_file(self):
file("a-real-file.txt", "w").write("Here I am.")
self.assertTrue(file_exists("a-real-file.txt"))
def test_finds_directory(self):
os.makedirs("a-real-directory")
self.assertTrue(file_exists("a-real-directory"))
def test_says_no_if_not_found(self):
self.assertFalse(file_exists("a-nonexistent-file.txt"))
def test_is_not_upset_by_missing_directory(self):
self.assertFalse(
file_exists("a-nonexistent-directory/a-nonexistent-file.txt"))
def assertDirectoryRemoved(ignored):
self.assertFalse(file_exists(directory))
def assertFileRemoved(ignored):
self.assertFalse(file_exists(filename))
def path_exists(*path_components):
"""Does the given file or directory exist?"""
return file_exists(os.path.join(*path_components))
def assertDirectoryRemoved(ignored):
self.assertFalse(file_exists(directory))
def test_finds_file(self):
file("a-real-file.txt", "w").write("Here I am.")
self.assertTrue(file_exists("a-real-file.txt"))